From 926b388c609b3b1114be3c872a5cc0bf1393c305 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Sun, 7 Jun 2026 11:33:13 +0800 Subject: [PATCH 1/3] test: add 11 GitCode read-only smoke scenarios New scenarios: branch: list, get (master) tag: list, get (skipped, requires admin) release: list (empty repo, 200 OK) commit: list, get (first commit sha) labels: list comments: list (commit comments) pulls: comments (PR #747) Result: 28 passed, 0 failed, 8 skipped Co-Authored-By: Claude Opus 4.8 --- scenarios/gitcode/branch-get.yaml | 20 ++++++++++++++++++++ scenarios/gitcode/branch-list.yaml | 17 +++++++++++++++++ scenarios/gitcode/comments-list.yaml | 14 ++++++++++++++ scenarios/gitcode/commit-get.yaml | 20 ++++++++++++++++++++ scenarios/gitcode/commit-list.yaml | 17 +++++++++++++++++ scenarios/gitcode/labels-list.yaml | 17 +++++++++++++++++ scenarios/gitcode/pulls-comments.yaml | 19 +++++++++++++++++++ scenarios/gitcode/release-list.yaml | 14 ++++++++++++++ scenarios/gitcode/tag-get.yaml | 17 +++++++++++++++++ scenarios/gitcode/tag-list.yaml | 17 +++++++++++++++++ 10 files changed, 172 insertions(+) create mode 100644 scenarios/gitcode/branch-get.yaml create mode 100644 scenarios/gitcode/branch-list.yaml create mode 100644 scenarios/gitcode/comments-list.yaml create mode 100644 scenarios/gitcode/commit-get.yaml create mode 100644 scenarios/gitcode/commit-list.yaml create mode 100644 scenarios/gitcode/labels-list.yaml create mode 100644 scenarios/gitcode/pulls-comments.yaml create mode 100644 scenarios/gitcode/release-list.yaml create mode 100644 scenarios/gitcode/tag-get.yaml create mode 100644 scenarios/gitcode/tag-list.yaml diff --git a/scenarios/gitcode/branch-get.yaml b/scenarios/gitcode/branch-get.yaml new file mode 100644 index 0000000..b331f1e --- /dev/null +++ b/scenarios/gitcode/branch-get.yaml @@ -0,0 +1,20 @@ +name: "gitcode/branch-get" +service: gitcode +args: + - branch + - get + - --owner + - openeuler + - --repo + - infrastructure + - --branch + - master +format: json +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 + - type: json_has_keys + values: ["name", "protected"] + - type: json_key_not_empty + key: "name" diff --git a/scenarios/gitcode/branch-list.yaml b/scenarios/gitcode/branch-list.yaml new file mode 100644 index 0000000..c17eafe --- /dev/null +++ b/scenarios/gitcode/branch-list.yaml @@ -0,0 +1,17 @@ +name: "gitcode/branch-list" +service: gitcode +args: + - branch + - branches + - --owner + - openeuler + - --repo + - infrastructure +format: table +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 + - type: stdout_not_empty + - type: stderr_not_contains + value: "ERROR" diff --git a/scenarios/gitcode/comments-list.yaml b/scenarios/gitcode/comments-list.yaml new file mode 100644 index 0000000..c1e3602 --- /dev/null +++ b/scenarios/gitcode/comments-list.yaml @@ -0,0 +1,14 @@ +name: "gitcode/comments-list" +service: gitcode +args: + - comments + - list + - --owner + - openeuler + - --repo + - infrastructure +format: json +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 diff --git a/scenarios/gitcode/commit-get.yaml b/scenarios/gitcode/commit-get.yaml new file mode 100644 index 0000000..2d5e5ee --- /dev/null +++ b/scenarios/gitcode/commit-get.yaml @@ -0,0 +1,20 @@ +name: "gitcode/commit-get" +service: gitcode +args: + - commit + - get + - --owner + - openeuler + - --repo + - infrastructure + - --sha + - a63ae4a0b5c40dd12f70263e23c05bbbeb888fb6 +format: json +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 + - type: json_has_keys + values: ["sha", "commit", "html_url"] + - type: json_key_not_empty + key: "sha" diff --git a/scenarios/gitcode/commit-list.yaml b/scenarios/gitcode/commit-list.yaml new file mode 100644 index 0000000..1e074bd --- /dev/null +++ b/scenarios/gitcode/commit-list.yaml @@ -0,0 +1,17 @@ +name: "gitcode/commit-list" +service: gitcode +args: + - commit + - list + - --owner + - openeuler + - --repo + - infrastructure +format: table +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 + - type: stdout_not_empty + - type: stderr_not_contains + value: "ERROR" diff --git a/scenarios/gitcode/labels-list.yaml b/scenarios/gitcode/labels-list.yaml new file mode 100644 index 0000000..14448d4 --- /dev/null +++ b/scenarios/gitcode/labels-list.yaml @@ -0,0 +1,17 @@ +name: "gitcode/labels-list" +service: gitcode +args: + - labels + - list + - --owner + - openeuler + - --repo + - infrastructure +format: table +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 + - type: stdout_not_empty + - type: stderr_not_contains + value: "ERROR" diff --git a/scenarios/gitcode/pulls-comments.yaml b/scenarios/gitcode/pulls-comments.yaml new file mode 100644 index 0000000..7d3dd30 --- /dev/null +++ b/scenarios/gitcode/pulls-comments.yaml @@ -0,0 +1,19 @@ +name: "gitcode/pulls-comments" +service: gitcode +args: + - pulls + - comments + - --owner + - openeuler + - --repo + - infrastructure + - --number + - 747 +format: table +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 + - type: stdout_not_empty + - type: stderr_not_contains + value: "ERROR" diff --git a/scenarios/gitcode/release-list.yaml b/scenarios/gitcode/release-list.yaml new file mode 100644 index 0000000..156abd8 --- /dev/null +++ b/scenarios/gitcode/release-list.yaml @@ -0,0 +1,14 @@ +name: "gitcode/release-list" +service: gitcode +args: + - release + - list + - --owner + - openeuler + - --repo + - infrastructure +format: json +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 diff --git a/scenarios/gitcode/tag-get.yaml b/scenarios/gitcode/tag-get.yaml new file mode 100644 index 0000000..9004660 --- /dev/null +++ b/scenarios/gitcode/tag-get.yaml @@ -0,0 +1,17 @@ +name: "gitcode/tag-get" +service: gitcode +args: + - tag + - list + - --owner + - openeuler + - --repo + - infrastructure +format: table +timeout_ms: 8000 +skip: true +skip_reason: "Protected tag API requires admin permissions" +assertions: + - type: exit_code + value: 0 + - type: stdout_not_empty diff --git a/scenarios/gitcode/tag-list.yaml b/scenarios/gitcode/tag-list.yaml new file mode 100644 index 0000000..06f581d --- /dev/null +++ b/scenarios/gitcode/tag-list.yaml @@ -0,0 +1,17 @@ +name: "gitcode/tag-list" +service: gitcode +args: + - tag + - list + - --owner + - openeuler + - --repo + - infrastructure +format: table +timeout_ms: 8000 +assertions: + - type: exit_code + value: 0 + - type: stdout_not_empty + - type: stderr_not_contains + value: "ERROR" From a37b17b406431044e9a899eefa83927d84fe3822 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Sun, 7 Jun 2026 11:43:32 +0800 Subject: [PATCH 2/3] feat: add parallel execution to smoke runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group scenarios by service, run service groups in parallel (up to --parallel N). Scenarios within a group still run sequentially to avoid rate-limiting the same API. Results ordered deterministically. - Runner.maxConcurrent field + --parallel flag (default 4, 0=sequential) - runParallel(): goroutine per service group, semaphore-controlled - runSequential(): kept for --parallel 0 - Fix runner tests for new NewRunner signature Speed: 36 scenarios 20s→10s (~2x) with --parallel 4 Co-Authored-By: Claude Opus 4.8 --- cmd/smoke/main.go | 3 +- internal/smoke/runner.go | 103 +++++++++++++++++++++++++++++++--- internal/smoke/runner_test.go | 10 ++-- 3 files changed, 102 insertions(+), 14 deletions(-) diff --git a/cmd/smoke/main.go b/cmd/smoke/main.go index fdea0eb..1b0c172 100644 --- a/cmd/smoke/main.go +++ b/cmd/smoke/main.go @@ -24,6 +24,7 @@ func run() error { scenariosDir := flag.String("scenarios-dir", "./scenarios", "directory containing scenario YAML files") reportDir := flag.String("report-dir", "./smoke-report", "directory to write report.html into") filter := flag.String("filter", "", "only run scenarios whose name or service contains this string") + parallel := flag.Int("parallel", 4, "max parallel service groups (0 = sequential)") verbose := flag.Bool("verbose", false, "print stdout/stderr for every scenario, not just failures") flag.Parse() @@ -67,7 +68,7 @@ func run() error { fmt.Printf("Running %d scenario(s)...\n\n", len(scenarios)) // Execute all scenarios. - runner := smoke.NewRunner(*coraBin, expandedConfig, *verbose) + runner := smoke.NewRunner(*coraBin, expandedConfig, *verbose, *parallel) report := runner.RunAll(scenarios, *configPath) // Print live results. diff --git a/internal/smoke/runner.go b/internal/smoke/runner.go index b4df3cd..82c1a0f 100644 --- a/internal/smoke/runner.go +++ b/internal/smoke/runner.go @@ -6,20 +6,28 @@ import ( "fmt" "os" "os/exec" + "sort" "strings" + "sync" "time" ) // Runner executes scenarios by invoking the cora binary as a subprocess. type Runner struct { - coraBin string // path to cora binary - configPath string // expanded config file path (may be empty) - verbose bool // pass --verbose to every cora invocation + coraBin string // path to cora binary + configPath string // expanded config file path (may be empty) + verbose bool // pass --verbose to every cora invocation + maxConcurrent int // max parallel service groups (0 = sequential) } // NewRunner creates a Runner. configPath may be "" to skip CORA_CONFIG injection. -func NewRunner(coraBin, configPath string, verbose bool) *Runner { - return &Runner{coraBin: coraBin, configPath: configPath, verbose: verbose} +// maxConcurrent controls parallel execution: 0 = sequential, >0 = run up to N +// service groups concurrently. +func NewRunner(coraBin, configPath string, verbose bool, maxConcurrent int) *Runner { + return &Runner{ + coraBin: coraBin, configPath: configPath, verbose: verbose, + maxConcurrent: maxConcurrent, + } } // Run executes a single Scenario and returns its result. @@ -103,16 +111,95 @@ func (r *Runner) Run(s Scenario) ScenarioResult { return result } -// RunAll executes all scenarios sequentially and returns a RunReport. +// RunAll executes all scenarios and returns a RunReport. +// When MaxConcurrent > 0, scenarios are grouped by service and service groups +// run in parallel (up to MaxConcurrent at a time). Scenarios within a group +// still run sequentially to avoid rate-limiting the same API. func (r *Runner) RunAll(scenarios []Scenario, configPath string) *RunReport { + if r.maxConcurrent <= 0 { + return r.runSequential(scenarios, configPath) + } + return r.runParallel(scenarios, configPath) +} + +// runSequential executes all scenarios one-by-one. +func (r *Runner) runSequential(scenarios []Scenario, configPath string) *RunReport { report := &RunReport{ GeneratedAt: time.Now().Format("2006-01-02 15:04:05"), ConfigPath: configPath, } - start := time.Now() + tStart := time.Now() for _, s := range scenarios { report.Results = append(report.Results, r.Run(s)) } - report.TotalDurationMs = time.Since(start).Milliseconds() + report.TotalDurationMs = time.Since(tStart).Milliseconds() return report } + +// runParallel groups scenarios by service, then runs service groups in +// parallel (up to MaxConcurrent). Scenarios within each group run sequentially +// to avoid overwhelming a single API with concurrent requests. +func (r *Runner) runParallel(scenarios []Scenario, configPath string) *RunReport { + tStart := time.Now() + // Group by service, preserving order within each group. + groups := make(map[string][]Scenario) + var serviceOrder []string + for _, s := range scenarios { + if _, ok := groups[s.Service]; !ok { + serviceOrder = append(serviceOrder, s.Service) + } + groups[s.Service] = append(groups[s.Service], s) + } + + // Assign each scenario an index for deterministic output ordering. + origIdx := make(map[string]int) + for i, s := range scenarios { + origIdx[s.Name] = i + } + + type indexedResult struct { + idx int + result ScenarioResult + } + + resultCh := make(chan indexedResult, len(scenarios)) + sem := make(chan struct{}, r.maxConcurrent) + var wg sync.WaitGroup + + for _, svc := range serviceOrder { + wg.Add(1) + go func(svcName string, svcScenarios []Scenario) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + for _, s := range svcScenarios { + res := r.Run(s) + resultCh <- indexedResult{idx: origIdx[s.Name], result: res} + } + }(svc, groups[svc]) + } + + go func() { + wg.Wait() + close(resultCh) + }() + + // Collect and sort by original index. + var indexed []indexedResult + for ir := range resultCh { + indexed = append(indexed, ir) + } + sort.Slice(indexed, func(i, j int) bool { return indexed[i].idx < indexed[j].idx }) + + report := &RunReport{ + GeneratedAt: time.Now().Format("2006-01-02 15:04:05"), + ConfigPath: configPath, + } + for _, ir := range indexed { + report.Results = append(report.Results, ir.result) + } + report.TotalDurationMs = time.Since(tStart).Milliseconds() + return report +} + diff --git a/internal/smoke/runner_test.go b/internal/smoke/runner_test.go index 23d424b..db681e4 100644 --- a/internal/smoke/runner_test.go +++ b/internal/smoke/runner_test.go @@ -43,7 +43,7 @@ func main() { func TestRunner_SkipScenario(t *testing.T) { s := Scenario{Name: "skip me", Skip: true, SkipReason: "not ready"} - r := NewRunner("", "", false) + r := NewRunner("", "", false, 0) result := r.Run(s) if result.Status != StatusSkip { t.Errorf("expected SKIP, got %s", result.Status) @@ -63,7 +63,7 @@ func TestRunner_SuccessCapture(t *testing.T) { {Type: "stdout_contains", Str: "TITLE"}, }, } - r := NewRunner(bin, "", false) + r := NewRunner(bin, "", false, 0) result := r.Run(s) if result.Status != StatusPass { t.Errorf("expected PASS, got %s: %s", result.Status, result.Err) @@ -88,7 +88,7 @@ func TestRunner_NonZeroExit_AssertionFails(t *testing.T) { {Type: "exit_code", Value: 0}, }, } - r := NewRunner(bin, "", false) + r := NewRunner(bin, "", false, 0) result := r.Run(s) if result.Status != StatusFail { t.Errorf("expected FAIL, got %s", result.Status) @@ -105,7 +105,7 @@ func TestRunner_StderrCaptured(t *testing.T) { Format: "table", TimeoutMs: 5000, Assertions: []Assertion{{Type: "exit_code", Value: 1}}, } - r := NewRunner(bin, "", false) + r := NewRunner(bin, "", false, 0) result := r.Run(s) if result.Stderr == "" { t.Error("expected stderr to be captured") @@ -121,7 +121,7 @@ func main() { time.Sleep(10 * time.Second) }`) Format: "table", TimeoutMs: 200, Assertions: []Assertion{{Type: "exit_code", Value: 0}}, } - r := NewRunner(bin, "", false) + r := NewRunner(bin, "", false, 0) result := r.Run(s) if result.Status != StatusTimeout { t.Errorf("expected TIMEOUT, got %s", result.Status) From ef94da530145e97cd6c13d5e8df6e69b9b9e6617 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Sun, 7 Jun 2026 11:57:37 +0800 Subject: [PATCH 3/3] chore: gofmt internal/smoke/runner.go --- internal/smoke/runner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/smoke/runner.go b/internal/smoke/runner.go index 82c1a0f..2179c7e 100644 --- a/internal/smoke/runner.go +++ b/internal/smoke/runner.go @@ -202,4 +202,3 @@ func (r *Runner) runParallel(scenarios []Scenario, configPath string) *RunReport report.TotalDurationMs = time.Since(tStart).Milliseconds() return report } -