diff --git a/.github/workflows/golden-test.yml b/.github/workflows/golden-test.yml index 7fd9886..3d32018 100644 --- a/.github/workflows/golden-test.yml +++ b/.github/workflows/golden-test.yml @@ -9,8 +9,16 @@ on: jobs: golden-test: - name: Golden Test + name: Golden Test ${{ matrix.grouping }} runs-on: ubuntu-latest + strategy: + matrix: + include: + - grouping: 'w/ grouping' + group_errors: 'true' + - grouping: 'w/o grouping' + group_errors: 'false' + steps: - uses: actions/checkout@v4 name: Check out repository @@ -24,13 +32,13 @@ jobs: go-version: 1.22.x cache: false - - name: Golden Test + - name: Golden Test ${{ matrix.grouping }} id: golden_test # Run golden test by comparing HEAD and the base branch (the target branch of the PR). # GitHub Actions terminates the job if it hits the resource limits. Here we limit the # memory usage to 8GiB to avoid that. run: | - make golden-test GOMEMLIMIT=8192MiB ARGS="-base-branch ${{ github.event.pull_request.base.ref }} -result-file ${{ runner.temp }}/golden-test-result.md" + make golden-test GOMEMLIMIT=8192MiB ARGS="-base-branch ${{ github.event.pull_request.base.ref }} -result-file ${{ runner.temp }}/golden-test-result.md -group-error-messages=${{ matrix.group_errors }}" - uses: actions/github-script@v7 with: @@ -61,7 +69,7 @@ jobs: repo: repo, issue_number: issueNumber }); - const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test')); + const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test ${{ matrix.grouping }}')); // Update or create the PR comment. if (botComment) { diff --git a/tools/cmd/golden-test/main.go b/tools/cmd/golden-test/main.go index 9c255ad..25d97e0 100644 --- a/tools/cmd/golden-test/main.go +++ b/tools/cmd/golden-test/main.go @@ -55,7 +55,7 @@ type BranchResult struct { // Run runs the golden tests on the base branch and the test branch and writes the summary and // diff to the writer. -func Run(writer io.Writer, baseBranch, testBranch string) error { +func Run(writer io.Writer, baseBranch, testBranch, groupingFlag string) error { // First verify that the git repository is clean. out, err := exec.Command("git", "status", "--porcelain=v1").CombinedOutput() if err != nil { @@ -134,7 +134,7 @@ func Run(writer io.Writer, baseBranch, testBranch string) error { // Run the built NilAway binary on the stdlib and parse the diagnostics. var buf bytes.Buffer - cmd := exec.Command("bin/nilaway", "-include-errors-in-files", "/", "-json", "-pretty-print=false", "-group-error-messages=true", "std") + cmd := exec.Command("bin/nilaway", "-include-errors-in-files", "/", "-json", "-pretty-print=false", "-group-error-messages="+groupingFlag, "std") cmd.Stdout = &buf // Inherit env vars such that users can control the resource usages via GOMEMLIMIT, GOGC // etc. env vars. @@ -149,7 +149,7 @@ func Run(writer io.Writer, baseBranch, testBranch string) error { branch.Result = diagnostics } - WriteDiff(writer, branches) + WriteDiff(writer, branches, groupingFlag) return nil } @@ -178,12 +178,16 @@ func ParseDiagnostics(reader io.Reader) (map[Diagnostic]bool, error) { // WriteDiff writes the summary and the diff (if the base and test are different) between the base // and test diagnostics to the writer. If the writer is os.Stdout, it will write the diff in color. -func WriteDiff(writer io.Writer, branches [2]*BranchResult) { +func WriteDiff(writer io.Writer, branches [2]*BranchResult, groupingFlag string) { // Compute the diagnostic differences between base and test branches. minuses, pluses := Diff(branches[0].Result, branches[1].Result), Diff(branches[1].Result, branches[0].Result) // Write the summary lines first. - MustFprint(fmt.Fprintf(writer, "## Golden Test\n\n")) + if groupingFlag == "true" { + MustFprint(fmt.Fprintf(writer, "## Golden Test w/ grouping\n\n")) + } else { + MustFprint(fmt.Fprintf(writer, "## Golden Test w/o grouping\n\n")) + } if len(pluses) == 0 && len(minuses) == 0 { MustFprint(fmt.Fprint(writer, "> [!NOTE] \n")) MustFprint(fmt.Fprintf(writer, "> ✅ NilAway errors reported on standard libraries are **identical**.\n")) @@ -281,6 +285,7 @@ func main() { baseBranch := fset.String("base-branch", "main", "the base branch to compare against") testBranch := fset.String("test-branch", "", "the test branch to run golden tests (default current branch)") resultFile := fset.String("result-file", "", "the file to write the diff to, default stdout") + groupingFlag := fset.Bool("group-error-messages", true, "group similar error messages (default true)") if err := fset.Parse(os.Args[1:]); err != nil { log.Printf("failed to parse flags: %v\n", err) flag.PrintDefaults() @@ -296,7 +301,7 @@ func main() { writer = w } - if err := Run(writer, *baseBranch, *testBranch); err != nil { + if err := Run(writer, *baseBranch, *testBranch, fmt.Sprintf("%t", *groupingFlag)); err != nil { log.Printf("failed to run golden test: %v", err) var e *exec.ExitError if errors.As(err, &e) { diff --git a/tools/cmd/golden-test/main_test.go b/tools/cmd/golden-test/main_test.go index 63f932b..28aef71 100644 --- a/tools/cmd/golden-test/main_test.go +++ b/tools/cmd/golden-test/main_test.go @@ -64,7 +64,7 @@ func TestWriteDiff(t *testing.T) { {Name: "base", ShortSHA: "123456", Result: base}, {Name: "test", ShortSHA: "456789", Result: test}, } - WriteDiff(&buf, branches) + WriteDiff(&buf, branches, "true") require.Contains(t, buf.String(), "## Golden Test") // Must contain the title. require.Contains(t, buf.String(), "are **identical**") @@ -72,7 +72,7 @@ func TestWriteDiff(t *testing.T) { base[Diagnostic{Posn: "src/file2:10:2", Message: "nil pointer dereference"}] = true test[Diagnostic{Posn: "src/file4:10:2", Message: "bar error"}] = true buf.Reset() - WriteDiff(&buf, branches) + WriteDiff(&buf, branches, "true") s := buf.String() require.Contains(t, buf.String(), "## Golden Test") // Must contain the title. require.Contains(t, s, "are **different**")