From 24d4412b79bd4d992263081bd517acf6cde9bc81 Mon Sep 17 00:00:00 2001 From: Tatsuya Kyushima <49891479+kyu08@users.noreply.github.com> Date: Thu, 29 Feb 2024 00:55:57 +0900 Subject: [PATCH] feat: add an option to output test summary at last (#395) * feat: test summary * Make show summary configurable * enable color if enabledColor * change `NoSummary` to `Summary`. * implement `PrintSummary` as `reporter`'s method * delete unnecessary line break * Add nil check for `cfg` * fix unit tests for `NewRunner` * update code snippet generated by `scenarigo config init` in README * change `testSummary#add`'s argument(`r reporter.Reporter` to `testResultString string`) for the testability * add unit tests for `testSummary#add` * add unit tests for `testSummary#String`, `testSummary#failedFiles` * add unit test for `run` when summary is enabled * move `summary.go`, `summary_test.go` to `reporter` package from `scenarigo` package * move logic to `reporter` package from `scenarigo` package * rename `testSummary#add` to `testSummary#append` * rename `reporter/summary.go` to `reporter/test_summary.go`, `reporter/summary_test.go` to `reporter/test_summary_test.go` * show failed files with failColor * delete unnecessary line break * delete `enabledColor` field from `testSummary` * delete line break * If `testSummary == nil` do nothing in `testSummary#append`, change interface of `testSummary#append` * keep only failed file names --- README.md | 1 + .../cmd/config/default.scenarigo.yaml | 1 + cmd/scenarigo/cmd/run.go | 4 + cmd/scenarigo/cmd/run_test.go | 47 +++++ .../cmd/testdata/scenarigo-summary.yaml | 16 ++ reporter/context.go | 11 ++ reporter/reporter.go | 12 ++ reporter/test_summary.go | 99 ++++++++++ reporter/test_summary_test.go | 178 ++++++++++++++++++ schema/config.go | 1 + 10 files changed, 370 insertions(+) create mode 100644 cmd/scenarigo/cmd/testdata/scenarigo-summary.yaml create mode 100644 reporter/test_summary.go create mode 100644 reporter/test_summary_test.go diff --git a/README.md b/README.md index cd23862a..87df0af5 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ plugins: # Specify configurations to build plugins. output: verbose: false # Enable verbose output. colored: false # Enable colored output with ANSI color escape codes. It is enabled by default but disabled when a NO_COLOR environment variable is set (regardless of its value). + summary: false # Enable summary output. report: json: filename: ./report.json # Specify a filename for test report output in JSON. diff --git a/cmd/scenarigo/cmd/config/default.scenarigo.yaml b/cmd/scenarigo/cmd/config/default.scenarigo.yaml index bace8f31..81580fb8 100644 --- a/cmd/scenarigo/cmd/config/default.scenarigo.yaml +++ b/cmd/scenarigo/cmd/config/default.scenarigo.yaml @@ -10,6 +10,7 @@ pluginDirectory: ./gen # Specify the root directory of plugins. output: verbose: false # Enable verbose output. # colored: false # Enable colored output with ANSI color escape codes. It is enabled by default but disabled when a NO_COLOR environment variable is set (regardless of its value). + # summary: false # Enable summary output. # report: # json: # filename: ./report.json # Specify a filename for test report output in JSON. diff --git a/cmd/scenarigo/cmd/run.go b/cmd/scenarigo/cmd/run.go index 62794acf..1484f0f1 100644 --- a/cmd/scenarigo/cmd/run.go +++ b/cmd/scenarigo/cmd/run.go @@ -66,6 +66,10 @@ func run(cmd *cobra.Command, args []string) error { reporterOpts = append(reporterOpts, reporter.WithNoColor()) } + if cfg != nil && cfg.Output.Summary { + reporterOpts = append(reporterOpts, reporter.WithTestSummary()) + } + var reportErr error success := reporter.Run( func(rptr reporter.Reporter) { diff --git a/cmd/scenarigo/cmd/run_test.go b/cmd/scenarigo/cmd/run_test.go index ec761225..92c17296 100644 --- a/cmd/scenarigo/cmd/run_test.go +++ b/cmd/scenarigo/cmd/run_test.go @@ -111,6 +111,53 @@ FAIL FAIL setup 0.000s FAIL `, filepath.Join(wd, "testdata", "plugin.so")), "\n"), + }, + "print summary": { + args: []string{}, + config: "./testdata/scenarigo-summary.yaml", + expectError: ErrTestFailed.Error(), + expectOutput: strings.TrimPrefix(` +--- FAIL: scenarios/fail.yaml (0.00s) + --- FAIL: scenarios/fail.yaml//echo (0.00s) + --- FAIL: scenarios/fail.yaml//echo/POST_/echo (0.00s) + request: + method: POST + url: http://127.0.0.1:12345/echo + header: + User-Agent: + - scenarigo/v1.0.0 + body: + message: request + response: + status: 200 OK + statusCode: 200 + header: + Content-Length: + - "23" + Content-Type: + - application/json + Date: + - Mon, 01 Jan 0001 00:00:00 GMT + body: + message: request + elapsed time: 0.000000 sec + expected response but got request + 12 | expect: + 13 | code: 200 + 14 | body: + > 15 | message: "response" + ^ +FAIL +FAIL scenarios/fail.yaml 0.000s +FAIL +ok scenarios/pass.yaml 0.000s + +2 tests run: 1 passed, 1 failed, 0 skipped + +Failed tests: + - scenarios/fail.yaml + +`, "\n"), }, "create reports": { args: []string{}, diff --git a/cmd/scenarigo/cmd/testdata/scenarigo-summary.yaml b/cmd/scenarigo/cmd/testdata/scenarigo-summary.yaml new file mode 100644 index 00000000..79b32682 --- /dev/null +++ b/cmd/scenarigo/cmd/testdata/scenarigo-summary.yaml @@ -0,0 +1,16 @@ +schemaVersion: config/v1 + +scenarios: + - ./scenarios/fail.yaml + - ./scenarios/pass.yaml +pluginDirectory: ./ # Specify the root directory of plugins. + +output: + verbose: false # Enable verbose output. + # colored: false # Enable colored output with ANSI color escape codes. It is enabled by default but disabled when a NO_COLOR environment variable is set (regardless of its value). + summary: true # Enable summary output. + # report: + # json: + # filename: ./report.json # Specify a filename for test report output in JSON. + # junit: + # filename: ./junit.xml # Specify a filename for test report output in JUnit XML format. diff --git a/reporter/context.go b/reporter/context.go index 261fa992..6d7c0dd9 100644 --- a/reporter/context.go +++ b/reporter/context.go @@ -38,6 +38,14 @@ func WithNoColor() Option { } } +// WithTestSummary returns an option to enable test summary. +func WithTestSummary() Option { + return func(ctx *testContext) { + ctx.enabledTestSummary = true + ctx.testSummary = newTestSummary() + } +} + // testContext holds all fields that are common to all tests. type testContext struct { m sync.Mutex @@ -62,6 +70,9 @@ type testContext struct { noColor bool + enabledTestSummary bool + testSummary *testSummary + // for FromT matcher *matcher } diff --git a/reporter/reporter.go b/reporter/reporter.go index 0cfcd507..6de60f2d 100644 --- a/reporter/reporter.go +++ b/reporter/reporter.go @@ -47,12 +47,16 @@ type Reporter interface { getLogs() *logRecorder getChildren() []Reporter isRoot() bool + + // for test summary + printTestSummary() } // Run runs f with new Reporter which applied opts. // It reports whether f succeeded. func Run(f func(r Reporter), opts ...Option) bool { r := run(f, opts...) + r.printTestSummary() return !r.Failed() } @@ -225,6 +229,13 @@ func (r *reporter) Parallel() { } } +func (r *reporter) printTestSummary() { + if !r.context.enabledTestSummary { + return + } + _, _ = r.context.printf(r.context.testSummary.String(r.context.noColor)) +} + func (r *reporter) appendChildren(children ...*reporter) { r.m.Lock() r.children = append(r.children, children...) @@ -276,6 +287,7 @@ func (r *reporter) runWithRetry(name string, f func(t Reporter), policy RetryPol r.appendChildren(child) if r.isRoot() { printReport(child) + child.context.testSummary.append(name, child) } return !child.Failed() } diff --git a/reporter/test_summary.go b/reporter/test_summary.go new file mode 100644 index 00000000..ae8c67af --- /dev/null +++ b/reporter/test_summary.go @@ -0,0 +1,99 @@ +package reporter + +import ( + "fmt" + "sync" + + "github.com/fatih/color" +) + +type testSummary struct { + mu sync.Mutex + passedCount int + failed []string + skippedCount int +} + +func newTestSummary() *testSummary { + return &testSummary{ + mu: sync.Mutex{}, + passedCount: 0, + failed: []string{}, + skippedCount: 0, + } +} + +func (s *testSummary) append(testFileRelPath string, r Reporter) { + if s == nil { + return + } + testResultString := TestResultString(r) + s.mu.Lock() + defer s.mu.Unlock() + switch testResultString { + case TestResultPassed.String(): + s.passedCount++ + case TestResultFailed.String(): + s.failed = append(s.failed, testFileRelPath) + case TestResultSkipped.String(): + s.skippedCount++ + default: // Do nothing + } +} + +// String converts testSummary to the string like below. +// 11 tests run: 9 passed, 2 failed, 0 skipped +// +// Failed tests: +// - scenarios/scenario1.yaml +// - scenarios/scenario2.yaml +func (s *testSummary) String(noColor bool) string { + totalText := fmt.Sprintf("%d tests run", s.passedCount+len(s.failed)+s.skippedCount) + passedText := s.passColor(noColor).Sprintf("%d passed", s.passedCount) + failedText := s.failColor(noColor).Sprintf("%d failed", len(s.failed)) + skippedText := s.skipColor(noColor).Sprintf("%d skipped", s.skippedCount) + failedFiles := s.failColor(noColor).Sprintf(s.failedFiles()) + return fmt.Sprintf( + "\n%s: %s, %s, %s\n\n%s", + totalText, passedText, failedText, skippedText, failedFiles, + ) +} + +func (s *testSummary) failedFiles() string { + if len(s.failed) == 0 { + return "" + } + + result := "" + + for _, f := range s.failed { + if result == "" { + result = "Failed tests:\n" + } + result += fmt.Sprintf("\t- %s\n", f) + } + result += "\n" + + return result +} + +func (s *testSummary) passColor(noColor bool) *color.Color { + if noColor { + return color.New() + } + return color.New(color.FgGreen) +} + +func (s *testSummary) failColor(noColor bool) *color.Color { + if noColor { + return color.New() + } + return color.New(color.FgHiRed) +} + +func (s *testSummary) skipColor(noColor bool) *color.Color { + if noColor { + return color.New() + } + return color.New(color.FgYellow) +} diff --git a/reporter/test_summary_test.go b/reporter/test_summary_test.go new file mode 100644 index 00000000..a319ec26 --- /dev/null +++ b/reporter/test_summary_test.go @@ -0,0 +1,178 @@ +package reporter + +import ( + "strings" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func Test_testSummaryAppend(t *testing.T) { + t.Parallel() + tests := map[string]struct { + testSummary testSummary + testFileRelPath string + reportFunc func(r *reporter) + expect testSummary + }{ + "passed": { + testSummary: testSummary{ + mu: sync.Mutex{}, + passedCount: 0, + failed: []string{}, + skippedCount: 0, + }, + testFileRelPath: "scenario/test.yaml", + reportFunc: func(r *reporter) {}, + expect: testSummary{ + mu: sync.Mutex{}, + passedCount: 1, + failed: []string{}, + skippedCount: 0, + }, + }, + "failed": { + testSummary: testSummary{ + mu: sync.Mutex{}, + passedCount: 0, + failed: []string{}, + skippedCount: 0, + }, + testFileRelPath: "scenario/test.yaml", + reportFunc: func(r *reporter) { r.Fail() }, + expect: testSummary{ + mu: sync.Mutex{}, + passedCount: 0, + failed: []string{"scenario/test.yaml"}, + skippedCount: 0, + }, + }, + "skipped": { + testSummary: testSummary{ + mu: sync.Mutex{}, + passedCount: 0, + failed: []string{}, + skippedCount: 0, + }, + testFileRelPath: "scenario/test.yaml", + reportFunc: func(r *reporter) { r.skipped = 1 }, + expect: testSummary{ + mu: sync.Mutex{}, + passedCount: 0, + failed: []string{}, + skippedCount: 1, + }, + }, + } + + for name, test := range tests { + tt := test + t.Run(name, func(t *testing.T) { + t.Parallel() + + r := newReporter() + tt.reportFunc(r) + tt.testSummary.append(tt.testFileRelPath, r) + + if diff := cmp.Diff(tt.expect, tt.testSummary, + cmpopts.IgnoreFields(testSummary{}, "mu"), + cmp.AllowUnexported(testSummary{}), + ); diff != "" { + t.Errorf("differs (-want +got):\n%s", diff) + } + }) + } +} + +func Test_testSummaryString(t *testing.T) { + t.Parallel() + tests := map[string]struct { + testSummary testSummary + expect string + }{ + "no failed test": { + testSummary: testSummary{ + mu: sync.Mutex{}, + passedCount: 2, + failed: []string{}, + skippedCount: 1, + }, + expect: ` +3 tests run: 2 passed, 0 failed, 1 skipped + +`, + }, + "some tests failed": { + testSummary: testSummary{ + mu: sync.Mutex{}, + passedCount: 1, + failed: []string{"scenario/test1.yaml", "scenario/test2.yaml"}, + skippedCount: 1, + }, + expect: ` +4 tests run: 1 passed, 2 failed, 1 skipped + +Failed tests: + - scenario/test1.yaml + - scenario/test2.yaml + +`, + }, + } + + for name, test := range tests { + tt := test + t.Run(name, func(t *testing.T) { + t.Parallel() + got := tt.testSummary.String(true) + if diff := cmp.Diff(tt.expect, got); diff != "" { + t.Errorf("differs (-want +got):\n%s", diff) + } + }) + } +} + +func Test_testSummaryFailedFiles(t *testing.T) { + t.Parallel() + tests := map[string]struct { + testSummary testSummary + expect string + }{ + "no test failed": { + testSummary: testSummary{ + mu: sync.Mutex{}, + passedCount: 2, + failed: []string{}, + skippedCount: 0, + }, + expect: ``, + }, + "some tests failed": { + testSummary: testSummary{ + mu: sync.Mutex{}, + passedCount: 0, + failed: []string{"scenario/test1.yaml", "scenario/test2.yaml"}, + skippedCount: 0, + }, + expect: strings.TrimPrefix(` +Failed tests: + - scenario/test1.yaml + - scenario/test2.yaml + +`, "\n"), + }, + } + + for name, test := range tests { + tt := test + t.Run(name, func(t *testing.T) { + t.Parallel() + got := tt.testSummary.failedFiles() + if diff := cmp.Diff(tt.expect, got); diff != "" { + t.Errorf("differs (-want +got):\n%s", diff) + } + }) + } +} diff --git a/schema/config.go b/schema/config.go index 1b7c4312..6c7cbe96 100644 --- a/schema/config.go +++ b/schema/config.go @@ -57,6 +57,7 @@ type YTTConfig struct { type OutputConfig struct { Verbose bool `yaml:"verbose,omitempty"` Colored *bool `yaml:"colored,omitempty"` + Summary bool `yaml:"summary,omitempty"` Report ReportConfig `yaml:"report,omitempty"` }