feat: add an option to output test summary at last (scenarigo#395)

* feat: test summary

* Make show summary configurable

* enable color if enabledColor

* change `NoSummary` to `Summary`.

* implement `PrintSummary` as `reporter`'s method

* delete unnecessary line break

* Add nil check for `cfg`

* fix unit tests for `NewRunner`

* update code snippet generated by `scenarigo config init` in README

* change `testSummary#add`'s argument(`r reporter.Reporter` to `testResultString string`) for the testability

* add unit tests for `testSummary#add`

* add unit tests for `testSummary#String`, `testSummary#failedFiles`

* add unit test for `run` when summary is enabled

* move `summary.go`, `summary_test.go` to `reporter` package from `scenarigo` package

* move logic to `reporter` package from `scenarigo` package

* rename `testSummary#add` to `testSummary#append`

* rename `reporter/summary.go` to `reporter/test_summary.go`, `reporter/summary_test.go` to `reporter/test_summary_test.go`

* show failed files with failColor

* delete unnecessary line break

* delete `enabledColor` field from `testSummary`

* delete line break

* If `testSummary == nil` do nothing in `testSummary#append`, change interface of `testSummary#append`

* keep only failed file names

README.md

@@ -85,6 +85,7 @@ plugins:                  # Specify configurations to build plugins.
 output:
   verbose: false # Enable verbose output.
   colored: false # Enable colored output with ANSI color escape codes. It is enabled by default but disabled when a NO_COLOR environment variable is set (regardless of its value).
+  summary: false # Enable summary output.
   report:
     json:
       filename: ./report.json # Specify a filename for test report output in JSON.

cmd/scenarigo/cmd/config/default.scenarigo.yaml

@@ -10,6 +10,7 @@ pluginDirectory: ./gen      # Specify the root directory of plugins.
 output:
   verbose: false   # Enable verbose output.
   # colored: false # Enable colored output with ANSI color escape codes. It is enabled by default but disabled when a NO_COLOR environment variable is set (regardless of its value).
+  # summary: false # Enable summary output.
   # report:
   #   json:
   #     filename: ./report.json # Specify a filename for test report output in JSON.

cmd/scenarigo/cmd/run.go

@@ -66,6 +66,10 @@ func run(cmd *cobra.Command, args []string) error {
 		reporterOpts = append(reporterOpts, reporter.WithNoColor())
 	}
 
+	if cfg != nil && cfg.Output.Summary {
+		reporterOpts = append(reporterOpts, reporter.WithTestSummary())
+	}
+
 	var reportErr error
 	success := reporter.Run(
 		func(rptr reporter.Reporter) {

cmd/scenarigo/cmd/run_test.go

@@ -111,6 +111,53 @@ FAIL
 FAIL	setup	0.000s
 FAIL
 `, filepath.Join(wd, "testdata", "plugin.so")), "\n"),
+		},
+		"print summary": {
+			args:        []string{},
+			config:      "./testdata/scenarigo-summary.yaml",
+			expectError: ErrTestFailed.Error(),
+			expectOutput: strings.TrimPrefix(`
+--- FAIL: scenarios/fail.yaml (0.00s)
+    --- FAIL: scenarios/fail.yaml//echo (0.00s)
+        --- FAIL: scenarios/fail.yaml//echo/POST_/echo (0.00s)
+                request:
+                  method: POST
+                  url: http://127.0.0.1:12345/echo
+                  header:
+                    User-Agent:
+                    - scenarigo/v1.0.0
+                  body:
+                    message: request
+                response:
+                  status: 200 OK
+                  statusCode: 200
+                  header:
+                    Content-Length:
+                    - "23"
+                    Content-Type:
+                    - application/json
+                    Date:
+                    - Mon, 01 Jan 0001 00:00:00 GMT
+                  body:
+                    message: request
+                elapsed time: 0.000000 sec
+                expected response but got request
+                      12 |   expect:
+                      13 |     code: 200
+                      14 |     body:
+                    > 15 |       message: "response"
+                                          ^
+FAIL
+FAIL	scenarios/fail.yaml	0.000s
+FAIL
+ok  	scenarios/pass.yaml	0.000s
+
+2 tests run: 1 passed, 1 failed, 0 skipped
+
+Failed tests:
+	- scenarios/fail.yaml
+
+`, "\n"),
 		},
 		"create reports": {
 			args:        []string{},

cmd/scenarigo/cmd/testdata/scenarigo-summary.yaml

@@ -0,0 +1,16 @@
+schemaVersion: config/v1
+
+scenarios:
+  - ./scenarios/fail.yaml
+  - ./scenarios/pass.yaml
+pluginDirectory: ./ # Specify the root directory of plugins.
+
+output:
+  verbose: false          # Enable verbose output.
+  # colored: false        # Enable colored output with ANSI color escape codes. It is enabled by default but disabled when a NO_COLOR environment variable is set (regardless of its value).
+  summary: true           # Enable summary output.
+  # report:
+  #   json:
+  #     filename: ./report.json # Specify a filename for test report output in JSON.
+  #   junit:
+  #     filename: ./junit.xml # Specify a filename for test report output in JUnit XML format.

reporter/context.go

@@ -38,6 +38,14 @@ func WithNoColor() Option {
 	}
 }
 
+// WithTestSummary returns an option to enable test summary.
+func WithTestSummary() Option {
+	return func(ctx *testContext) {
+		ctx.enabledTestSummary = true
+		ctx.testSummary = newTestSummary()
+	}
+}
+
 // testContext holds all fields that are common to all tests.
 type testContext struct {
 	m sync.Mutex
@@ -62,6 +70,9 @@ type testContext struct {
 
 	noColor bool
 
+	enabledTestSummary bool
+	testSummary        *testSummary
+
 	// for FromT
 	matcher *matcher
 }

reporter/reporter.go

@@ -47,12 +47,16 @@ type Reporter interface {
 	getLogs() *logRecorder
 	getChildren() []Reporter
 	isRoot() bool
+
+	// for test summary
+	printTestSummary()
 }
 
 // Run runs f with new Reporter which applied opts.
 // It reports whether f succeeded.
 func Run(f func(r Reporter), opts ...Option) bool {
 	r := run(f, opts...)
+	r.printTestSummary()
 	return !r.Failed()
 }
 
@@ -225,6 +229,13 @@ func (r *reporter) Parallel() {
 	}
 }
 
+func (r *reporter) printTestSummary() {
+	if !r.context.enabledTestSummary {
+		return
+	}
+	_, _ = r.context.printf(r.context.testSummary.String(r.context.noColor))
+}
+
 func (r *reporter) appendChildren(children ...*reporter) {
 	r.m.Lock()
 	r.children = append(r.children, children...)
@@ -276,6 +287,7 @@ func (r *reporter) runWithRetry(name string, f func(t Reporter), policy RetryPol
 	r.appendChildren(child)
 	if r.isRoot() {
 		printReport(child)
+		child.context.testSummary.append(name, child)
 	}
 	return !child.Failed()
 }

reporter/test_summary.go

@@ -0,0 +1,99 @@
+package reporter
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/fatih/color"
+)
+
+type testSummary struct {
+	mu           sync.Mutex
+	passedCount  int
+	failed       []string
+	skippedCount int
+}
+
+func newTestSummary() *testSummary {
+	return &testSummary{
+		mu:           sync.Mutex{},
+		passedCount:  0,
+		failed:       []string{},
+		skippedCount: 0,
+	}
+}
+
+func (s *testSummary) append(testFileRelPath string, r Reporter) {
+	if s == nil {
+		return
+	}
+	testResultString := TestResultString(r)
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	switch testResultString {
+	case TestResultPassed.String():
+		s.passedCount++
+	case TestResultFailed.String():
+		s.failed = append(s.failed, testFileRelPath)
+	case TestResultSkipped.String():
+		s.skippedCount++
+	default: // Do nothing
+	}
+}
+
+// String converts testSummary to the string like below.
+// 11 tests run: 9 passed, 2 failed, 0 skipped
+//
+// Failed tests:
+//   - scenarios/scenario1.yaml
+//   - scenarios/scenario2.yaml
+func (s *testSummary) String(noColor bool) string {
+	totalText := fmt.Sprintf("%d tests run", s.passedCount+len(s.failed)+s.skippedCount)
+	passedText := s.passColor(noColor).Sprintf("%d passed", s.passedCount)
+	failedText := s.failColor(noColor).Sprintf("%d failed", len(s.failed))
+	skippedText := s.skipColor(noColor).Sprintf("%d skipped", s.skippedCount)
+	failedFiles := s.failColor(noColor).Sprintf(s.failedFiles())
+	return fmt.Sprintf(
+		"\n%s: %s, %s, %s\n\n%s",
+		totalText, passedText, failedText, skippedText, failedFiles,
+	)
+}
+
+func (s *testSummary) failedFiles() string {
+	if len(s.failed) == 0 {
+		return ""
+	}
+
+	result := ""
+
+	for _, f := range s.failed {
+		if result == "" {
+			result = "Failed tests:\n"
+		}
+		result += fmt.Sprintf("\t- %s\n", f)
+	}
+	result += "\n"
+
+	return result
+}
+
+func (s *testSummary) passColor(noColor bool) *color.Color {
+	if noColor {
+		return color.New()
+	}
+	return color.New(color.FgGreen)
+}
+
+func (s *testSummary) failColor(noColor bool) *color.Color {
+	if noColor {
+		return color.New()
+	}
+	return color.New(color.FgHiRed)
+}
+
+func (s *testSummary) skipColor(noColor bool) *color.Color {
+	if noColor {
+		return color.New()
+	}
+	return color.New(color.FgYellow)
+}