From 0ebc153826c1551524bd6801ab330c6e3650bdcb Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Thu, 30 May 2024 01:47:48 +0100 Subject: [PATCH 1/9] support multiple calls to the Attach() function from a single step --- internal/formatters/fmt_output_test.go | 30 ++++++++++++++++--- .../cucumber/scenario_with_attachment | 26 +++++++++++++++- .../events/scenario_with_attachment | 9 ++++-- .../features/scenario_with_attachment.feature | 3 +- suite.go | 17 +++++++---- 5 files changed, 72 insertions(+), 13 deletions(-) diff --git a/internal/formatters/fmt_output_test.go b/internal/formatters/fmt_output_test.go index f9b4e668..41884fb3 100644 --- a/internal/formatters/fmt_output_test.go +++ b/internal/formatters/fmt_output_test.go @@ -19,7 +19,10 @@ import ( const fmtOutputTestsFeatureDir = "formatter-tests/features" +var tT *testing.T + func Test_FmtOutput(t *testing.T) { + tT = t pkg := os.Getenv("GODOG_TESTED_PACKAGE") os.Setenv("GODOG_TESTED_PACKAGE", "github.com/cucumber/godog") @@ -64,7 +67,8 @@ func fmtOutputTest(fmtName, testName, featureFilePath string) func(*testing.T) { ctx.Step(`^(?:a )?pending step$`, pendingStepDef) ctx.Step(`^(?:a )?passing step$`, passingStepDef) ctx.Step(`^odd (\d+) and even (\d+) number$`, oddEvenStepDef) - ctx.Step(`^(?:a )?a step with attachment$`, stepWithAttachment) + ctx.Step(`^(?:a )?a step with a single attachment call for multiple attachments$`, stepWithSingleAttachmentCall) + ctx.Step(`^(?:a )?a step with multiple attachment calls$`, stepWithMultipleAttachmentCalls) } return func(t *testing.T) { @@ -127,11 +131,29 @@ func pendingStepDef() error { return godog.ErrPending } func failingStepDef() error { return fmt.Errorf("step failed") } -func stepWithAttachment(ctx context.Context) (context.Context, error) { - ctxOut := godog.Attach(ctx, +func stepWithSingleAttachmentCall(ctx context.Context) (context.Context, error) { + if len(godog.Attachments(ctx)) > 0 { + assert.FailNow(tT, "Unexpected Attachments found - should have been empty") + } + + ctx = godog.Attach(ctx, godog.Attachment{Body: []byte("TheData1"), FileName: "TheFilename1", MediaType: "text/plain"}, godog.Attachment{Body: []byte("TheData2"), FileName: "TheFilename2", MediaType: "text/plain"}, ) - return ctxOut, nil + return ctx, nil +} +func stepWithMultipleAttachmentCalls(ctx context.Context) (context.Context, error) { + if len(godog.Attachments(ctx)) > 0 { + assert.FailNow(tT, "Unexpected Attachments found - should have been empty") + } + + ctx = godog.Attach(ctx, + godog.Attachment{Body: []byte("TheData1"), FileName: "TheFilename1", MediaType: "text/plain"}, + ) + ctx = godog.Attach(ctx, + godog.Attachment{Body: []byte("TheData2"), FileName: "TheFilename2", MediaType: "text/plain"}, + ) + + return ctx, nil } diff --git a/internal/formatters/formatter-tests/cucumber/scenario_with_attachment b/internal/formatters/formatter-tests/cucumber/scenario_with_attachment index 71e0ab82..43dd7de0 100644 --- a/internal/formatters/formatter-tests/cucumber/scenario_with_attachment +++ b/internal/formatters/formatter-tests/cucumber/scenario_with_attachment @@ -17,7 +17,7 @@ "steps": [ { "keyword": "Given ", - "name": "a step with attachment", + "name": "a step with a single attachment call for multiple attachments", "line": 7, "match": { "location": "fmt_output_test.go:119" @@ -38,6 +38,30 @@ "data": "VGhlRGF0YTI" } ] + }, + { + "keyword": "And ", + "name": "a step with multiple attachment calls", + "line": 8, + "match": { + "location": "fmt_output_test.go:119" + }, + "result": { + "status": "passed", + "duration": 0 + }, + "embeddings": [ + { + "name": "TheFilename1", + "mime_type": "text/plain", + "data": "VGhlRGF0YTE" + }, + { + "name": "TheFilename2", + "mime_type": "text/plain", + "data": "VGhlRGF0YTI" + } + ] } ] } diff --git a/internal/formatters/formatter-tests/events/scenario_with_attachment b/internal/formatters/formatter-tests/events/scenario_with_attachment index d803a76c..4faa1e2c 100644 --- a/internal/formatters/formatter-tests/events/scenario_with_attachment +++ b/internal/formatters/formatter-tests/events/scenario_with_attachment @@ -1,10 +1,15 @@ {"event":"TestRunStarted","version":"0.1.0","timestamp":-6795364578871,"suite":"events"} -{"event":"TestSource","location":"formatter-tests/features/scenario_with_attachment.feature:1","source":"Feature: scenario with attachment\n describes\n an attachment\n feature\n\n Scenario: step with attachment\n Given a step with attachment\n"} +{"event":"TestSource","location":"formatter-tests/features/scenario_with_attachment.feature:1","source":"Feature: scenario with attachment\n describes\n an attachment\n feature\n\n Scenario: step with attachment\n Given a step with a single attachment call for multiple attachments\n And a step with multiple attachment calls\n"} {"event":"TestCaseStarted","location":"formatter-tests/features/scenario_with_attachment.feature:6","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_attachment.feature:7","definition_id":"fmt_output_test.go:XXX -\u003e github.com/cucumber/godog/internal/formatters_test.stepWithAttachment","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_attachment.feature:7","definition_id":"fmt_output_test.go:XXX -\u003e github.com/cucumber/godog/internal/formatters_test.stepWithSingleAttachmentCall","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_with_attachment.feature:7","timestamp":-6795364578871} {"event":"Attachment","location":"formatter-tests/features/scenario_with_attachment.feature:7","timestamp":-6795364578871,"contentEncoding":"BASE64","fileName":"TheFilename1","mimeType":"text/plain","body":"TheData1"} {"event":"Attachment","location":"formatter-tests/features/scenario_with_attachment.feature:7","timestamp":-6795364578871,"contentEncoding":"BASE64","fileName":"TheFilename2","mimeType":"text/plain","body":"TheData2"} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_with_attachment.feature:7","timestamp":-6795364578871,"status":"passed"} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_attachment.feature:8","definition_id":"fmt_output_test.go:XXX -\u003e github.com/cucumber/godog/internal/formatters_test.stepWithMultipleAttachmentCalls","arguments":[]} +{"event":"TestStepStarted","location":"formatter-tests/features/scenario_with_attachment.feature:8","timestamp":-6795364578871} +{"event":"Attachment","location":"formatter-tests/features/scenario_with_attachment.feature:8","timestamp":-6795364578871,"contentEncoding":"BASE64","fileName":"TheFilename1","mimeType":"text/plain","body":"TheData1"} +{"event":"Attachment","location":"formatter-tests/features/scenario_with_attachment.feature:8","timestamp":-6795364578871,"contentEncoding":"BASE64","fileName":"TheFilename2","mimeType":"text/plain","body":"TheData2"} +{"event":"TestStepFinished","location":"formatter-tests/features/scenario_with_attachment.feature:8","timestamp":-6795364578871,"status":"passed"} {"event":"TestCaseFinished","location":"formatter-tests/features/scenario_with_attachment.feature:6","timestamp":-6795364578871,"status":"passed"} {"event":"TestRunFinished","status":"passed","timestamp":-6795364578871,"snippets":"","memory":""} diff --git a/internal/formatters/formatter-tests/features/scenario_with_attachment.feature b/internal/formatters/formatter-tests/features/scenario_with_attachment.feature index 0299cf35..d16c9176 100644 --- a/internal/formatters/formatter-tests/features/scenario_with_attachment.feature +++ b/internal/formatters/formatter-tests/features/scenario_with_attachment.feature @@ -4,4 +4,5 @@ Feature: scenario with attachment feature Scenario: step with attachment - Given a step with attachment + Given a step with a single attachment call for multiple attachments + And a step with multiple attachment calls diff --git a/suite.go b/suite.go index 1cf1da3e..b4c15457 100644 --- a/suite.go +++ b/suite.go @@ -77,8 +77,11 @@ type Attachment struct { type attachmentKey struct{} func Attach(ctx context.Context, attachments ...Attachment) context.Context { - return context.WithValue(ctx, attachmentKey{}, attachments) + existing := Attachments(ctx) + updated := append(existing, attachments...) + return context.WithValue(ctx, attachmentKey{}, updated) } + func Attachments(ctx context.Context) []Attachment { v := ctx.Value(attachmentKey{}) @@ -88,6 +91,10 @@ func Attachments(ctx context.Context) []Attachment { return v.([]Attachment) } +func clearAttach(ctx context.Context) context.Context { + return context.WithValue(ctx, attachmentKey{}, nil) +} + func pickleAttachments(ctx context.Context) []*models.PickleAttachment { pickledAttachments := []*models.PickleAttachment{} @@ -161,7 +168,7 @@ func (s *suite) runStep(ctx context.Context, pickle *Scenario, step *Step, scena } pickledAttachments := pickleAttachments(ctx) - ctx = Attach(ctx) + ctx = clearAttach(ctx) // Run after step handlers. rctx, err = s.runAfterStepHooks(ctx, step, status, err) @@ -212,7 +219,7 @@ func (s *suite) runStep(ctx context.Context, pickle *Scenario, step *Step, scena if err != nil { pickledAttachments := pickleAttachments(ctx) - ctx = Attach(ctx) + ctx = clearAttach(ctx) sr := models.NewStepResult(models.Failed, pickle.Id, step.Id, match, pickledAttachments, nil) s.storage.MustInsertPickleStepResult(sr) @@ -237,7 +244,7 @@ func (s *suite) runStep(ctx context.Context, pickle *Scenario, step *Step, scena } pickledAttachments := pickleAttachments(ctx) - ctx = Attach(ctx) + ctx = clearAttach(ctx) sr := models.NewStepResult(models.Undefined, pickle.Id, step.Id, match, pickledAttachments, nil) s.storage.MustInsertPickleStepResult(sr) @@ -248,7 +255,7 @@ func (s *suite) runStep(ctx context.Context, pickle *Scenario, step *Step, scena if scenarioErr != nil { pickledAttachments := pickleAttachments(ctx) - ctx = Attach(ctx) + ctx = clearAttach(ctx) sr := models.NewStepResult(models.Skipped, pickle.Id, step.Id, match, pickledAttachments, nil) s.storage.MustInsertPickleStepResult(sr) From 276cca3a81c8272dcd0d5de09949a9f4c5f307c8 Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Thu, 30 May 2024 01:48:36 +0100 Subject: [PATCH 2/9] run_progress_test.go changed so it's not sensitive to the name of the clone target directory --- run_progress_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_progress_test.go b/run_progress_test.go index e11c8648..6b66d013 100644 --- a/run_progress_test.go +++ b/run_progress_test.go @@ -56,7 +56,7 @@ func Test_ProgressFormatterWhenStepPanics(t *testing.T) { require.True(t, failed) actual := buf.String() - assert.Contains(t, actual, "godog/run_progress_test.go:") + assert.Contains(t, actual, "run_progress_test.go:") } func Test_ProgressFormatterWithPanicInMultistep(t *testing.T) { From 137cbf3e498f839bf47b696e2c69e46ab477626e Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Thu, 30 May 2024 03:17:05 +0100 Subject: [PATCH 3/9] applied code review comments also added _example/attachments --- README.md | 4 ++ _examples/attachments/README.md | 16 +++++ _examples/attachments/attachments_test.go | 68 +++++++++++++++++++ .../attachments/features/attachments.feature | 7 ++ internal/formatters/fmt_cucumber.go | 6 +- internal/models/results.go | 4 +- suite.go | 6 +- 7 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 _examples/attachments/README.md create mode 100644 _examples/attachments/attachments_test.go create mode 100644 _examples/attachments/features/attachments.feature diff --git a/README.md b/README.md index df52ca19..c37a67b9 100644 --- a/README.md +++ b/README.md @@ -292,6 +292,10 @@ When steps are orthogonal and small, you can combine them just like you do with `TestFeatures` acts as a regular Go test, so you can leverage your IDE facilities to run and debug it. +### Attachments + +An example showing how to make attachments (aka embeddings) to the results is shown in [_examples/attachments](/_examples/attachments/) + ## Code of Conduct Everyone interacting in this codebase and issue tracker is expected to follow the Cucumber [code of conduct](https://github.com/cucumber/cucumber/blob/master/CODE_OF_CONDUCT.md). diff --git a/_examples/attachments/README.md b/_examples/attachments/README.md new file mode 100644 index 00000000..c9a9b0f8 --- /dev/null +++ b/_examples/attachments/README.md @@ -0,0 +1,16 @@ +# An example of Making attachments to the reports + +The JSON (and in future NDJSON) report formats allow the inclusion of data attachments. + +These attachments could be console logs or file data or images for instance. + +The example in this directory shows how the godog API is used to add attachments to the JSON report. + + +## Run the example + +You must use the '-v' flag or you will not see the cucumber JSON output. + +go test -v atttachment_test_go + + diff --git a/_examples/attachments/attachments_test.go b/_examples/attachments/attachments_test.go new file mode 100644 index 00000000..9d699405 --- /dev/null +++ b/_examples/attachments/attachments_test.go @@ -0,0 +1,68 @@ +package attachments_test + +// This example shows how to set up test suite runner with Go subtests and godog command line parameters. +// Sample commands: +// * run all scenarios from default directory (features): go test -test.run "^TestFeatures/" +// * run all scenarios and list subtest names: go test -test.v -test.run "^TestFeatures/" +// * run all scenarios from one feature file: go test -test.v -godog.paths features/nodogs.feature -test.run "^TestFeatures/" +// * run all scenarios from multiple feature files: go test -test.v -godog.paths features/nodogs.feature,features/godogs.feature -test.run "^TestFeatures/" +// * run single scenario as a subtest: go test -test.v -test.run "^TestFeatures/Eat_5_out_of_12$" +// * show usage help: go test -godog.help +// * show usage help if there were other test files in directory: go test -godog.help godogs_test.go +// * run scenarios with multiple formatters: go test -test.v -godog.format cucumber:cuc.json,pretty -test.run "^TestFeatures/" + +import ( + "context" + "os" + "testing" + + "github.com/cucumber/godog" + "github.com/cucumber/godog/colors" +) + +var opts = godog.Options{ + Output: colors.Colored(os.Stdout), + Format: "cucumber", // cucumber json format +} + +func TestFeatures(t *testing.T) { + o := opts + o.TestingT = t + + status := godog.TestSuite{ + Name: "attachments", + Options: &o, + ScenarioInitializer: InitializeScenario, + }.Run() + + if status == 2 { + t.SkipNow() + } + + if status != 0 { + t.Fatalf("zero status code expected, %d received", status) + } +} + +func InitializeScenario(ctx *godog.ScenarioContext) { + + ctx.Step(`^I have attached two documents in sequence$`, func(ctx context.Context) (context.Context, error) { + // the attached bytes will be base64 encoded by the framework and placed in the embeddings section of the cuke report + ctx = godog.Attach(ctx, + godog.Attachment{Body: []byte("TheData1"), FileName: "Data Attachment", MediaType: "text/plain"}, + ) + ctx = godog.Attach(ctx, + godog.Attachment{Body: []byte("{ \"a\" : 1 }"), FileName: "Json Attachment", MediaType: "application/json"}, + ) + + return ctx, nil + }) + ctx.Step(`^I have attached two documents at once$`, func(ctx context.Context) (context.Context, error) { + ctx = godog.Attach(ctx, + godog.Attachment{Body: []byte("TheData1"), FileName: "Data Attachment 1", MediaType: "text/plain"}, + godog.Attachment{Body: []byte("TheData2"), FileName: "Data Attachment 2", MediaType: "text/plain"}, + ) + + return ctx, nil + }) +} diff --git a/_examples/attachments/features/attachments.feature b/_examples/attachments/features/attachments.feature new file mode 100644 index 00000000..27b4cc4b --- /dev/null +++ b/_examples/attachments/features/attachments.feature @@ -0,0 +1,7 @@ +Feature: Attaching content to the cucumber report + The cucumber JSON and NDJSON support the inclusion of attachments. + These can be text or images or any data really. + + Scenario: Attaching files to the report + Given I have attached two documents in sequence + And I have attached two documents at once diff --git a/internal/formatters/fmt_cucumber.go b/internal/formatters/fmt_cucumber.go index 765403ba..613e58fa 100644 --- a/internal/formatters/fmt_cucumber.go +++ b/internal/formatters/fmt_cucumber.go @@ -154,7 +154,7 @@ type cukeStep struct { Match cukeMatch `json:"match"` Result cukeResult `json:"result"` DataTable []*cukeDataTableRow `json:"rows,omitempty"` - Embeddings []*cukeEmbedding `json:"embeddings,omitempty"` + Embeddings []cukeEmbedding `json:"embeddings,omitempty"` } type cukeDataTableRow struct { @@ -303,10 +303,10 @@ func (f *Cuke) buildCukeStep(pickle *messages.Pickle, stepResult models.PickleSt } if stepResult.Attachments != nil { - attachments := []*cukeEmbedding{} + attachments := []cukeEmbedding{} for _, a := range stepResult.Attachments { - attachments = append(attachments, &cukeEmbedding{ + attachments = append(attachments, cukeEmbedding{ Name: a.Name, Data: base64.RawStdEncoding.EncodeToString(a.Data), MimeType: a.MimeType, diff --git a/internal/models/results.go b/internal/models/results.go index 10abb18b..15c68379 100644 --- a/internal/models/results.go +++ b/internal/models/results.go @@ -36,7 +36,7 @@ type PickleStepResult struct { Def *StepDefinition - Attachments []*PickleAttachment + Attachments []PickleAttachment } // NewStepResult ... @@ -44,7 +44,7 @@ func NewStepResult( status StepResultStatus, pickleID, pickleStepID string, match *StepDefinition, - attachments []*PickleAttachment, + attachments []PickleAttachment, err error, ) PickleStepResult { return PickleStepResult{ diff --git a/suite.go b/suite.go index b4c15457..cf091805 100644 --- a/suite.go +++ b/suite.go @@ -95,13 +95,13 @@ func clearAttach(ctx context.Context) context.Context { return context.WithValue(ctx, attachmentKey{}, nil) } -func pickleAttachments(ctx context.Context) []*models.PickleAttachment { +func pickleAttachments(ctx context.Context) []models.PickleAttachment { - pickledAttachments := []*models.PickleAttachment{} + pickledAttachments := []models.PickleAttachment{} attachments := Attachments(ctx) for _, a := range attachments { - pickledAttachments = append(pickledAttachments, &models.PickleAttachment{ + pickledAttachments = append(pickledAttachments, models.PickleAttachment{ Name: a.FileName, Data: a.Body, MimeType: a.MediaType, From 2aea80ca422a6bae418836504a630d7f8761e3ee Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Thu, 30 May 2024 03:18:43 +0100 Subject: [PATCH 4/9] applied code review comments also added _example/attachments --- _examples/attachments/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_examples/attachments/README.md b/_examples/attachments/README.md index c9a9b0f8..f37443e7 100644 --- a/_examples/attachments/README.md +++ b/_examples/attachments/README.md @@ -11,6 +11,6 @@ The example in this directory shows how the godog API is used to add attachments You must use the '-v' flag or you will not see the cucumber JSON output. -go test -v atttachment_test_go +go test -v atttachment_test.go From 15d425e3c2ce145e30592c995aec56a734688531 Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Thu, 30 May 2024 03:28:12 +0100 Subject: [PATCH 5/9] applied code review comments also added _example/attachments --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9bff11c..1c5a6716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ This document is formatted according to the principles of [Keep A CHANGELOG](htt ## Unreleased -- Provide support for attachments / embeddings - ([623](https://github.com/cucumber/godog/pull/623) - [johnlon](https://github.com/johnlon)) +- Provide support for attachments / embeddings including a new example in the examples dir - ([623](https://github.com/cucumber/godog/pull/623) - [johnlon](https://github.com/johnlon)) ## [v0.14.1] From 92e8016829d8bc688606f7bac2a8d08fc6b5bd27 Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Fri, 31 May 2024 20:28:10 +0100 Subject: [PATCH 6/9] corrected base64 encoding of attachments to use padding as that's what cuke JVM does --- internal/formatters/fmt_cucumber.go | 2 +- .../formatter-tests/cucumber/scenario_with_attachment | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/formatters/fmt_cucumber.go b/internal/formatters/fmt_cucumber.go index 613e58fa..1bfbaf39 100644 --- a/internal/formatters/fmt_cucumber.go +++ b/internal/formatters/fmt_cucumber.go @@ -308,7 +308,7 @@ func (f *Cuke) buildCukeStep(pickle *messages.Pickle, stepResult models.PickleSt for _, a := range stepResult.Attachments { attachments = append(attachments, cukeEmbedding{ Name: a.Name, - Data: base64.RawStdEncoding.EncodeToString(a.Data), + Data: base64.StdEncoding.EncodeToString(a.Data), MimeType: a.MimeType, }) } diff --git a/internal/formatters/formatter-tests/cucumber/scenario_with_attachment b/internal/formatters/formatter-tests/cucumber/scenario_with_attachment index 43dd7de0..633efae6 100644 --- a/internal/formatters/formatter-tests/cucumber/scenario_with_attachment +++ b/internal/formatters/formatter-tests/cucumber/scenario_with_attachment @@ -30,12 +30,12 @@ { "name": "TheFilename1", "mime_type": "text/plain", - "data": "VGhlRGF0YTE" + "data": "VGhlRGF0YTE=" }, { "name": "TheFilename2", "mime_type": "text/plain", - "data": "VGhlRGF0YTI" + "data": "VGhlRGF0YTI=" } ] }, @@ -54,12 +54,12 @@ { "name": "TheFilename1", "mime_type": "text/plain", - "data": "VGhlRGF0YTE" + "data": "VGhlRGF0YTE=" }, { "name": "TheFilename2", "mime_type": "text/plain", - "data": "VGhlRGF0YTI" + "data": "VGhlRGF0YTI=" } ] } From bf0d0d265d1edc7629478a0d90bfae8d3765b16b Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Fri, 31 May 2024 21:30:54 +0100 Subject: [PATCH 7/9] fixed test sample --- .../cucumber/scenario_with_attachment | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/internal/formatters/formatter-tests/cucumber/scenario_with_attachment b/internal/formatters/formatter-tests/cucumber/scenario_with_attachment index da7f898d..633efae6 100644 --- a/internal/formatters/formatter-tests/cucumber/scenario_with_attachment +++ b/internal/formatters/formatter-tests/cucumber/scenario_with_attachment @@ -62,30 +62,6 @@ "data": "VGhlRGF0YTI=" } ] - }, - { - "keyword": "And ", - "name": "a step with multiple attachment calls", - "line": 8, - "match": { - "location": "fmt_output_test.go:119" - }, - "result": { - "status": "passed", - "duration": 0 - }, - "embeddings": [ - { - "name": "TheFilename1", - "mime_type": "text/plain", - "data": "VGhlRGF0YTE" - }, - { - "name": "TheFilename2", - "mime_type": "text/plain", - "data": "VGhlRGF0YTI" - } - ] } ] } From 9e8338617a0eeb257f3acae9ed9fe3f32986c2ac Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Fri, 21 Jun 2024 12:45:17 +0100 Subject: [PATCH 8/9] minor fixes to the attachments doco --- _examples/attachments/README.md | 2 +- _examples/attachments/attachments_test.go | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/_examples/attachments/README.md b/_examples/attachments/README.md index f37443e7..162bbad2 100644 --- a/_examples/attachments/README.md +++ b/_examples/attachments/README.md @@ -11,6 +11,6 @@ The example in this directory shows how the godog API is used to add attachments You must use the '-v' flag or you will not see the cucumber JSON output. -go test -v atttachment_test.go +go test -v attachments_test.go diff --git a/_examples/attachments/attachments_test.go b/_examples/attachments/attachments_test.go index 9d699405..3287bb63 100644 --- a/_examples/attachments/attachments_test.go +++ b/_examples/attachments/attachments_test.go @@ -1,15 +1,8 @@ package attachments_test -// This example shows how to set up test suite runner with Go subtests and godog command line parameters. -// Sample commands: -// * run all scenarios from default directory (features): go test -test.run "^TestFeatures/" -// * run all scenarios and list subtest names: go test -test.v -test.run "^TestFeatures/" -// * run all scenarios from one feature file: go test -test.v -godog.paths features/nodogs.feature -test.run "^TestFeatures/" -// * run all scenarios from multiple feature files: go test -test.v -godog.paths features/nodogs.feature,features/godogs.feature -test.run "^TestFeatures/" -// * run single scenario as a subtest: go test -test.v -test.run "^TestFeatures/Eat_5_out_of_12$" -// * show usage help: go test -godog.help -// * show usage help if there were other test files in directory: go test -godog.help godogs_test.go -// * run scenarios with multiple formatters: go test -test.v -godog.format cucumber:cuc.json,pretty -test.run "^TestFeatures/" +// This example shows how to attach data to the cucumber reports +// Run the sample with : go test -v attachments_test.go +// Then review the "embeddings" within the JSON emitted on the console. import ( "context" From 03ba46e2d2133a56e3ab5a5558bf6470defb1ff3 Mon Sep 17 00:00:00 2001 From: Johnlon <836248+Johnlon@users.noreply.github.com> Date: Fri, 21 Jun 2024 12:47:40 +0100 Subject: [PATCH 9/9] PostStepHook introduced to be what I think AfterStepHook ought to have been --- CHANGELOG.md | 1 + _examples/intercept/README.md | 13 +++ .../intercept/features/intercept.feature | 16 ++++ _examples/intercept/interceptor_test.go | 81 +++++++++++++++++++ suite.go | 21 ++++- test_context.go | 19 ++++- 6 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 _examples/intercept/README.md create mode 100644 _examples/intercept/features/intercept.feature create mode 100644 _examples/intercept/interceptor_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c5a6716..86c4d824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This document is formatted according to the principles of [Keep A CHANGELOG](htt ## Unreleased +- Provide support for a hooks to inject, replace or remove an error state from a step - ([TBD](https://github.com/cucumber/godog/pull/TBD) - [johnlon](https://github.com/johnlon)) - Provide support for attachments / embeddings including a new example in the examples dir - ([623](https://github.com/cucumber/godog/pull/623) - [johnlon](https://github.com/johnlon)) ## [v0.14.1] diff --git a/_examples/intercept/README.md b/_examples/intercept/README.md new file mode 100644 index 00000000..2ec90576 --- /dev/null +++ b/_examples/intercept/README.md @@ -0,0 +1,13 @@ +# An example of intercepting the step result for post-processing + +There are situations where it is useful to be able to post-process the outcome of all the steps in a suite in a generic manner in order to manipulate the status result, any errors returned or the context.Context. + +In order to facilitate this use case godog provides a seam where is is possible to inject a handler function to manipulate these values. + +## Run the example + +You must use the '-v' flag or you will not see the cucumber JSON output. + +go test -v interceptor_test.go + + diff --git a/_examples/intercept/features/intercept.feature b/_examples/intercept/features/intercept.feature new file mode 100644 index 00000000..4e11256f --- /dev/null +++ b/_examples/intercept/features/intercept.feature @@ -0,0 +1,16 @@ +Feature: Intercepting steps in order to manipulate the response + This test suite installs an interceptor that perfoms some manipulation on the test step result. + The manipulation in this test is arbitrary and for illustration purposes. + The interceptor inverts passing/failing results of any steps containing the text FLIP_ME. + + Scenario: The trigger word is not present so a pass status should be untouched + When passing step should be passed + + Scenario: The trigger word is not present so a fail status should be untouched + When failing step should be failed + + Scenario: Trigger word should should flip a fail to a pass + When failing step with the word FLIP_ME should be passed + + Scenario: Trigger word should should flip a pass to a fail + When passing step with the word FLIP_ME should be failed diff --git a/_examples/intercept/interceptor_test.go b/_examples/intercept/interceptor_test.go new file mode 100644 index 00000000..c6089221 --- /dev/null +++ b/_examples/intercept/interceptor_test.go @@ -0,0 +1,81 @@ +package interceptor_test + +// This example shows how to manipulate the results of test steps in a generic manner. +// Run the sample with : go test -v interceptor_test.go +// Then review the json report and confirm that the statuss report make sense given the scenarios. + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + + "github.com/cucumber/godog" + "github.com/cucumber/godog/colors" +) + +var opts = godog.Options{ + Output: colors.Colored(os.Stdout), + Format: "cucumber", // cucumber json format +} + +func TestFeatures(t *testing.T) { + o := opts + o.TestingT = t + + status := godog.TestSuite{ + Name: "intercept", + Options: &o, + ScenarioInitializer: InitializeScenario, + }.Run() + + if status == 2 { + t.SkipNow() + } + + if status != 0 { + t.Fatalf("zero status code expected, %d received", status) + } +} + +func InitializeScenario(ctx *godog.ScenarioContext) { + + ctx.StepContext().Post(func(ctx context.Context, st *godog.Step, status godog.StepResultStatus, err error) (context.Context, godog.StepResultStatus, error) { + if strings.Contains(st.Text, "FLIP_ME") { + if status == godog.StepFailed { + fmt.Printf("FLIP_ME to PASSED\n") + status = godog.StepPassed + err = nil // knock out any error too + } else if status == godog.StepPassed { + fmt.Printf("FLIP_ME to FAILED\n") + status = godog.StepFailed + err = fmt.Errorf("FLIP_ME to FAILED") + } else { + fmt.Printf("FLIP_ME but stays %v\n", status) + } + } else { + fmt.Printf("NOT FLIP_ME so stays %v\n", status) + } + fmt.Printf("FINAL STATUS %v, %v\n", status, err) + return ctx, status, err + }) + + ctx.Step(`^passing step should be passed$`, func(ctx context.Context) (context.Context, error) { + // this is a pass and we hope it will be reported as a pass + return ctx, nil + }) + ctx.Step(`^failing step should be failed$`, func(ctx context.Context) (context.Context, error) { + // this is a fail and we hope it will be reported as a fail + return ctx, fmt.Errorf("intentional failure") + }) + ctx.Step(`^passing step with the word FLIP_ME should be failed$`, func(ctx context.Context) (context.Context, error) { + // this is a pass but we hope it will be reported as a fail + return ctx, nil + }) + ctx.Step(`^failing step with the word FLIP_ME should be passed$`, func(ctx context.Context) (context.Context, error) { + // this is a fail but we hope it will be reported as a pass + return ctx, fmt.Errorf("this failure should be flipped to passed") + }) + +} diff --git a/suite.go b/suite.go index cf091805..3c25733b 100644 --- a/suite.go +++ b/suite.go @@ -64,6 +64,7 @@ type suite struct { // suite event handlers beforeScenarioHandlers []BeforeScenarioHook beforeStepHandlers []BeforeStepHook + postStepHandlers []PostStepHook afterStepHandlers []AfterStepHook afterScenarioHandlers []AfterScenarioHook } @@ -123,10 +124,9 @@ func (s *suite) runStep(ctx context.Context, pickle *Scenario, step *Step, scena var match *models.StepDefinition rctx = ctx - status := StepUndefined - // user multistep definitions may panic defer func() { + if e := recover(); e != nil { pe, isErr := e.(error) switch { @@ -154,6 +154,8 @@ func (s *suite) runStep(ctx context.Context, pickle *Scenario, step *Step, scena err = getTestingT(ctx).isFailed() } + status := StepUndefined + switch { case errors.Is(err, ErrPending): status = StepPending @@ -170,8 +172,11 @@ func (s *suite) runStep(ctx context.Context, pickle *Scenario, step *Step, scena pickledAttachments := pickleAttachments(ctx) ctx = clearAttach(ctx) + // Run post step handlers. + rctx, status, err = s.runPostStepHooks(ctx, step, status, err) + // Run after step handlers. - rctx, err = s.runAfterStepHooks(ctx, step, status, err) + rctx, err = s.runAfterStepHooks(rctx, step, status, err) shouldFail := s.shouldFail(err) @@ -296,6 +301,14 @@ func (s *suite) runBeforeStepHooks(ctx context.Context, step *Step, err error) ( return ctx, err } +func (s *suite) runPostStepHooks(ctx context.Context, step *Step, status StepResultStatus, err error) (context.Context, StepResultStatus, error) { + for _, f := range s.postStepHandlers { + ctx, status, err = f(ctx, step, status, err) + } + + return ctx, status, err +} + func (s *suite) runAfterStepHooks(ctx context.Context, step *Step, status StepResultStatus, err error) (context.Context, error) { for _, f := range s.afterStepHandlers { hctx, herr := f(ctx, step, status, err) @@ -462,6 +475,8 @@ func (s *suite) runSubStep(ctx context.Context, text string, def *models.StepDef status = StepFailed } + ctx, status, err = s.runPostStepHooks(ctx, st, status, err) + ctx, err = s.runAfterStepHooks(ctx, st, status, err) }() diff --git a/test_context.go b/test_context.go index b1006415..f1b3a5d5 100644 --- a/test_context.go +++ b/test_context.go @@ -148,12 +148,26 @@ func (ctx StepContext) Before(h BeforeStepHook) { // BeforeStepHook defines a hook before step. type BeforeStepHook func(ctx context.Context, st *Step) (context.Context, error) +// Post registers a function or method +// to be run directly after step to post process the result. +// This function is a decorator the allows complete manipulation of all features of the step results. +// The Post handler differs from an After handler in two important ways: +// - the After handler cannot modify the step status that is chained through the various hooks +// - the After step cannot replace an existing error with a new error, instead it appends errors together +func (ctx StepContext) Post(h PostStepHook) { + ctx.suite.postStepHandlers = append(ctx.suite.postStepHandlers, h) +} + // After registers a function or method // to be run after every step. // // It may be convenient to return a different kind of error // in order to print more state details which may help -// in case of step failure +// in case of step failure. +// Any errors returned by an After handler do not replace the errors returned by the step but +// instead are prepended to the step error messages. +// If you want to entirely replace the status, error or context returned by the step then use a +// Post handler. // // In some cases, for example when running a headless // browser, to take a screenshot after failure. @@ -161,6 +175,9 @@ func (ctx StepContext) After(h AfterStepHook) { ctx.suite.afterStepHandlers = append(ctx.suite.afterStepHandlers, h) } +// PostStepHook defines a hook as a suffix to the step to allow manipulation of step step status, error or context in a generic manner. +type PostStepHook func(ctx context.Context, st *Step, status StepResultStatus, err error) (context.Context, StepResultStatus, error) + // AfterStepHook defines a hook after step. type AfterStepHook func(ctx context.Context, st *Step, status StepResultStatus, err error) (context.Context, error)