diff --git a/cmd/falco/help.go b/cmd/falco/help.go index ab1a29ec..0ce15133 100644 --- a/cmd/falco/help.go +++ b/cmd/falco/help.go @@ -30,7 +30,7 @@ func printSplash() { writeln(white, strings.TrimSpace(` ========================================================= ____ __ - / __/______ / /_____ ____ + / __/______ / /_____ ____ / /_ / __ `+` // // __// __ \ / __// /_/ // // /__ / /_/ / /_/ \____//_/ \___/ \____/ Fastly VCL developer tool @@ -145,6 +145,7 @@ Flags: -r, --remote : Connect with Fastly API -t, --timeout : Set timeout to running test -f, --filter : Override glob filter to find test files + -c, --coverage : Collect test coverage information and report it in the output -json : Output results as JSON -request : Override request config --max_backends : Override max backends limitation diff --git a/cmd/falco/main.go b/cmd/falco/main.go index e56fa239..1c895694 100644 --- a/cmd/falco/main.go +++ b/cmd/falco/main.go @@ -289,11 +289,13 @@ func runTest(runner *Runner, rslv resolver.Resolver) error { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") if err := enc.Encode(struct { - Tests []*tester.TestResult `json:"tests"` - Summary *tester.TestCounter `json:"summary"` + Tests []*tester.TestResult `json:"tests"` + Summary *tester.TestCounter `json:"summary"` + Coverage *tester.TestCoverage `json:"coverage,omitempty"` }{ - Tests: factory.Results, - Summary: factory.Statistics, + Tests: factory.Results, + Summary: factory.Statistics, + Coverage: factory.Coverage, }); err != nil { writeln(red, err.Error()) return ErrExit @@ -379,6 +381,13 @@ func runTest(runner *Runner, rslv resolver.Resolver) error { write(white, "%d total, ", totalCount) writeln(white, "%d assertions", factory.Statistics.Asserts) + if factory.Coverage != nil { + write(white, "Coverage: ") + write(white, "%% Stmts: %.2f, ", factory.Coverage.Statement*100) + write(white, "%% Branch: %.2f, ", factory.Coverage.Branch*100) + writeln(white, "%% Funcs: %.2f", factory.Coverage.Function*100) + } + if factory.Statistics.Fails > 0 { return ErrExit } diff --git a/config/config.go b/config/config.go index 2e2b8687..d5058aea 100644 --- a/config/config.go +++ b/config/config.go @@ -62,6 +62,7 @@ type SimulatorConfig struct { type TestConfig struct { Timeout int `cli:"t,timeout" yaml:"timeout"` Filter string `cli:"f,filter" default:"*.test.vcl"` + Coverage bool `cli:"c,coverage" yaml:"coverage" default:"false"` IncludePaths []string // Copy from root field OverrideHost string `yaml:"host"` diff --git a/docs/configuration.md b/docs/configuration.md index 5ac02036..97c2f8cf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,7 @@ Here is a full configuration file example: // .falco.yaml ## Basic configurations -include_paths: [".", "/path/to/include"] +include_paths: [".", "/path/to/include"] remote: true max_backends: 5 max_acls: 1000 @@ -65,6 +65,7 @@ All configurations of configuration files and CLI arguments are described follow | simulator.edge_dictionary.[name] | Object | - | - | Local edge dictionary name | | testing | Object | null | - | Testing configuration object | | testing.timeout | Integer | 10 | -t, --timeout | Set timeout to stop testing | +| testing.coverage | Boolean | false | -c, --coverage | Collect test coverage information and report it in the output | | linter | Object | null | - | Override linter rules | | linter.verbose | String | error | -v, -vv | Verbose level, `warning` or `info` is valid | | linter.rules | Object | null | - | Override linter rules | diff --git a/interpreter/coverage.go b/interpreter/coverage.go new file mode 100644 index 00000000..89d72169 --- /dev/null +++ b/interpreter/coverage.go @@ -0,0 +1,429 @@ +package interpreter + +import ( + "strconv" + + "github.com/ysugimoto/falco/ast" + "github.com/ysugimoto/falco/token" +) + +const ( + FUNCTION_COVERAGE = "falco_coverage_function" + STATEMENT_COVERAGE = "falco_coverage_statement" + BRANCH_COVERAGE = "falco_coverage_branch" +) + +func getFunctionId(s ast.SubroutineDeclaration) string { + return s.GetMeta().Token.File + "_" + s.Name.Value +} + +func getStatementId(stmt ast.Statement) string { + t := stmt.GetMeta().Token + l := strconv.Itoa(t.Line) + p := strconv.Itoa(t.Position) + return t.File + "_stmt_l" + l + "_p" + p +} + +func getExpressionId(exp ast.Expression) string { + t := exp.GetMeta().Token + l := strconv.Itoa(t.Line) + p := strconv.Itoa(t.Position) + return t.File + "_exp_l" + l + "_p" + p +} + +func (i *Interpreter) instrument() { + i.ctx.Tables[FUNCTION_COVERAGE] = &ast.TableDeclaration{ + Meta: &ast.Meta{Token: token.Null}, + Name: &ast.Ident{Value: FUNCTION_COVERAGE}, + ValueType: &ast.Ident{Value: "STRING"}, + Properties: []*ast.TableProperty{}, + } + i.ctx.Tables[STATEMENT_COVERAGE] = &ast.TableDeclaration{ + Meta: &ast.Meta{Token: token.Null}, + Name: &ast.Ident{Value: STATEMENT_COVERAGE}, + ValueType: &ast.Ident{Value: "STRING"}, + Properties: []*ast.TableProperty{}, + } + i.ctx.Tables[BRANCH_COVERAGE] = &ast.TableDeclaration{ + Meta: &ast.Meta{Token: token.Null}, + Name: &ast.Ident{Value: BRANCH_COVERAGE}, + ValueType: &ast.Ident{Value: "STRING"}, + Properties: []*ast.TableProperty{}, + } + + for _, sub := range i.ctx.Subroutines { + i.instrumentSubroutine(sub) + } + for _, sub := range i.ctx.SubroutineFunctions { + i.instrumentSubroutine(sub) + } +} + +func (i *Interpreter) instrumentSubroutine(sub *ast.SubroutineDeclaration) { + id := getFunctionId(*sub) + + i.ctx.Tables[FUNCTION_COVERAGE].Properties = append( + i.ctx.Tables[FUNCTION_COVERAGE].Properties, + createInitialTableProperty(id), + ) + + sub.Block.Statements = append( + []ast.Statement{ + createMarkAsCovered(FUNCTION_COVERAGE, id), + }, + i.instrumentStatements(sub.Block.Statements)..., + ) +} + +func (i *Interpreter) instrumentStatements(stmts []ast.Statement) []ast.Statement { + var result []ast.Statement + + for _, stmt := range stmts { + result = append(result, i.instrumentStatement(stmt)...) + result = append(result, i.instrumentExpressionInsideStatement(stmt)...) + result = append(result, stmt) + } + + return result +} + +func (i *Interpreter) instrumentStatement(stmt ast.Statement) []ast.Statement { + var result []ast.Statement + + switch s := stmt.(type) { + case *ast.BlockStatement: + s.Statements = i.instrumentStatements(s.Statements) + + case *ast.IfStatement: + result = append( + result, + i.instrumentIfStatement(s)..., + ) + + case *ast.SwitchStatement: + result = append( + result, + i.instrumentSwitchStatement(s)..., + ) + + default: + stmtId := getStatementId(stmt) + + i.ctx.Tables[STATEMENT_COVERAGE].Properties = append( + i.ctx.Tables[STATEMENT_COVERAGE].Properties, + createInitialTableProperty(stmtId), + ) + + result = append( + result, + createMarkAsCovered(STATEMENT_COVERAGE, stmtId), + ) + } + + return result +} + +func (i *Interpreter) instrumentIfStatement(stmt *ast.IfStatement) []ast.Statement { + var result []ast.Statement + + result = append( + result, + createMarkAsCoveredForIfStatement(stmt)..., + ) + + ifs := append( + []*ast.IfStatement{stmt}, + stmt.Another..., + ) + + for _, s := range ifs { + stmtId := getStatementId(s) + + i.ctx.Tables[STATEMENT_COVERAGE].Properties = append( + i.ctx.Tables[STATEMENT_COVERAGE].Properties, + createInitialTableProperty(stmtId), + ) + i.ctx.Tables[BRANCH_COVERAGE].Properties = append( + i.ctx.Tables[BRANCH_COVERAGE].Properties, + createInitialTableProperty(stmtId+"_true"), + createInitialTableProperty(stmtId+"_false"), + ) + s.Consequence.Statements = i.instrumentStatements(s.Consequence.Statements) + } + + if stmt.Alternative != nil { + elseStmt := stmt.Alternative + stmtId := getStatementId(elseStmt) + + i.ctx.Tables[STATEMENT_COVERAGE].Properties = append( + i.ctx.Tables[STATEMENT_COVERAGE].Properties, + createInitialTableProperty(stmtId), + ) + elseStmt.Consequence.Statements = i.instrumentStatements(elseStmt.Consequence.Statements) + } + + return result +} + +func (i *Interpreter) instrumentSwitchStatement(stmt *ast.SwitchStatement) []ast.Statement { + var result []ast.Statement + + i.ctx.Tables[STATEMENT_COVERAGE].Properties = append( + i.ctx.Tables[STATEMENT_COVERAGE].Properties, + createInitialTableProperty(getStatementId(stmt.Control)), + ) + + result = append( + result, + createMarkAsCovered(STATEMENT_COVERAGE, getStatementId(stmt.Control)), + ) + + for _, c := range stmt.Cases { + i.ctx.Tables[STATEMENT_COVERAGE].Properties = append( + i.ctx.Tables[STATEMENT_COVERAGE].Properties, + createInitialTableProperty(getStatementId(c)), + ) + i.ctx.Tables[BRANCH_COVERAGE].Properties = append( + i.ctx.Tables[BRANCH_COVERAGE].Properties, + createInitialTableProperty(getStatementId(c)), + ) + + c.Statements = append( + []ast.Statement{ + createMarkAsCovered(STATEMENT_COVERAGE, getStatementId(c)), + createMarkAsCovered(BRANCH_COVERAGE, getStatementId(c)), + }, + i.instrumentStatements(c.Statements)..., + ) + } + + return result +} + +func (i *Interpreter) instrumentExpressionInsideStatement(stmt ast.Statement) []ast.Statement { + var result []ast.Statement + + switch s := stmt.(type) { + case *ast.AddStatement: + result = append(result, i.instrumentExpression(s.Value)...) + + case *ast.ErrorStatement: + result = append(result, i.instrumentExpression(s.Code)...) + result = append(result, i.instrumentExpression(s.Argument)...) + + case *ast.FunctionCallStatement: + for _, arg := range s.Arguments { + result = append(result, i.instrumentExpression(arg)...) + } + + case *ast.IfStatement: + result = append(result, i.instrumentExpression(s.Condition)...) + for _, a := range s.Another { + result = append(result, i.instrumentExpression(a.Condition)...) + } + + case *ast.LogStatement: + result = append(result, i.instrumentExpression(s.Value)...) + + case *ast.ReturnStatement: + result = append(result, i.instrumentExpression(s.ReturnExpression)...) + + case *ast.SetStatement: + result = append(result, i.instrumentExpression(s.Value)...) + + case *ast.SwitchStatement: + result = append(result, i.instrumentExpression(s.Control.Expression)...) + for _, c := range s.Cases { + if c.Test != nil { + result = append(result, i.instrumentExpression(c.Test)...) + } + } + + case *ast.SyntheticBase64Statement: + result = append(result, i.instrumentExpression(s.Value)...) + + case *ast.SyntheticStatement: + result = append(result, i.instrumentExpression(s.Value)...) + } + + return result +} + +func (i *Interpreter) instrumentExpression(exp ast.Expression) []ast.Statement { + var result []ast.Statement + + switch e := exp.(type) { + case *ast.FunctionCallExpression: + for _, arg := range e.Arguments { + result = append(result, i.instrumentExpression(arg)...) + } + + case *ast.GroupedExpression: + result = append(result, i.instrumentExpression(e.Right)...) + + case *ast.InfixExpression: + result = append(result, i.instrumentExpression(e.Left)...) + result = append(result, i.instrumentExpression(e.Right)...) + + case *ast.PrefixExpression: + result = append(result, i.instrumentExpression(e.Right)...) + + case *ast.PostfixExpression: + result = append(result, i.instrumentExpression(e.Left)...) + + case *ast.IfExpression: + result = append(result, i.instrumentIfExpression(e)) + } + + return result +} + +func (i *Interpreter) instrumentIfExpression(exp *ast.IfExpression) ast.Statement { + markAsBranchCovered := createMarkAsBranchCovered(exp.Condition, getExpressionId(exp)) + + i.ctx.Tables[BRANCH_COVERAGE].Properties = append( + i.ctx.Tables[BRANCH_COVERAGE].Properties, + createInitialTableProperty(getExpressionId(exp)+"_true"), + createInitialTableProperty(getExpressionId(exp)+"_false"), + ) + + markAsBranchCovered.Consequence.Statements = append( + markAsBranchCovered.Consequence.Statements, + i.instrumentExpression(exp.Consequence)..., + ) + + markAsBranchCovered.Alternative.Consequence.Statements = append( + markAsBranchCovered.Alternative.Consequence.Statements, + i.instrumentExpression(exp.Alternative)..., + ) + + return markAsBranchCovered +} + +func createInitialTableProperty(key string) *ast.TableProperty { + return &ast.TableProperty{ + Key: &ast.String{ + Meta: &ast.Meta{ + Token: token.Token{Type: token.STRING, Literal: key}, + }, + Value: key, + }, + Value: &ast.String{ + Meta: &ast.Meta{ + Token: token.Token{Type: token.STRING, Literal: "false"}, + }, + Value: "false", + }, + } +} + +func createMarkAsCovered(table, key string) *ast.FunctionCallStatement { + return &ast.FunctionCallStatement{ + Meta: &ast.Meta{Token: token.Null}, + Function: &ast.Ident{ + Meta: &ast.Meta{ + Token: token.Token{Type: token.STRING, Literal: "testing.table_set"}, + }, + Value: "testing.table_set", + }, + Arguments: []ast.Expression{ + &ast.Ident{ + Meta: &ast.Meta{ + Token: token.Token{Type: token.IDENT, Literal: table}, + }, + Value: table, + }, + &ast.String{ + Meta: &ast.Meta{ + Token: token.Token{Type: token.STRING, Literal: key}, + }, + Value: key, + }, + &ast.String{ + Meta: &ast.Meta{ + Token: token.Token{Type: token.STRING, Literal: "true"}, + }, + Value: "true", + }, + }, + } +} + +func createMarkAsCoveredForIfStatement(ifStmt *ast.IfStatement) []ast.Statement { + var result []ast.Statement + var current *ast.BlockStatement = nil + + ifs := append( + []*ast.IfStatement{ifStmt}, + ifStmt.Another..., + ) + + for _, s := range ifs { + markAsStatementCovered := createMarkAsCovered(STATEMENT_COVERAGE, getStatementId(s)) + markAsBranchCovered := createMarkAsBranchCovered(s.Condition, getStatementId(s)) + + if current == nil { + result = append(result, markAsStatementCovered, markAsBranchCovered) + } else { + current.Statements = append(current.Statements, markAsStatementCovered, markAsBranchCovered) + } + + current = markAsBranchCovered.Alternative.Consequence + } + + return result +} + +func createMarkAsBranchCovered(condition ast.Expression, baseId string) *ast.IfStatement { + return &ast.IfStatement{ + Meta: &ast.Meta{Token: token.Null}, + Keyword: "if", + Condition: condition, + Another: []*ast.IfStatement{}, + Consequence: &ast.BlockStatement{ + Meta: &ast.Meta{Token: token.Null}, + Statements: []ast.Statement{ + createMarkAsCovered(BRANCH_COVERAGE, baseId+"_true"), + }, + }, + Alternative: &ast.ElseStatement{ + Meta: &ast.Meta{Token: token.Null}, + Consequence: &ast.BlockStatement{ + Meta: &ast.Meta{Token: token.Null}, + Statements: []ast.Statement{ + createMarkAsCovered(BRANCH_COVERAGE, baseId+"_false"), + }, + }, + }, + } +} + +type Coverage struct { + Function CoverageTable + Statement CoverageTable + Branch CoverageTable +} + +type CoverageTable map[string]bool + +func (i *Interpreter) GetCoverage() Coverage { + if i.ctx.Tables[FUNCTION_COVERAGE] == nil || i.ctx.Tables[STATEMENT_COVERAGE] == nil || i.ctx.Tables[BRANCH_COVERAGE] == nil { + return Coverage{} + } + + return Coverage{ + Function: convertToCoverageTable(i.ctx.Tables[FUNCTION_COVERAGE]), + Statement: convertToCoverageTable(i.ctx.Tables[STATEMENT_COVERAGE]), + Branch: convertToCoverageTable(i.ctx.Tables[BRANCH_COVERAGE]), + } +} + +func convertToCoverageTable(decl *ast.TableDeclaration) CoverageTable { + table := make(CoverageTable) + + for _, prop := range decl.Properties { + table[prop.Key.Value] = prop.Value.(*ast.String).Value == "true" + } + + return table +} diff --git a/interpreter/coverage_test.go b/interpreter/coverage_test.go new file mode 100644 index 00000000..c8a9dc0a --- /dev/null +++ b/interpreter/coverage_test.go @@ -0,0 +1,325 @@ +package interpreter + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ysugimoto/falco/ast" + "github.com/ysugimoto/falco/config" + "github.com/ysugimoto/falco/formatter" + "github.com/ysugimoto/falco/interpreter/context" + "github.com/ysugimoto/falco/resolver" +) + +func streamToString(stream io.Reader) string { + buf := new(bytes.Buffer) + buf.ReadFrom(stream) + return buf.String() +} + +func assertSubroutine(t *testing.T, expected *ast.SubroutineDeclaration, actual *ast.SubroutineDeclaration) { + f := formatter.New(&config.FormatConfig{}) + expectedCode := streamToString(f.Format(&ast.VCL{Statements: []ast.Statement{expected}})) + actualCode := streamToString(f.Format(&ast.VCL{Statements: []ast.Statement{actual}})) + + if expectedCode != actualCode { + t.Errorf("Subroutine %s is not matched:\n%s\nExpected:\n%s\nActual:\n%s", actual.Name.Value, cmp.Diff(expectedCode, actualCode), expectedCode, actualCode) + } +} + +func TestInstrument(t *testing.T) { + tests := []struct { + name string + main string + instrumented string + expectedTable Coverage + }{ + { + name: "Function and statement coverage", + main: ` +sub func1 { + set req.http.Foo = "foo"; +} +sub func2 { + set req.http.Bar = "bar"; +} + `, + instrumented: ` +sub func1 { + testing.table_set(falco_coverage_function, "main_func1", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l3_p3", "true"); + set req.http.Foo = "foo"; +} +sub func2 { + testing.table_set(falco_coverage_function, "main_func2", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l6_p3", "true"); + set req.http.Bar = "bar"; +} + `, + expectedTable: Coverage{ + Function: CoverageTable{ + "main_func1": false, + "main_func2": false, + }, + Statement: CoverageTable{ + "main_stmt_l3_p3": false, + "main_stmt_l6_p3": false, + }, + Branch: CoverageTable{}, + }, + }, + { + name: "Branch coverage: if statement", + main: ` +sub func1 { + if (req.http.Foo == "1") { + set req.http.Bar = "1"; + } else if (req.http.Foo == "2") { + set req.http.Bar = "2"; + } else { + if (req.http.Foo == "3") { + set req.http.Bar = "3"; + } + set req.http.Bar = "4"; + } +} + `, + instrumented: ` +sub func1 { + testing.table_set(falco_coverage_function, "main_func1", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l3_p3", "true"); + if (req.http.Foo == "1") { + testing.table_set(falco_coverage_branch, "main_stmt_l3_p3_true", "true"); + } else { + testing.table_set(falco_coverage_branch, "main_stmt_l3_p3_false", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l5_p10", "true"); + if (req.http.Foo == "2") { + testing.table_set(falco_coverage_branch, "main_stmt_l5_p10_true", "true"); + } else { + testing.table_set(falco_coverage_branch, "main_stmt_l5_p10_false", "true"); + } + } + if (req.http.Foo == "1") { + testing.table_set(falco_coverage_statement, "main_stmt_l4_p5", "true"); + set req.http.Bar = "1"; + } else if (req.http.Foo == "2") { + testing.table_set(falco_coverage_statement, "main_stmt_l6_p5", "true"); + set req.http.Bar = "2"; + } else { + testing.table_set(falco_coverage_statement, "main_stmt_l8_p5", "true"); + if (req.http.Foo == "3") { + testing.table_set(falco_coverage_branch, "main_stmt_l8_p5_true", "true"); + } else { + testing.table_set(falco_coverage_branch, "main_stmt_l8_p5_false", "true"); + } + if (req.http.Foo == "3") { + testing.table_set(falco_coverage_statement, "main_stmt_l9_p7", "true"); + set req.http.Bar = "3"; + } + testing.table_set(falco_coverage_statement, "main_stmt_l11_p5", "true"); + set req.http.Bar = "4"; + } +} + `, + expectedTable: Coverage{ + Function: CoverageTable{ + "main_func1": false, + }, + Statement: CoverageTable{ + "main_stmt_l3_p3": false, + "main_stmt_l4_p5": false, + "main_stmt_l5_p10": false, + "main_stmt_l6_p5": false, + "main_stmt_l7_p5": false, + "main_stmt_l8_p5": false, + "main_stmt_l9_p7": false, + "main_stmt_l11_p5": false, + }, + Branch: CoverageTable{ + "main_stmt_l3_p3_false": false, + "main_stmt_l3_p3_true": false, + "main_stmt_l5_p10_false": false, + "main_stmt_l5_p10_true": false, + "main_stmt_l8_p5_false": false, + "main_stmt_l8_p5_true": false, + }, + }, + }, + { + name: "Branch coverage: switch statement", + main: ` +sub func1 { + switch(req.http.Foo){ + case "1": + set req.http.Bar = "1"; + break; + case "2": + set req.http.Bar = "2"; + break; + default: + set req.http.Bar = "3"; + break; + } +} + `, + instrumented: ` +sub func1 { + testing.table_set(falco_coverage_function, "main_func1", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l3_p9", "true"); + switch (req.http.Foo) { + case "1": + testing.table_set(falco_coverage_statement, "main_stmt_l4_p5", "true"); + testing.table_set(falco_coverage_branch, "main_stmt_l4_p5", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l5_p7", "true"); + set req.http.Bar = "1"; + testing.table_set(falco_coverage_statement, "main_stmt_l6_p7", "true"); + break; + case "2": + testing.table_set(falco_coverage_statement, "main_stmt_l7_p5", "true"); + testing.table_set(falco_coverage_branch, "main_stmt_l7_p5", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l8_p7", "true"); + set req.http.Bar = "2"; + testing.table_set(falco_coverage_statement, "main_stmt_l9_p7", "true"); + break; + default: + testing.table_set(falco_coverage_statement, "main_stmt_l10_p5", "true"); + testing.table_set(falco_coverage_branch, "main_stmt_l10_p5", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l11_p7", "true"); + set req.http.Bar = "3"; + testing.table_set(falco_coverage_statement, "main_stmt_l12_p7", "true"); + break; + } +} + `, + expectedTable: Coverage{ + Function: CoverageTable{ + "main_func1": false, + }, + Statement: CoverageTable{ + "main_stmt_l3_p9": false, + "main_stmt_l4_p5": false, + "main_stmt_l5_p7": false, + "main_stmt_l6_p7": false, + "main_stmt_l7_p5": false, + "main_stmt_l8_p7": false, + "main_stmt_l9_p7": false, + "main_stmt_l10_p5": false, + "main_stmt_l11_p7": false, + "main_stmt_l12_p7": false, + }, + Branch: CoverageTable{ + "main_stmt_l4_p5": false, + "main_stmt_l7_p5": false, + "main_stmt_l10_p5": false, + }, + }, + }, + { + name: "If expression", + main: ` +sub func1 { + error 600 if(req.http.Foo == "1", "1", "2"); + header.set(req, "bar", if(req.http.Foo == "1", "1", "2")); + set req.http.Bar = if(req.http.Foo == "1", "1", "bar_" + if(req.http.Foo == "2", "2", "3")); +} + `, + instrumented: ` +sub func1 { + testing.table_set(falco_coverage_function, "main_func1", "true"); + testing.table_set(falco_coverage_statement, "main_stmt_l3_p3", "true"); + if (req.http.Foo == "1") { + testing.table_set(falco_coverage_branch, "main_exp_l3_p13_true", "true"); + } else { + testing.table_set(falco_coverage_branch, "main_exp_l3_p13_false", "true"); + } + error 600 if(req.http.Foo == "1", "1", "2"); + testing.table_set(falco_coverage_statement, "main_stmt_l4_p3", "true"); + if (req.http.Foo == "1") { + testing.table_set(falco_coverage_branch, "main_exp_l4_p26_true", "true"); + } else { + testing.table_set(falco_coverage_branch, "main_exp_l4_p26_false", "true"); + } + header.set(req, "bar", if(req.http.Foo == "1", "1", "2")); + testing.table_set(falco_coverage_statement, "main_stmt_l5_p3", "true"); + if (req.http.Foo == "1") { + testing.table_set(falco_coverage_branch, "main_exp_l5_p22_true", "true"); + } else { + testing.table_set(falco_coverage_branch, "main_exp_l5_p22_false", "true"); + if (req.http.Foo == "2") { + testing.table_set(falco_coverage_branch, "main_exp_l5_p60_true", "true"); + } else { + testing.table_set(falco_coverage_branch, "main_exp_l5_p60_false", "true"); + } + } + set req.http.Bar = if(req.http.Foo == "1", "1", "bar_" + if(req.http.Foo == "2", "2", "3")); +} + `, + expectedTable: Coverage{ + Function: CoverageTable{ + "main_func1": false, + }, + Statement: CoverageTable{ + "main_stmt_l3_p3": false, + "main_stmt_l4_p3": false, + "main_stmt_l5_p3": false, + }, + Branch: CoverageTable{ + "main_exp_l3_p13_false": false, + "main_exp_l3_p13_true": false, + "main_exp_l4_p26_false": false, + "main_exp_l4_p26_true": false, + "main_exp_l5_p22_false": false, + "main_exp_l5_p22_true": false, + "main_exp_l5_p60_false": false, + "main_exp_l5_p60_true": false, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name+": instrumented code", func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "http://localhost", nil) + actualI := New(context.WithResolver( + resolver.NewStaticResolver("main", tt.main), + )) + actualI.ProcessInit(r) + expectedI := New(context.WithResolver( + resolver.NewStaticResolver("instrumented", tt.instrumented), + )) + expectedI.ProcessInit(r) + + actualI.instrument() + + for name, actual := range actualI.ctx.Subroutines { + expected := expectedI.ctx.Subroutines[name] + assertSubroutine(t, expected, actual) + } + }) + + t.Run(tt.name+": coverage table", func(t *testing.T) { + i := New(context.WithResolver( + resolver.NewStaticResolver("main", tt.main), + )) + r := httptest.NewRequest(http.MethodGet, "http://localhost", nil) + i.ProcessInit(r) + + i.instrument() + + coverage := i.GetCoverage() + if diff := cmp.Diff(tt.expectedTable.Function, coverage.Function); diff != "" { + t.Errorf("Function coverage is not matched: %s", diff) + } + if diff := cmp.Diff(tt.expectedTable.Statement, coverage.Statement); diff != "" { + t.Errorf("Statement coverage is not matched: %s", diff) + } + if diff := cmp.Diff(tt.expectedTable.Branch, coverage.Branch); diff != "" { + t.Errorf("Branch coverage is not matched: %s", diff) + } + }) + } +} diff --git a/interpreter/testing.go b/interpreter/testing.go index f0a0248d..286b1680 100644 --- a/interpreter/testing.go +++ b/interpreter/testing.go @@ -8,13 +8,14 @@ import ( "github.com/pkg/errors" "github.com/ysugimoto/falco/ast" + "github.com/ysugimoto/falco/config" icontext "github.com/ysugimoto/falco/interpreter/context" "github.com/ysugimoto/falco/interpreter/value" ) const testBackendResponseBody = "falco_test_response" -func (i *Interpreter) TestProcessInit(r *http.Request) error { +func (i *Interpreter) TestProcessInit(r *http.Request, c *config.TestConfig) error { var err error if err = i.ProcessInit(r); err != nil { return errors.WithStack(err) @@ -56,6 +57,11 @@ func (i *Interpreter) TestProcessInit(r *http.Request) error { } i.ctx.Response = i.cloneResponse(i.ctx.BackendResponse) i.ctx.Object = i.cloneResponse(i.ctx.BackendResponse) + + if c.Coverage { + i.instrument() + } + return nil } diff --git a/tester/coverage.go b/tester/coverage.go new file mode 100644 index 00000000..bff29b86 --- /dev/null +++ b/tester/coverage.go @@ -0,0 +1,66 @@ +package tester + +import i "github.com/ysugimoto/falco/interpreter" + +type TestCoverage struct { + Function float64 `json:"function"` + Statement float64 `json:"statement"` + Branch float64 `json:"branch"` +} + +func getCoverage(results []*TestResult) *TestCoverage { + merged := i.Coverage{ + Function: make(i.CoverageTable), + Statement: make(i.CoverageTable), + Branch: make(i.CoverageTable), + } + + for _, r := range results { + for _, c := range r.Cases { + for k, v := range c.Coverage.Function { + if _, ok := merged.Function[k]; !ok { + merged.Function[k] = false + } + if v { + merged.Function[k] = true + } + } + for k, v := range c.Coverage.Statement { + if _, ok := merged.Statement[k]; !ok { + merged.Statement[k] = false + } + if v { + merged.Statement[k] = true + } + } + for k, v := range c.Coverage.Branch { + if _, ok := merged.Branch[k]; !ok { + merged.Branch[k] = false + } + if v { + merged.Branch[k] = true + } + } + } + } + + return &TestCoverage{ + Function: calculateCoverage(merged.Function), + Statement: calculateCoverage(merged.Statement), + Branch: calculateCoverage(merged.Branch), + } +} + +func calculateCoverage(table i.CoverageTable) float64 { + var covered, total int + for _, v := range table { + total++ + if v { + covered++ + } + } + if total == 0 { + return 0 + } + return float64(covered) / float64(total) +} diff --git a/tester/coverage_test.go b/tester/coverage_test.go new file mode 100644 index 00000000..8392dc73 --- /dev/null +++ b/tester/coverage_test.go @@ -0,0 +1,128 @@ +package tester + +import ( + "testing" + + "github.com/ysugimoto/falco/interpreter" +) + +func TestGetCoverage(t *testing.T) { + tests := []struct { + name string + results []*TestResult + assertions TestCoverage + }{ + { + name: "should merge coverage tables of all test cases", + results: []*TestResult{ + { + Cases: []*TestCase{ + { + Coverage: interpreter.Coverage{ + Function: interpreter.CoverageTable{ + "func1": false, + "func2": false, + }, + Statement: interpreter.CoverageTable{ + "stmt1": true, + "stmt2": false, + }, + Branch: interpreter.CoverageTable{ + "branch1": true, + "branch2": true, + }, + }, + }, + { + Coverage: interpreter.Coverage{ + Function: interpreter.CoverageTable{ + "func1": true, + "func2": false, + }, + Statement: interpreter.CoverageTable{ + "stmt1": true, + "stmt2": true, + }, + Branch: interpreter.CoverageTable{ + "branch1": true, + "branch2": false, + }, + }, + }, + }, + }, + }, + assertions: TestCoverage{ + Function: 0.5, + Statement: 1.0, + Branch: 1.0, + }, + }, + { + name: "should merge coverage tables of all test results", + results: []*TestResult{ + { + Cases: []*TestCase{ + { + Coverage: interpreter.Coverage{ + Function: interpreter.CoverageTable{ + "func1": false, + "func2": false, + }, + Statement: interpreter.CoverageTable{ + "stmt1": true, + "stmt2": false, + }, + Branch: interpreter.CoverageTable{ + "branch1": true, + "branch2": true, + }, + }, + }, + }, + }, + { + Cases: []*TestCase{ + { + Coverage: interpreter.Coverage{ + Function: interpreter.CoverageTable{ + "func1": true, + "func2": false, + }, + Statement: interpreter.CoverageTable{ + "stmt1": true, + "stmt2": true, + }, + Branch: interpreter.CoverageTable{ + "branch1": true, + "branch2": false, + }, + }, + }, + }, + }, + }, + assertions: TestCoverage{ + Function: 0.5, + Statement: 1.0, + Branch: 1.0, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := getCoverage(tt.results) + + if c.Function != tt.assertions.Function { + t.Errorf("Unexpected function coverage: %v\n", c.Function) + } + if c.Statement != tt.assertions.Statement { + t.Errorf("Unexpected statement coverage: %v\n", c.Statement) + } + if c.Branch != tt.assertions.Branch { + t.Errorf("Unexpected branch coverage: %v\n", c.Branch) + } + }) + } +} diff --git a/tester/entity.go b/tester/entity.go index 1ededa20..51d38436 100644 --- a/tester/entity.go +++ b/tester/entity.go @@ -3,16 +3,18 @@ package tester import ( "encoding/json" + i "github.com/ysugimoto/falco/interpreter" "github.com/ysugimoto/falco/interpreter/function/errors" "github.com/ysugimoto/falco/lexer" ) type TestCase struct { - Name string - Group string - Error error - Scope string - Time int64 // msec order + Name string + Group string + Error error + Scope string + Time int64 // msec order + Coverage i.Coverage } func (t *TestCase) MarshalJSON() ([]byte, error) { @@ -59,6 +61,7 @@ func (t *TestResult) IsPassed() bool { type TestFactory struct { Results []*TestResult Statistics *TestCounter + Coverage *TestCoverage Logs []string } diff --git a/tester/tester.go b/tester/tester.go index 76b30714..c144056a 100644 --- a/tester/tester.go +++ b/tester/tester.go @@ -82,10 +82,16 @@ func (t *Tester) Run(main string) (*TestFactory, error) { } results = append(results, result) } + // Calculate coverage + var coverage *TestCoverage + if t.config.Coverage { + coverage = getCoverage(results) + } return &TestFactory{ Results: results, Statistics: t.counter, + Coverage: coverage, Logs: t.debugger.stack, }, nil } @@ -138,7 +144,7 @@ func (t *Tester) run(testFile string) (*TestResult, error) { i := t.setupInterpreter(defs) mockRequest := httptest.NewRequest(http.MethodGet, "http://localhost", nil) - if err := i.TestProcessInit(mockRequest); err != nil { + if err := i.TestProcessInit(mockRequest, t.config); err != nil { errChan <- errors.WithStack(err) return } @@ -147,10 +153,11 @@ func (t *Tester) run(testFile string) (*TestResult, error) { start := time.Now() err := i.ProcessTestSubroutine(s, st) cases = append(cases, &TestCase{ - Name: suite, - Error: errors.Cause(err), - Scope: s.String(), - Time: time.Since(start).Milliseconds(), + Name: suite, + Error: errors.Cause(err), + Scope: s.String(), + Time: time.Since(start).Milliseconds(), + Coverage: i.GetCoverage(), }) if err != nil { t.counter.Fail() @@ -188,7 +195,7 @@ func (t *Tester) runDescribedTests( // describe should run as group testing, create interpreter once through tests i := t.setupInterpreter(defs) - if err := i.TestProcessInit(mockRequest); err != nil { + if err := i.TestProcessInit(mockRequest, t.config); err != nil { return cases, err } @@ -222,11 +229,12 @@ func (t *Tester) runDescribedTests( start := time.Now() err := i.ProcessTestSubroutine(s, sub) cases = append(cases, &TestCase{ - Name: suite, - Group: d.Name.String(), - Error: errors.Cause(err), - Scope: s.String(), - Time: time.Since(start).Milliseconds(), + Name: suite, + Group: d.Name.String(), + Error: errors.Cause(err), + Scope: s.String(), + Time: time.Since(start).Milliseconds(), + Coverage: i.GetCoverage(), }) if err != nil { t.counter.Fail()