checks: first-pass implementation of gobenchdata checks

.gitignore

@@ -5,6 +5,7 @@ gobenchdata
 dist
 gobenchdata-web.json
 gobenchdata-web.yml
+/gobenchdata-checks.yml
 
 # dependencies
 vendor

bench/benchmark.go

@@ -21,6 +21,20 @@ type Run struct {
 	Suites  []Suite
 }
 
+// FindBenchmark returns benchmark by package and bench name
+func (r *Run) FindBenchmark(pkg, bench string) (*Benchmark, bool) {
+	for _, s := range r.Suites {
+		if s.Pkg == pkg {
+			for _, b := range s.Benchmarks {
+				if b.Name == bench {
+					return &b, true
+				}
+			}
+		}
+	}
+	return nil, false
+}
+
 // Suite is a suite of benchmark runs
 type Suite struct {
 	Goos       string

checks/config.go

@@ -1,5 +1,90 @@
 package checks
 
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"regexp"
+
+	"gopkg.in/yaml.v2"
+)
+
+func defaultConfigPath(dir string) string { return path.Join(dir, "gobenchdata-checks.yml") }
+
+// Config declares checks configurations
 type Config struct {
-	// TODO
+	Checks []Check `yaml:"checks"`
+}
+
+// LoadConfig reads configuration from the given directory
+func LoadConfig(dir string) (*Config, error) {
+	b, err := ioutil.ReadFile(defaultConfigPath(dir))
+	if err != nil {
+		return nil, fmt.Errorf("failed to open checks config: %w", err)
+	}
+	var conf Config
+	return &conf, yaml.Unmarshal(b, &conf)
+}
+
+// Check describes a set of benchmarks to run a diff on and check against thresholds
+type Check struct {
+	Name        string `yaml:"name"`
+	Description string `yaml:"description"`
+	Required    bool   `yaml:"required"`
+
+	// regex matchers
+	Package    string   `yaml:"package"`
+	Benchmarks []string `yaml:"benchmarks"`
+
+	// `antonmedv/expr` expressions: https://github.com/antonmedv/expr
+	//
+	// two parameters are provided:
+	// * `base`: bench.Benchmark
+	// * `current`: bench.Benchmark
+	// return a float32 diff in your results, which is then checked against with Thresholds
+	DiffFunc   string     `yaml:"diff"`
+	Thresholds Thresholds `yaml:"thresholds"`
+}
+
+func (c *Check) matchPackage(pkg string) (bool, error) {
+	// treat empty as wildcard
+	if c.Package == "" {
+		return true, nil
+	}
+	// otherwise check for regex
+	m, err := regexp.Compile(c.Package)
+	if err != nil {
+		return false, fmt.Errorf("check %s: invalid package matcher: %w", c.Name, err)
+	}
+	return m.Match([]byte(pkg)), nil
+}
+
+func (c *Check) matchBenchmark(bench string) (bool, error) {
+	// treat empty as wildcard
+	if len(c.Benchmarks) == 0 {
+		return true, nil
+	}
+
+	target := []byte(bench)
+	for _, b := range c.Benchmarks {
+		// treat empty as wildcard
+		if b == "" {
+			return true, nil
+		}
+		// otherwise check for regex
+		m, err := regexp.Compile(b)
+		if err != nil {
+			return false, fmt.Errorf("check %s: invalid benchmark matcher: %w", c.Name, err)
+		}
+		if m.Match(target) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// Thresholds declares values from ChangeFunc to fail
+type Thresholds struct {
+	Min float64 `yaml:"min"`
+	Max float64 `yaml:"max"`
 }

checks/config_test.go

@@ -0,0 +1,82 @@
+package checks
+
+import "testing"
+
+func TestCheck_matchPackage(t *testing.T) {
+	type fields struct {
+		Package string
+	}
+	type args struct {
+		pkg string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    bool
+		wantErr bool
+	}{
+		// test some common intuitive cases
+		{"match on empty", fields{""}, args{"go.bobheadxi.dev/gobenchdata"}, true, false},
+		{"match on exact", fields{"^go.bobheadxi.dev/gobenchdata$"}, args{"go.bobheadxi.dev/gobenchdata"}, true, false},
+		{"fail on exact", fields{"^go.bobheadxi.dev/gobenchdata$"}, args{"go.bobheadxi.dev/gobenchdata/demo"}, false, false},
+		{"match on substring", fields{"go.bobheadxi.dev"}, args{"go.bobheadxi.dev/gobenchdata"}, true, false},
+		{"match on simple regex", fields{"go.bobheadxi.dev/."}, args{"go.bobheadxi.dev/gobenchdata"}, true, false},
+		{"match on excaped", fields{"go.bobheadxi.dev\\/gobenchdata\\/demo"}, args{"go.bobheadxi.dev/gobenchdata/demo"}, true, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &Check{
+				Name:    t.Name(),
+				Package: tt.fields.Package,
+			}
+			got, err := c.matchPackage(tt.args.pkg)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Check.matchPackage() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("Check.matchPackage() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCheck_matchBenchmark(t *testing.T) {
+	type fields struct {
+		Benchmarks []string
+	}
+	type args struct {
+		bench string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    bool
+		wantErr bool
+	}{
+		{"match on nil", fields{nil}, args{"BenchRobert()"}, true, false},
+		{"match on empty", fields{[]string{}}, args{"BenchRobert()"}, true, false},
+		{"match on empty string", fields{[]string{""}}, args{"BenchRobert()"}, true, false},
+		{"match on simple", fields{[]string{"BenchRobert()"}}, args{"BenchRobert()"}, true, false},
+		{"match on exact", fields{[]string{"^BenchRobert\\(\\)$"}}, args{"BenchRobert()"}, true, false},
+		{"fail on exact", fields{[]string{"^BenchRobert\\(\\)$"}}, args{"BenchRobert10()"}, false, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &Check{
+				Name:       t.Name(),
+				Benchmarks: tt.fields.Benchmarks,
+			}
+			got, err := c.matchBenchmark(tt.args.bench)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Check.matchBenchmark() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("Check.matchBenchmark() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

checks/evaluate.go

@@ -0,0 +1,151 @@
+package checks
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/antonmedv/expr"
+	"github.com/antonmedv/expr/vm"
+	"go.bobheadxi.dev/gobenchdata/bench"
+)
+
+// Results reports the output of Evaluate
+type Results struct {
+	Checks map[string]*CheckResult
+
+	Failed bool
+}
+
+// CheckResult reports the output of a Check
+type CheckResult struct {
+	Diffs []DiffResult
+
+	Thresholds Thresholds
+	Required   bool
+	Failed     bool
+}
+
+// DiffResult is the result of a diff
+type DiffResult struct {
+	Package   string
+	Benchmark string
+	Value     float64
+
+	Failed bool
+}
+
+// EnvDiffFunc describes variables provided to a DiffFunc
+type EnvDiffFunc struct {
+	Check *Check
+	prog  *vm.Program
+}
+
+func (e EnvDiffFunc) execute(base, current *bench.Benchmark) (float64, error) {
+	out, err := expr.Run(e.prog, map[string]interface{}{
+		"check":   e.Check,
+		"base":    base,
+		"current": current,
+	})
+	if err != nil {
+		return 0, fmt.Errorf("check '%s': diff function errored: %w", e.Check.Name, err)
+	}
+	switch i := out.(type) {
+	case float64:
+		return i, nil
+	case float32:
+		return float64(i), nil
+	case int:
+		return float64(i), nil
+	default:
+		return 0, fmt.Errorf("check '%s': result '%+v' could not be cast to a float64", e.Check.Name, i)
+	}
+}
+
+// Evaluate checks against benchmark runs
+func Evaluate(checks []Check, base bench.RunHistory, current bench.RunHistory) (*Results, error) {
+	// put most recent at top
+	sort.Sort(base)
+	sort.Sort(current)
+	baseRun := base[base.Len()-1]
+	currentRun := current[current.Len()-1]
+
+	// set up results
+	results := &Results{
+		Checks: map[string]*CheckResult{},
+		Failed: false,
+	}
+	for _, c := range checks {
+		results.Checks[c.Name] = &CheckResult{
+			Diffs:      []DiffResult{},
+			Thresholds: c.Thresholds,
+			Required:   c.Required,
+			Failed:     false,
+		}
+	}
+
+	// evaluate all checks
+	for _, suite := range currentRun.Suites {
+		// find checks to run on this suite
+		execChecks := []*EnvDiffFunc{}
+		for _, check := range checks {
+			if ok, err := check.matchPackage(suite.Pkg); err != nil {
+				return nil, err
+			} else if ok {
+				prog, err := expr.Compile(check.DiffFunc)
+				if err != nil {
+					return nil, fmt.Errorf("check '%s': invalid diff function provided: %w", check.Name, err)
+				}
+				execChecks = append(execChecks, &EnvDiffFunc{
+					Check: &check,
+					prog:  prog,
+				})
+			}
+		}
+
+		// skip this suite if there are no checks
+		if len(execChecks) == 0 {
+			continue
+		}
+
+		// find matching benchmarks
+		for _, bench := range suite.Benchmarks {
+			// find corresponding base benchmark
+			baseBench, ok := baseRun.FindBenchmark(suite.Pkg, bench.Name)
+			if !ok {
+				// TODO: should this fail?
+				fmt.Printf("warn: could not find benchmark '%s.%s' in most recent 'base' run", suite.Pkg, bench.Name)
+				continue
+			}
+
+			// run all matching checks
+			for _, env := range execChecks {
+				if match, err := env.Check.matchBenchmark(bench.Name); err != nil {
+					return nil, err
+				} else if match {
+					res, err := env.execute(baseBench, &bench)
+					if err != nil {
+						return nil, err
+					}
+
+					// update result
+					checkRes := results.Checks[env.Check.Name]
+					failed := res < checkRes.Thresholds.Min || res > checkRes.Thresholds.Max
+					if failed {
+						checkRes.Failed = true
+						if checkRes.Required {
+							results.Failed = true
+						}
+					}
+					checkRes.Diffs = append(checkRes.Diffs, DiffResult{
+						Package:   suite.Pkg,
+						Benchmark: bench.Name,
+						Value:     res,
+						Failed:    failed,
+					})
+				}
+			}
+		}
+	}
+
+	return results, nil
+}