From 12a564559f0e03d609f9c9c962e46a7d1642455e Mon Sep 17 00:00:00 2001 From: Pranav Gaikwad Date: Thu, 9 Jan 2025 15:49:18 -0500 Subject: [PATCH] first pass at adding exclude support for builtin provider Signed-off-by: Pranav Gaikwad --- engine/conditions.go | 5 +- engine/scopes.go | 26 +++++++ provider/internal/builtin/service_client.go | 4 + provider/provider.go | 40 ++++++++-- provider/provider_test.go | 81 +++++++++++++++++++++ 5 files changed, 146 insertions(+), 10 deletions(-) diff --git a/engine/conditions.go b/engine/conditions.go index e09130c0..da796be0 100644 --- a/engine/conditions.go +++ b/engine/conditions.go @@ -311,6 +311,7 @@ func gatherChain(start ConditionEntry, entries []ConditionEntry) []ConditionEntr // Chain Templates are used by rules and providers to pass context around during rule execution. type ChainTemplate struct { - Filepaths []string `yaml:"filepaths"` - Extras map[string]interface{} `yaml:"extras"` + Filepaths []string `yaml:"filepaths,omitempty"` + Extras map[string]interface{} `yaml:"extras,omitempty"` + ExcludedPaths []string `yaml:"excludedPaths,omitempty"` } diff --git a/engine/scopes.go b/engine/scopes.go index 108ee286..25fe70ce 100644 --- a/engine/scopes.go +++ b/engine/scopes.go @@ -86,3 +86,29 @@ func IncludedPathsScope(paths []string, log logr.Logger) Scope { log: log, } } + +type excludedPathsScope struct { + paths []string +} + +var _ Scope = &excludedPathsScope{} + +func (e *excludedPathsScope) Name() string { + return "ExcludedPathsScope" +} + +func (e *excludedPathsScope) AddToContext(conditionCtx *ConditionContext) error { + templ := ChainTemplate{} + if existingTempl, ok := conditionCtx.Template[TemplateContextPathScopeKey]; ok { + templ = existingTempl + } + templ.ExcludedPaths = e.paths + conditionCtx.Template[TemplateContextPathScopeKey] = templ + return nil +} + +func ExcludedPathsScope(paths []string) Scope { + return &excludedPathsScope{ + paths: paths, + } +} diff --git a/provider/internal/builtin/service_client.go b/provider/internal/builtin/service_client.go index dce1a631..a222bcad 100644 --- a/provider/internal/builtin/service_client.go +++ b/provider/internal/builtin/service_client.go @@ -80,6 +80,7 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if err != nil { return response, fmt.Errorf("unable to find files using pattern `%s`: %v", c.Pattern, err) } + _, matchingFiles = cond.ProviderContext.GetScopedFilepaths(matchingFiles...) } response.TemplateContext = map[string]interface{}{"filepaths": matchingFiles} @@ -201,6 +202,7 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if err != nil { return response, fmt.Errorf("unable to find XML files: %v", err) } + _, xmlFiles = cond.ProviderContext.GetScopedFilepaths(xmlFiles...) for _, file := range xmlFiles { nodes, err := queryXMLFile(file, query) if err != nil { @@ -279,6 +281,7 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if err != nil { return response, fmt.Errorf("unable to find XML files: %v", err) } + _, xmlFiles = cond.ProviderContext.GetScopedFilepaths(xmlFiles...) for _, file := range xmlFiles { nodes, err := queryXMLFile(file, query) if err != nil { @@ -331,6 +334,7 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if err != nil { return response, fmt.Errorf("unable to find files using pattern `%s`: %v", pattern, err) } + _, jsonFiles = cond.ProviderContext.GetScopedFilepaths(jsonFiles...) for _, file := range jsonFiles { f, err := os.Open(file) if err != nil { diff --git a/provider/provider.go b/provider/provider.go index 81d9576e..65d4c40c 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -330,16 +330,40 @@ type ExternalLinks struct { type ProviderContext struct { Tags map[string]interface{} `yaml:"tags"` Template map[string]engine.ChainTemplate `yaml:"template"` - RuleID string `yaml:ruleID` -} - -func (p *ProviderContext) GetScopedFilepaths() (bool, []string) { - for key, value := range p.Template { - if key == engine.TemplateContextPathScopeKey { - return true, value.Filepaths + RuleID string `yaml:"ruleID"` +} + +// GetScopedFilepaths returns a list of filepaths based on either included or excluded paths in context +// when tmpl.Filepaths is set, it is including specific files. we return the value of tmpl.Filepaths as-is +// when tmpl.ExcludedPaths is set, it will exclude the files but we don't know set of all files to exclude from +// as a result, we need an input list of paths to exclude files from. this is upto providers how to pass that list +// when both are set, exclusion happens on union of input paths and included paths. +func (p *ProviderContext) GetScopedFilepaths(paths ...string) (bool, []string) { + if value, ok := p.Template[engine.TemplateContextPathScopeKey]; ok { + includedPaths := []string{} + if (value.Filepaths != nil) && len(value.Filepaths) > 0 { + includedPaths = value.Filepaths + } + includedPaths = append(includedPaths, paths...) + if len(includedPaths) == 0 { + return false, includedPaths + } + filtered := []string{} + for _, path := range includedPaths { + excluded := false + for _, excldPattern := range value.ExcludedPaths { + if pattern, err := regexp.Compile(excldPattern); err == nil && + pattern.MatchString(path) { + excluded = true + } + } + if !excluded { + filtered = append(filtered, path) + } } + return true, filtered } - return false, nil + return false, paths } func HasCapability(caps []Capability, name string) bool { diff --git a/provider/provider_test.go b/provider/provider_test.go index b53e1b8a..3154ae95 100644 --- a/provider/provider_test.go +++ b/provider/provider_test.go @@ -412,3 +412,84 @@ func Test_GetConfigs(t *testing.T) { }) } } + +func TestProviderContext_GetScopedFilepaths(t *testing.T) { + tests := []struct { + name string + template map[string]engine.ChainTemplate + inputPaths []string + want []string + }{ + { + name: "tc-0: only included filepaths present in context, must return that list as-is", + template: map[string]engine.ChainTemplate{ + engine.TemplateContextPathScopeKey: { + Filepaths: []string{"a/", "b/", "c/"}, + }, + }, + inputPaths: []string{}, + want: []string{"a/", "b/", "c/"}, + }, + { + name: "tc-1: included paths present in context, a list of additional paths provided as input, no exclusion, must return union of two lists", + template: map[string]engine.ChainTemplate{ + engine.TemplateContextPathScopeKey: { + Filepaths: []string{"a/", "b/"}, + }, + }, + inputPaths: []string{"c/"}, + want: []string{"a/", "b/", "c/"}, + }, + { + name: "tc-2: included paths present in context, a list of additional paths provided as input, and an exclusion list present, must return correctly filtered list", + template: map[string]engine.ChainTemplate{ + engine.TemplateContextPathScopeKey: { + Filepaths: []string{"a/", "b/c/", "b/c/a.java", "d/e/f.py"}, + ExcludedPaths: []string{ + "b/c/", + }, + }, + }, + inputPaths: []string{"c/p.xml"}, + want: []string{"a/", "d/e/f.py", "c/p.xml"}, + }, + { + name: "tc-3: no included or excluded paths present in context, must return input paths as-is", + template: map[string]engine.ChainTemplate{}, + inputPaths: []string{"a/", "b/", "c/"}, + want: []string{"a/", "b/", "c/"}, + }, + { + name: "tc-4: no included or excluded paths, must return input paths as-is", + template: map[string]engine.ChainTemplate{ + engine.TemplateContextPathScopeKey: { + ExcludedPaths: []string{}, + }, + }, + inputPaths: []string{"a/", "b/", "c/"}, + want: []string{"a/", "b/", "c/"}, + }, + { + name: "tc-5: included and excluded paths given but no input paths, must return correct list of included paths with excluded ones removed", + template: map[string]engine.ChainTemplate{ + engine.TemplateContextPathScopeKey: { + Filepaths: []string{"a/b.py", "c/d/e/f.java", "l/m/n/p.py"}, + ExcludedPaths: []string{".*e.*"}, + }, + }, + inputPaths: []string{}, + want: []string{"a/b.py", "l/m/n/p.py"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &ProviderContext{ + Template: tt.template, + RuleID: "test", + } + if _, got := p.GetScopedFilepaths(tt.inputPaths...); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ProviderContext.FilterExcludedPaths() = %v, want %v", got, tt.want) + } + }) + } +}