From 0051d292561e7e7274e7300c7c21333e0057989a Mon Sep 17 00:00:00 2001 From: Doug Simmons Date: Wed, 7 Oct 2020 18:34:56 -0700 Subject: [PATCH 1/2] Add consider file support Adds support for a per repo .drone.yml inventory file called the "consider file" which, when specified, will influence how drone-tree-config finds the .drone.yml files for the target repo. Unrelated change: Fix markdown-lint errors in README.md --- README.md | 82 +++++++++--- cmd/drone-tree-config/main.go | 2 + plugin/configloader.go | 116 +++++++++++++++++ plugin/options.go | 8 ++ plugin/plugin.go | 43 +++++-- plugin/plugin_test.go | 135 ++++++++++++++++++++ plugin/testdata/github/.drone-consider.json | 9 ++ 7 files changed, 365 insertions(+), 30 deletions(-) create mode 100644 plugin/testdata/github/.drone-consider.json diff --git a/README.md b/README.md index a4892de..be14152 100644 --- a/README.md +++ b/README.md @@ -16,31 +16,32 @@ Currently supports ## Usage -#### Environment variables: +#### Environment variables -- `PLUGIN_CONCAT`: Concats all found configs to a multi-machine build. Defaults to `false`. -- `PLUGIN_FALLBACK`: Rebuild all .drone.yml if no changes where made. Defaults to `false`. -- `PLUGIN_MAXDEPTH`: Max depth to search for `drone.yml`, only active in fallback mode. Defaults to `2` (would still find `/a/b/.drone.yml`). -- `PLUGIN_DEBUG`: Set this to `true` to enable debug messages. -- `PLUGIN_ADDRESS`: Listen address for the plugins webserver. Defaults to `:3000`. -- `PLUGIN_SECRET`: Shared secret with drone. You can generate the token using `openssl rand -hex 16`. -- `PLUGIN_ALLOW_LIST_FILE`: (Optional) Path to regex pattern file. Matches the repo slug(s) against a list of regex patterns. Defaults to `""`, match everything. +* `PLUGIN_CONCAT`: Concats all found configs to a multi-machine build. Defaults to `false`. +* `PLUGIN_FALLBACK`: Rebuild all .drone.yml if no changes where made. Defaults to `false`. +* `PLUGIN_MAXDEPTH`: Max depth to search for `.drone.yml`, only active in fallback mode. Defaults to `2` (would still find `/a/b/.drone.yml`). +* `PLUGIN_DEBUG`: Set this to `true` to enable debug messages. +* `PLUGIN_ADDRESS`: Listen address for the plugins webserver. Defaults to `:3000`. +* `PLUGIN_SECRET`: Shared secret with drone. You can generate the token using `openssl rand -hex 16`. +* `PLUGIN_ALLOW_LIST_FILE`: (Optional) Path to regex pattern file. Matches the repo slug(s) against a list of regex patterns. Defaults to `""`, match everything. +* `PLUGIN_CONSIDER_FILE`: (Optional) Consider file name. Only consider the `.drone.yml` files listed in this file. When defined, all enabled repos must contain a consider file. Backend specific options -- `SERVER`: Custom SCM server (also used by Gitlab / Bitbucket) -- GitHub: - - `GITHUB_TOKEN`: Github personal access token. Only needs repo rights. See [here][1]. -- GitLab: - - `GITLAB_TOKEN`: Gitlab personal access token. Only needs `read_repository` rights. See [here][2] -- Bitbucket - - `BITBUCKET_AUTH_SERVER`: Custom auth server (uses SERVER if empty) - - `BITBUCKET_CLIENT`: Credentials for Bitbucket access - - `BITBUCKET_SECRET`: Credentials for Bitbucket access +* `SERVER`: Custom SCM server (also used by Gitlab / Bitbucket) +* GitHub: + * `GITHUB_TOKEN`: Github personal access token. Only needs repo rights. See [here][1]. +* GitLab: + * `GITLAB_TOKEN`: Gitlab personal access token. Only needs `read_repository` rights. See [here][2] +* Bitbucket + * `BITBUCKET_AUTH_SERVER`: Custom auth server (uses SERVER if empty) + * `BITBUCKET_CLIENT`: Credentials for Bitbucket access + * `BITBUCKET_SECRET`: Credentials for Bitbucket access If `PLUGIN_CONCAT` is not set, the first found `.drone.yml` will be used. -#### Example docker-compose: +#### Example docker-compose ```yaml version: '2' @@ -83,7 +84,7 @@ services: Edit the Secrets (`***`), `` and `` to your needs. `` is used between Drone and drone-tree-config. -#### Enable repos via regex matching: +#### Enable repos via regex matching By default, this plugin matches against ALL repo slugs. If you want to enable the plugin for specific repos only, turn on regex matching by specifying a `PLUGIN_ALLOW_LIST_FILE`. @@ -117,9 +118,50 @@ File: drone-tree-config-matchfile: ^myorg/myrepo$ ``` -* Matches against all repos in the `bitbeats` org +* Matches against all repos in the `bitbeats` org * Matches against `myorg/myrepo` [1]: https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line [2]: https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html [3]: https://github.com/google/re2/wiki/Syntax + +#### Consider file + + If a `PLUGIN_CONSIDER_FILE` is defined, drone-tree-config will first read the content of the target file and will only consider + the `.drone.yml` files specified, when matching. + +Depending on the size and the complexity of the repository, using a "consider file" can +significantly reduce the number of API calls made to the provider (github, bitbucket, other). The reduction in API calls +reduces the risk of being rate limited and can result in less processing time for drone-tree-config. + +Given the config; + +```yaml + - PLUGIN_CONSIDER_FILE=.drone-consider +``` + +A local git repo clone; + +```shell +$ tree -a my-repo-clone/ + my-repo-clone/ + ├── .drone-consier + ├── foo + │ └── .drone.yml + ├── bar + │   └── .drone.yml + └── baz + +``` + +Content of the .drone-consider to check in; + +```shell +$ cat my-repo-clone/.drone-consider +foo/.drone.yml +bar/.drone.yml +``` + +The downside of a "consider file" is that it has to be kept in sync. As a suggestion, to help with this, a step can be +added to each `.drone.yml` which verifies the "consider file" is in sync with the actual content of the repo. For +example, this can be accomplished by comparing the output of `find ./ -name .drone.yml` with the content of the "consider file". diff --git a/cmd/drone-tree-config/main.go b/cmd/drone-tree-config/main.go index 06d8bd1..9216644 100644 --- a/cmd/drone-tree-config/main.go +++ b/cmd/drone-tree-config/main.go @@ -26,6 +26,7 @@ type ( BitBucketAuthServer string `envconfig:"BITBUCKET_AUTH_SERVER"` BitBucketClient string `envconfig:"BITBUCKET_CLIENT"` BitBucketSecret string `envconfig:"BITBUCKET_SECRET"` + ConsiderFile string `envconfig:"PLUGIN_CONSIDER_FILE"` // Deprecated: Use AllowListFile instead. WhitelistFile string `envconfig:"PLUGIN_WHITELIST_FILE"` } @@ -70,6 +71,7 @@ func main() { plugin.WithGithubToken(spec.GitHubToken), plugin.WithGitlabToken(spec.GitLabToken), plugin.WithGitlabServer(spec.GitLabServer), + plugin.WithConsiderFile(spec.ConsiderFile), ), spec.Secret, logrus.StandardLogger(), diff --git a/plugin/configloader.go b/plugin/configloader.go index f120e13..d7b3a99 100644 --- a/plugin/configloader.go +++ b/plugin/configloader.go @@ -3,6 +3,7 @@ package plugin import ( "context" "path" + "strings" "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -47,6 +48,85 @@ func (p *Plugin) getConfigForChanges(ctx context.Context, req *request, changedF return configData, nil } +// getConsiderFile returns the 'drone.yml' entries in a consider file as a string slice +func (p *Plugin) getConsiderFile(ctx context.Context, req *request) ([]string, error) { + toReturn := make([]string, 0) + + // download considerFile from github + fc, err := p.getScmFile(ctx, req, p.considerFile) + if err != nil { + logrus.Errorf("%s skipping: %s is not present: %v", req.UUID, p.considerFile, err) + return toReturn, err + } + + // collect drone.yml files + for _, v := range strings.Split(fc, "\n") { + // skip empty lines and comments + if strings.TrimSpace(v) == "" || strings.HasPrefix(v, "#") { + continue + } + // skip lines which do not contain a 'drone.yml' reference + if !strings.HasSuffix(v, req.Repo.Config) { + logrus.Warnf("%s skipping invalid reference to %s in %s", req.UUID, v, p.considerFile) + continue + } + toReturn = append(toReturn, v) + } + + return toReturn, nil +} + +// getConfigForChangesUsingConsider loads 'drone.yml' from the consider file based on the changed files. +// Note: this call does not fail if there are invalid entries in a consider file +func (p *Plugin) getConfigForChangesUsingConsider(ctx context.Context, req *request, changedFiles []string) (string, error) { + configData := "" + consider := map[string]bool{} + cache := map[string]bool{} + + considerEntries, err := p.getConsiderFile(ctx, req) + if err != nil { + return "", err + } + // convert to a map for O(1) lookup + for _, v := range considerEntries { + consider[v] = true + } + + for _, file := range changedFiles { + dir := file + for dir != "." { + dir = path.Join(dir, "..") + file := path.Join(dir, req.Repo.Config) + + // check if file has already been checked + if _, ok := cache[file]; ok { + continue + } + cache[file] = true + + // look for file in consider map + if _, exists := consider[file]; exists { + // download file from git + fileContent, critical, err := p.getDroneConfig(ctx, req, file) + if err != nil { + if critical { + return "", err + } + continue + } + + // append + configData = p.droneConfigAppend(configData, fileContent) + if !p.concat { + logrus.Infof("%s concat is disabled. Using just first .drone.yml.", req.UUID) + break + } + } + } + } + return configData, nil +} + // getConfigForTree searches for all or first 'drone.yml' in the repo func (p *Plugin) getConfigForTree(ctx context.Context, req *request, dir string, depth int) (configData string, err error) { ls, err := req.Client.GetFileListing(ctx, dir, req.Build.After) @@ -87,6 +167,42 @@ func (p *Plugin) getConfigForTree(ctx context.Context, req *request, dir string, return configData, nil } +// getConfigForTreeUsingConsider loads all 'drone.yml' which are identified in the consider file. +func (p *Plugin) getConfigForTreeUsingConsider(ctx context.Context, req *request) (string, error) { + configData := "" + cache := map[string]bool{} + + consider, err := p.getConsiderFile(ctx, req) + if err != nil { + return "", err + } + + // collect drone.yml files + for _, v := range consider { + if _, ok := cache[v]; ok { + continue + } + cache[v] = true + + // download file from github + fc, critical, err := p.getDroneConfig(ctx, req, v) + if err != nil { + if critical { + return "", err + } + continue + } + + // append + configData = p.droneConfigAppend(configData, fc) + if !p.concat { + logrus.Infof("%s concat is disabled. Using just first .drone.yml.", req.UUID) + break + } + } + return configData, nil +} + // getDroneConfig downloads a drone config and validates it func (p *Plugin) getDroneConfig(ctx context.Context, req *request, file string) (configData string, critical bool, err error) { fileContent, err := p.getScmFile(ctx, req, file) diff --git a/plugin/options.go b/plugin/options.go index 92f43ab..09f2234 100644 --- a/plugin/options.go +++ b/plugin/options.go @@ -82,3 +82,11 @@ func WithAllowListFile(file string) func(*Plugin) { p.allowListFile = file } } + +// WithConsiderFile configures with a consider file which contains references to all 'drone.yml' files which should +// be considered for the repository. +func WithConsiderFile(considerFile string) func(*Plugin) { + return func(p *Plugin) { + p.considerFile = considerFile + } +} diff --git a/plugin/plugin.go b/plugin/plugin.go index df58abc..11cd3c2 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -28,6 +28,7 @@ type ( fallback bool maxDepth int allowListFile string + considerFile string } droneConfig struct { @@ -72,37 +73,59 @@ func (p *Plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone return nil, nil } - // get changed files - changedFiles, err := p.getScmChanges(ctx, &req) + configData, err := p.getConfig(ctx, &req) if err != nil { return nil, err } + return &drone.Config{Data: configData}, nil +} + +// getConfig retrieves drone config data from the repo +func (p *Plugin) getConfig(ctx context.Context, req *request) (string, error) { + // get changed files + changedFiles, err := p.getScmChanges(ctx, req) + if err != nil { + return "", err + } // get drone.yml for changed files or all of them if no changes/cron configData := "" if changedFiles != nil { - configData, err = p.getConfigForChanges(ctx, &req, changedFiles) + if p.considerFile != "" { + configData, err = p.getConfigForChangesUsingConsider(ctx, req, changedFiles) + } else { + configData, err = p.getConfigForChanges(ctx, req, changedFiles) + } } else if req.Build.Trigger == "@cron" { logrus.Warnf("%s @cron, rebuilding all", req.UUID) - configData, err = p.getConfigForTree(ctx, &req, "", 0) + if p.considerFile != "" { + configData, err = p.getConfigForTreeUsingConsider(ctx, req) + } else { + logrus.Warnf("recursively scanning for config files with max depth %d", p.maxDepth) + configData, err = p.getConfigForTree(ctx, req, "", 0) + } } else if p.fallback { logrus.Warnf("%s no changed files and fallback enabled, rebuilding all", req.UUID) - configData, err = p.getConfigForTree(ctx, &req, "", 0) + if p.considerFile != "" { + configData, err = p.getConfigForTreeUsingConsider(ctx, req) + } else { + logrus.Warnf("recursively scanning for config files with max depth %d", p.maxDepth) + configData, err = p.getConfigForTree(ctx, req, "", 0) + } } if err != nil { - return nil, err + return "", err } // no file found if configData == "" { - return nil, errors.New("did not find a .drone.yml") + return "", errors.New("did not find a .drone.yml") } // cleanup - configData = string(removeDocEndRegex.ReplaceAllString(configData, "")) + configData = removeDocEndRegex.ReplaceAllString(configData, "") configData = string(dedupRegex.ReplaceAll([]byte(configData), []byte("---"))) - - return &drone.Config{Data: configData}, nil + return configData, nil } var dedupRegex = regexp.MustCompile(`(?ms)(---[\s]*){2,}`) diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 8928704..22099b7 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -59,6 +59,39 @@ func TestPlugin(t *testing.T) { } } +func TestPluginWithConsider(t *testing.T) { + req := &config.Request{ + Build: drone.Build{ + Before: "2897b31ec3a1b59279a08a8ad54dc360686327f7", + After: "8ecad91991d5da985a2a8dd97cc19029dc1c2899", + Source: "master", + }, + Repo: drone.Repo{ + Namespace: "foosinn", + Name: "dronetest", + Branch: "master", + Slug: "foosinn/dronetest", + Config: ".drone.yml", + }, + } + plugin := New( + WithServer(ts.URL), + WithGithubToken(mockToken), + WithFallback(true), + WithMaxDepth(2), + WithConsiderFile(".drone-consider"), + ) + droneConfig, err := plugin.Find(noContext, req) + if err != nil { + t.Error(err) + return + } + + if want, got := "---\nkind: pipeline\nname: default\n\nsteps:\n- name: build\n image: golang\n commands:\n - go build\n - go test -short\n\n- name: integration\n image: golang\n commands:\n - go test -v\n", droneConfig.Data; want != got { + t.Errorf("Want %q got %q", want, got) + } +} + func TestConcat(t *testing.T) { req := &config.Request{ Build: drone.Build{ @@ -92,6 +125,40 @@ func TestConcat(t *testing.T) { } } +func TestConcatWithConsider(t *testing.T) { + req := &config.Request{ + Build: drone.Build{ + Before: "2897b31ec3a1b59279a08a8ad54dc360686327f7", + After: "8ecad91991d5da985a2a8dd97cc19029dc1c2899", + Source: "master", + }, + Repo: drone.Repo{ + Namespace: "foosinn", + Name: "dronetest", + Branch: "master", + Slug: "foosinn/dronetest", + Config: ".drone.yml", + }, + } + plugin := New( + WithServer(ts.URL), + WithGithubToken(mockToken), + WithConcat(true), + WithFallback(true), + WithMaxDepth(2), + WithConsiderFile(".drone-consider"), + ) + droneConfig, err := plugin.Find(noContext, req) + if err != nil { + t.Error(err) + return + } + + if want, got := "---\nkind: pipeline\nname: default\n\nsteps:\n- name: build\n image: golang\n commands:\n - go build\n - go test -short\n\n- name: integration\n image: golang\n commands:\n - go test -v\n---\nkind: pipeline\nname: default\n\nsteps:\n- name: frontend\n image: node\n commands:\n - npm install\n - npm test\n\n- name: backend\n image: golang\n commands:\n - go build\n - go test\n", droneConfig.Data; want != got { + t.Errorf("Want %q got %q", want, got) + } +} + func TestPullRequest(t *testing.T) { req := &config.Request{ Build: drone.Build{ @@ -123,6 +190,38 @@ func TestPullRequest(t *testing.T) { } } +func TestPullRequestWithConsider(t *testing.T) { + req := &config.Request{ + Build: drone.Build{ + Fork: "octocat/dronetest", + Ref: "refs/pull/3/head", + }, + Repo: drone.Repo{ + Namespace: "foosinn", + Name: "dronetest", + Slug: "foosinn/dronetest", + Config: ".drone.yml", + }, + } + plugin := New( + WithServer(ts.URL), + WithGithubToken(mockToken), + WithConcat(true), + WithFallback(true), + WithMaxDepth(2), + WithConsiderFile(".drone-consider"), + ) + droneConfig, err := plugin.Find(noContext, req) + if err != nil { + t.Error(err) + return + } + + if want, got := "---\nkind: pipeline\nname: default\n\nsteps:\n- name: frontend\n image: node\n commands:\n - npm install\n - npm test\n\n- name: backend\n image: golang\n commands:\n - go build\n - go test\n", droneConfig.Data; want != got { + t.Errorf("Want %q got %q", want, got) + } +} + func TestCron(t *testing.T) { req := &config.Request{ Build: drone.Build{ @@ -229,6 +328,37 @@ func TestMatchEnable(t *testing.T) { } } +func TestCronWithConsider(t *testing.T) { + req := &config.Request{ + Build: drone.Build{ + After: "8ecad91991d5da985a2a8dd97cc19029dc1c2899", + Trigger: "@cron", + }, + Repo: drone.Repo{ + Namespace: "foosinn", + Name: "dronetest", + Slug: "foosinn/dronetest", + Config: ".drone.yml", + }, + } + plugin := New( + WithServer(ts.URL), + WithGithubToken(mockToken), + WithConcat(true), + WithFallback(true), + WithMaxDepth(2), + WithConsiderFile(".drone-consider"), + ) + droneConfig, err := plugin.Find(noContext, req) + if err != nil { + t.Error(err) + return + } + + if want, got := "---\nkind: pipeline\nname: default\n\nsteps:\n- name: frontend\n image: node\n commands:\n - npm install\n - npm test\n\n- name: backend\n image: golang\n commands:\n - go build\n - go test\n---\nkind: pipeline\nname: default\n\nsteps:\n- name: build\n image: golang\n commands:\n - go build\n - go test -short\n\n- name: integration\n image: golang\n commands:\n - go test -v\n", droneConfig.Data; want != got { + t.Errorf("Want\n %q\ngot\n %q", want, got) + } +} func TestCronConcat(t *testing.T) { req := &config.Request{ Build: drone.Build{ @@ -282,6 +412,11 @@ func testMux() *http.ServeMux { f, _ := os.Open("testdata/github/.drone.yml.json") _, _ = io.Copy(w, f) }) + mux.HandleFunc("/repos/foosinn/dronetest/contents/.drone-consider", + func(w http.ResponseWriter, r *http.Request) { + f, _ := os.Open("testdata/github/.drone-consider.json") + _, _ = io.Copy(w, f) + }) mux.HandleFunc("/repos/foosinn/dronetest/pulls/3/files", func(w http.ResponseWriter, r *http.Request) { f, _ := os.Open("testdata/github/pull_3_files.json") diff --git a/plugin/testdata/github/.drone-consider.json b/plugin/testdata/github/.drone-consider.json new file mode 100644 index 0000000..4144c9b --- /dev/null +++ b/plugin/testdata/github/.drone-consider.json @@ -0,0 +1,9 @@ +{ + "name": ".drone-consider", + "path": ".drone-consider", + "sha": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "size": 73, + "type": "file", + "content": "LmRyb25lLnltbAphL2IvLmRyb25lLnltbAppbnZhbGlkL3BhdGgvbXktZHJvbmUueW1sCgo=", + "encoding": "base64" +} From 06486f61c8fb7ce4187ef37c2b0205d08d2d5c8d Mon Sep 17 00:00:00 2001 From: Doug Simmons Date: Sun, 11 Oct 2020 19:20:25 -0700 Subject: [PATCH 2/2] Update considerFile implementation due to feedback --- plugin/configloader.go | 126 ++++------------------------------------- plugin/consider.go | 58 +++++++++++++++++++ plugin/plugin.go | 34 +++++------ 3 files changed, 86 insertions(+), 132 deletions(-) create mode 100644 plugin/consider.go diff --git a/plugin/configloader.go b/plugin/configloader.go index d7b3a99..61711cd 100644 --- a/plugin/configloader.go +++ b/plugin/configloader.go @@ -3,7 +3,6 @@ package plugin import ( "context" "path" - "strings" "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -28,6 +27,11 @@ func (p *Plugin) getConfigForChanges(ctx context.Context, req *request, changedF cache[file] = true } + // when enabled, only process drone.yml from p.considerFile + if p.considerFile != "" && !req.ConsiderData.consider(file) { + continue + } + // download file from git fileContent, critical, err := p.getDroneConfig(ctx, req, file) if err != nil { @@ -48,87 +52,13 @@ func (p *Plugin) getConfigForChanges(ctx context.Context, req *request, changedF return configData, nil } -// getConsiderFile returns the 'drone.yml' entries in a consider file as a string slice -func (p *Plugin) getConsiderFile(ctx context.Context, req *request) ([]string, error) { - toReturn := make([]string, 0) - - // download considerFile from github - fc, err := p.getScmFile(ctx, req, p.considerFile) - if err != nil { - logrus.Errorf("%s skipping: %s is not present: %v", req.UUID, p.considerFile, err) - return toReturn, err - } - - // collect drone.yml files - for _, v := range strings.Split(fc, "\n") { - // skip empty lines and comments - if strings.TrimSpace(v) == "" || strings.HasPrefix(v, "#") { - continue - } - // skip lines which do not contain a 'drone.yml' reference - if !strings.HasSuffix(v, req.Repo.Config) { - logrus.Warnf("%s skipping invalid reference to %s in %s", req.UUID, v, p.considerFile) - continue - } - toReturn = append(toReturn, v) - } - - return toReturn, nil -} - -// getConfigForChangesUsingConsider loads 'drone.yml' from the consider file based on the changed files. -// Note: this call does not fail if there are invalid entries in a consider file -func (p *Plugin) getConfigForChangesUsingConsider(ctx context.Context, req *request, changedFiles []string) (string, error) { - configData := "" - consider := map[string]bool{} - cache := map[string]bool{} - - considerEntries, err := p.getConsiderFile(ctx, req) - if err != nil { - return "", err - } - // convert to a map for O(1) lookup - for _, v := range considerEntries { - consider[v] = true - } - - for _, file := range changedFiles { - dir := file - for dir != "." { - dir = path.Join(dir, "..") - file := path.Join(dir, req.Repo.Config) - - // check if file has already been checked - if _, ok := cache[file]; ok { - continue - } - cache[file] = true - - // look for file in consider map - if _, exists := consider[file]; exists { - // download file from git - fileContent, critical, err := p.getDroneConfig(ctx, req, file) - if err != nil { - if critical { - return "", err - } - continue - } - - // append - configData = p.droneConfigAppend(configData, fileContent) - if !p.concat { - logrus.Infof("%s concat is disabled. Using just first .drone.yml.", req.UUID) - break - } - } - } - } - return configData, nil -} - // getConfigForTree searches for all or first 'drone.yml' in the repo func (p *Plugin) getConfigForTree(ctx context.Context, req *request, dir string, depth int) (configData string, err error) { + if p.considerFile != "" { + // treats all 'drone.yml' entries in the consider file as the changedFiles + return p.getConfigForChanges(ctx, req, req.ConsiderData.listRepresentation) + } + ls, err := req.Client.GetFileListing(ctx, dir, req.Build.After) if err != nil { return "", err @@ -167,42 +97,6 @@ func (p *Plugin) getConfigForTree(ctx context.Context, req *request, dir string, return configData, nil } -// getConfigForTreeUsingConsider loads all 'drone.yml' which are identified in the consider file. -func (p *Plugin) getConfigForTreeUsingConsider(ctx context.Context, req *request) (string, error) { - configData := "" - cache := map[string]bool{} - - consider, err := p.getConsiderFile(ctx, req) - if err != nil { - return "", err - } - - // collect drone.yml files - for _, v := range consider { - if _, ok := cache[v]; ok { - continue - } - cache[v] = true - - // download file from github - fc, critical, err := p.getDroneConfig(ctx, req, v) - if err != nil { - if critical { - return "", err - } - continue - } - - // append - configData = p.droneConfigAppend(configData, fc) - if !p.concat { - logrus.Infof("%s concat is disabled. Using just first .drone.yml.", req.UUID) - break - } - } - return configData, nil -} - // getDroneConfig downloads a drone config and validates it func (p *Plugin) getDroneConfig(ctx context.Context, req *request, file string) (configData string, critical bool, err error) { fileContent, err := p.getScmFile(ctx, req, file) diff --git a/plugin/consider.go b/plugin/consider.go new file mode 100644 index 0000000..187cefd --- /dev/null +++ b/plugin/consider.go @@ -0,0 +1,58 @@ +package plugin + +import ( + "context" + "strings" + + "github.com/sirupsen/logrus" +) + +// ConsiderData holds the considerFile information in both list and map representations +type ConsiderData struct { + mapRepresentation map[string]bool + listRepresentation []string +} + +// consider returns true if the path provided matches an entry in the considerFile +func (c *ConsiderData) consider(path string) bool { + _, exists := c.mapRepresentation[path] + return exists +} + +// ------ + +// newConsiderDataFromRequest returns the ConsiderData which is loaded from the considerFile +func (p *Plugin) newConsiderDataFromRequest(ctx context.Context, req *request) (*ConsiderData, error) { + cd := new(ConsiderData) + cd.mapRepresentation = make(map[string]bool) + cd.listRepresentation = make([]string, 0) + + // bail early without calling the scm provider when there is no considerFile configured + if p.considerFile == "" { + return cd, nil + } + + // download considerFile from github + fc, err := p.getScmFile(ctx, req, p.considerFile) + if err != nil { + logrus.Errorf("%s skipping: %s is not present: %v", req.UUID, p.considerFile, err) + return cd, err + } + + // collect drone.yml files + for _, v := range strings.Split(fc, "\n") { + // skip empty lines and comments + if strings.TrimSpace(v) == "" || strings.HasPrefix(v, "#") { + continue + } + // skip lines which do not contain a 'drone.yml' reference + if !strings.HasSuffix(v, req.Repo.Config) { + logrus.Warnf("%s skipping invalid reference to %s in %s", req.UUID, v, p.considerFile) + continue + } + cd.listRepresentation = append(cd.listRepresentation, v) + cd.mapRepresentation[v] = true + } + + return cd, nil +} diff --git a/plugin/plugin.go b/plugin/plugin.go index 11cd3c2..e6dd38f 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -38,8 +38,9 @@ type ( request struct { *config.Request - UUID uuid.UUID - Client scm_clients.ScmClient + UUID uuid.UUID + Client scm_clients.ScmClient + ConsiderData *ConsiderData } ) @@ -65,7 +66,11 @@ func (p *Plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone return nil, err } - req := request{droneRequest, someUuid, client} + req := request{ + Request: droneRequest, + UUID: someUuid, + Client: client, + } // make sure this plugin is enabled for the requested repo slug if ok := p.allowlisted(&req); !ok { @@ -73,6 +78,11 @@ func (p *Plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone return nil, nil } + // load the considerFile entries, if configured for considerFile + if req.ConsiderData, err = p.newConsiderDataFromRequest(ctx, &req); err != nil { + return nil, err + } + configData, err := p.getConfig(ctx, &req) if err != nil { return nil, err @@ -91,27 +101,19 @@ func (p *Plugin) getConfig(ctx context.Context, req *request) (string, error) { // get drone.yml for changed files or all of them if no changes/cron configData := "" if changedFiles != nil { - if p.considerFile != "" { - configData, err = p.getConfigForChangesUsingConsider(ctx, req, changedFiles) - } else { - configData, err = p.getConfigForChanges(ctx, req, changedFiles) - } + configData, err = p.getConfigForChanges(ctx, req, changedFiles) } else if req.Build.Trigger == "@cron" { logrus.Warnf("%s @cron, rebuilding all", req.UUID) - if p.considerFile != "" { - configData, err = p.getConfigForTreeUsingConsider(ctx, req) - } else { + if p.considerFile == "" { logrus.Warnf("recursively scanning for config files with max depth %d", p.maxDepth) - configData, err = p.getConfigForTree(ctx, req, "", 0) } + configData, err = p.getConfigForTree(ctx, req, "", 0) } else if p.fallback { logrus.Warnf("%s no changed files and fallback enabled, rebuilding all", req.UUID) - if p.considerFile != "" { - configData, err = p.getConfigForTreeUsingConsider(ctx, req) - } else { + if p.considerFile == "" { logrus.Warnf("recursively scanning for config files with max depth %d", p.maxDepth) - configData, err = p.getConfigForTree(ctx, req, "", 0) } + configData, err = p.getConfigForTree(ctx, req, "", 0) } if err != nil { return "", err