Merge pull request #32 from mach6/mach6_consider_file

Add consider file support

README.md

@@ -16,31 +16,32 @@ Currently supports
 
 ## Usage
 
-#### Environment variables:
+#### Environment variables
 
-- `PLUGIN_CONCAT`: Concats all found configs to a multi-machine build. Defaults to `false`.
-- `PLUGIN_FALLBACK`: Rebuild all .drone.yml if no changes where made. Defaults to `false`.
-- `PLUGIN_MAXDEPTH`: Max depth to search for `drone.yml`, only active in fallback mode. Defaults to `2` (would still find `/a/b/.drone.yml`).
-- `PLUGIN_DEBUG`: Set this to `true` to enable debug messages.
-- `PLUGIN_ADDRESS`: Listen address for the plugins webserver. Defaults to `:3000`.
-- `PLUGIN_SECRET`: Shared secret with drone. You can generate the token using `openssl rand -hex 16`.
-- `PLUGIN_ALLOW_LIST_FILE`: (Optional) Path to regex pattern file. Matches the repo slug(s) against a list of regex patterns. Defaults to `""`, match everything.
+* `PLUGIN_CONCAT`: Concats all found configs to a multi-machine build. Defaults to `false`.
+* `PLUGIN_FALLBACK`: Rebuild all .drone.yml if no changes where made. Defaults to `false`.
+* `PLUGIN_MAXDEPTH`: Max depth to search for `.drone.yml`, only active in fallback mode. Defaults to `2` (would still find `/a/b/.drone.yml`).
+* `PLUGIN_DEBUG`: Set this to `true` to enable debug messages.
+* `PLUGIN_ADDRESS`: Listen address for the plugins webserver. Defaults to `:3000`.
+* `PLUGIN_SECRET`: Shared secret with drone. You can generate the token using `openssl rand -hex 16`.
+* `PLUGIN_ALLOW_LIST_FILE`: (Optional) Path to regex pattern file. Matches the repo slug(s) against a list of regex patterns. Defaults to `""`, match everything.
+* `PLUGIN_CONSIDER_FILE`: (Optional) Consider file name. Only consider the `.drone.yml` files listed in this file. When defined, all enabled repos must contain a consider file.
 
 Backend specific options
 
-- `SERVER`: Custom SCM server (also used by Gitlab / Bitbucket)
-- GitHub:
-  - `GITHUB_TOKEN`: Github personal access token. Only needs repo rights. See [here][1].
-- GitLab:
-  - `GITLAB_TOKEN`: Gitlab personal access token. Only needs `read_repository` rights. See [here][2]
-- Bitbucket
-  - `BITBUCKET_AUTH_SERVER`: Custom auth server (uses SERVER if empty)
-  - `BITBUCKET_CLIENT`: Credentials for Bitbucket access
-  - `BITBUCKET_SECRET`: Credentials for Bitbucket access
+* `SERVER`: Custom SCM server (also used by Gitlab / Bitbucket)
+* GitHub:
+  * `GITHUB_TOKEN`: Github personal access token. Only needs repo rights. See [here][1].
+* GitLab:
+  * `GITLAB_TOKEN`: Gitlab personal access token. Only needs `read_repository` rights. See [here][2]
+* Bitbucket
+  * `BITBUCKET_AUTH_SERVER`: Custom auth server (uses SERVER if empty)
+  * `BITBUCKET_CLIENT`: Credentials for Bitbucket access
+  * `BITBUCKET_SECRET`: Credentials for Bitbucket access
 
 If `PLUGIN_CONCAT` is not set, the first found `.drone.yml` will be used.
 
-#### Example docker-compose:
+#### Example docker-compose
 
 ```yaml
 version: '2'
@@ -83,7 +84,7 @@ services:
 
 Edit the Secrets (`***`), `<SECRET>` and `<GITHUB_TOKEN>` to your needs. `<SECRET>` is used between Drone and drone-tree-config.
 
-#### Enable repos via regex matching:
+#### Enable repos via regex matching
 
 By default, this plugin matches against ALL repo slugs. If you want to enable the plugin for specific repos only, turn on
 regex matching by specifying a `PLUGIN_ALLOW_LIST_FILE`.
@@ -117,9 +118,50 @@ File: drone-tree-config-matchfile:
 ^myorg/myrepo$
 ```
 
-* Matches against all repos in the `bitbeats` org 
+* Matches against all repos in the `bitbeats` org
 * Matches against `myorg/myrepo`
 
 [1]: https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line
 [2]: https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html
 [3]: https://github.com/google/re2/wiki/Syntax
+
+#### Consider file
+
+ If a `PLUGIN_CONSIDER_FILE` is defined, drone-tree-config will first read the content of the target file and will only consider
+ the `.drone.yml` files specified, when matching.
+
+Depending on the size and the complexity of the repository, using a "consider file" can
+significantly reduce the number of API calls made to the provider (github, bitbucket, other). The reduction in API calls
+reduces the risk of being rate limited and can result in less processing time for drone-tree-config.
+
+Given the config;
+
+```yaml
+   - PLUGIN_CONSIDER_FILE=.drone-consider
+```
+
+A local git repo clone;
+
+```shell
+$ tree -a my-repo-clone/
+ my-repo-clone/
+ ├── .drone-consier
+ ├── foo
+ │   └── .drone.yml
+ ├── bar
+ │   └── .drone.yml
+ └── baz
+
+```
+
+Content of the .drone-consider to check in;
+
+```shell
+$ cat my-repo-clone/.drone-consider
+foo/.drone.yml
+bar/.drone.yml
+```
+
+The downside of a "consider file" is that it has to be kept in sync. As a suggestion, to help with this, a step can be
+added to each `.drone.yml` which verifies the "consider file" is in sync with the actual content of the repo. For
+example, this can be accomplished by comparing the output of `find ./ -name .drone.yml` with the content of the "consider file".

cmd/drone-tree-config/main.go

@@ -26,6 +26,7 @@ type (
 		BitBucketAuthServer string `envconfig:"BITBUCKET_AUTH_SERVER"`
 		BitBucketClient     string `envconfig:"BITBUCKET_CLIENT"`
 		BitBucketSecret     string `envconfig:"BITBUCKET_SECRET"`
+		ConsiderFile        string `envconfig:"PLUGIN_CONSIDER_FILE"`
 		// Deprecated: Use AllowListFile instead.
 		WhitelistFile string `envconfig:"PLUGIN_WHITELIST_FILE"`
 	}
@@ -70,6 +71,7 @@ func main() {
 			plugin.WithGithubToken(spec.GitHubToken),
 			plugin.WithGitlabToken(spec.GitLabToken),
 			plugin.WithGitlabServer(spec.GitLabServer),
+			plugin.WithConsiderFile(spec.ConsiderFile),
 		),
 		spec.Secret,
 		logrus.StandardLogger(),

plugin/configloader.go

@@ -27,6 +27,11 @@ func (p *Plugin) getConfigForChanges(ctx context.Context, req *request, changedF
 				cache[file] = true
 			}
 
+			// when enabled, only process drone.yml from p.considerFile
+			if p.considerFile != "" && !req.ConsiderData.consider(file) {
+				continue
+			}
+
 			// download file from git
 			fileContent, critical, err := p.getDroneConfig(ctx, req, file)
 			if err != nil {
@@ -49,6 +54,11 @@ func (p *Plugin) getConfigForChanges(ctx context.Context, req *request, changedF
 
 // getConfigForTree searches for all or first 'drone.yml' in the repo
 func (p *Plugin) getConfigForTree(ctx context.Context, req *request, dir string, depth int) (configData string, err error) {
+	if p.considerFile != "" {
+		// treats all 'drone.yml' entries in the consider file as the changedFiles
+		return p.getConfigForChanges(ctx, req, req.ConsiderData.listRepresentation)
+	}
+
 	ls, err := req.Client.GetFileListing(ctx, dir, req.Build.After)
 	if err != nil {
 		return "", err

plugin/consider.go

@@ -0,0 +1,58 @@
+package plugin
+
+import (
+	"context"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+// ConsiderData holds the considerFile information in both list and map representations
+type ConsiderData struct {
+	mapRepresentation  map[string]bool
+	listRepresentation []string
+}
+
+// consider returns true if the path provided matches an entry in the considerFile
+func (c *ConsiderData) consider(path string) bool {
+	_, exists := c.mapRepresentation[path]
+	return exists
+}
+
+// ------
+
+// newConsiderDataFromRequest returns the ConsiderData which is loaded from the considerFile
+func (p *Plugin) newConsiderDataFromRequest(ctx context.Context, req *request) (*ConsiderData, error) {
+	cd := new(ConsiderData)
+	cd.mapRepresentation = make(map[string]bool)
+	cd.listRepresentation = make([]string, 0)
+
+	// bail early without calling the scm provider when there is no considerFile configured
+	if p.considerFile == "" {
+		return cd, nil
+	}
+
+	// download considerFile from github
+	fc, err := p.getScmFile(ctx, req, p.considerFile)
+	if err != nil {
+		logrus.Errorf("%s skipping: %s is not present: %v", req.UUID, p.considerFile, err)
+		return cd, err
+	}
+
+	// collect drone.yml files
+	for _, v := range strings.Split(fc, "\n") {
+		// skip empty lines and comments
+		if strings.TrimSpace(v) == "" || strings.HasPrefix(v, "#") {
+			continue
+		}
+		// skip lines which do not contain a 'drone.yml' reference
+		if !strings.HasSuffix(v, req.Repo.Config) {
+			logrus.Warnf("%s skipping invalid reference to %s in %s", req.UUID, v, p.considerFile)
+			continue
+		}
+		cd.listRepresentation = append(cd.listRepresentation, v)
+		cd.mapRepresentation[v] = true
+	}
+
+	return cd, nil
+}

plugin/options.go

@@ -82,3 +82,11 @@ func WithAllowListFile(file string) func(*Plugin) {
 		p.allowListFile = file
 	}
 }
+
+// WithConsiderFile configures with a consider file which contains references to all 'drone.yml' files which should
+// be considered for the repository.
+func WithConsiderFile(considerFile string) func(*Plugin) {
+	return func(p *Plugin) {
+		p.considerFile = considerFile
+	}
+}

plugin/plugin.go

@@ -28,6 +28,7 @@ type (
 		fallback      bool
 		maxDepth      int
 		allowListFile string
+		considerFile  string
 	}
 
 	droneConfig struct {
@@ -37,8 +38,9 @@ type (
 
 	request struct {
 		*config.Request
-		UUID   uuid.UUID
-		Client scm_clients.ScmClient
+		UUID         uuid.UUID
+		Client       scm_clients.ScmClient
+		ConsiderData *ConsiderData
 	}
 )
 
@@ -64,45 +66,68 @@ func (p *Plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone
 		return nil, err
 	}
 
-	req := request{droneRequest, someUuid, client}
+	req := request{
+		Request: droneRequest,
+		UUID:    someUuid,
+		Client:  client,
+	}
 
 	// make sure this plugin is enabled for the requested repo slug
 	if ok := p.allowlisted(&req); !ok {
 		// do the default behavior by returning nil, nil
 		return nil, nil
 	}
 
-	// get changed files
-	changedFiles, err := p.getScmChanges(ctx, &req)
+	// load the considerFile entries, if configured for considerFile
+	if req.ConsiderData, err = p.newConsiderDataFromRequest(ctx, &req); err != nil {
+		return nil, err
+	}
+
+	configData, err := p.getConfig(ctx, &req)
 	if err != nil {
 		return nil, err
 	}
+	return &drone.Config{Data: configData}, nil
+}
+
+// getConfig retrieves drone config data from the repo
+func (p *Plugin) getConfig(ctx context.Context, req *request) (string, error) {
+	// get changed files
+	changedFiles, err := p.getScmChanges(ctx, req)
+	if err != nil {
+		return "", err
+	}
 
 	// get drone.yml for changed files or all of them if no changes/cron
 	configData := ""
 	if changedFiles != nil {
-		configData, err = p.getConfigForChanges(ctx, &req, changedFiles)
+		configData, err = p.getConfigForChanges(ctx, req, changedFiles)
 	} else if req.Build.Trigger == "@cron" {
 		logrus.Warnf("%s @cron, rebuilding all", req.UUID)
-		configData, err = p.getConfigForTree(ctx, &req, "", 0)
+		if p.considerFile == "" {
+			logrus.Warnf("recursively scanning for config files with max depth %d", p.maxDepth)
+		}
+		configData, err = p.getConfigForTree(ctx, req, "", 0)
 	} else if p.fallback {
 		logrus.Warnf("%s no changed files and fallback enabled, rebuilding all", req.UUID)
-		configData, err = p.getConfigForTree(ctx, &req, "", 0)
+		if p.considerFile == "" {
+			logrus.Warnf("recursively scanning for config files with max depth %d", p.maxDepth)
+		}
+		configData, err = p.getConfigForTree(ctx, req, "", 0)
 	}
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
 	// no file found
 	if configData == "" {
-		return nil, errors.New("did not find a .drone.yml")
+		return "", errors.New("did not find a .drone.yml")
 	}
 
 	// cleanup
-	configData = string(removeDocEndRegex.ReplaceAllString(configData, ""))
+	configData = removeDocEndRegex.ReplaceAllString(configData, "")
 	configData = string(dedupRegex.ReplaceAll([]byte(configData), []byte("---")))
-
-	return &drone.Config{Data: configData}, nil
+	return configData, nil
 }
 
 var dedupRegex = regexp.MustCompile(`(?ms)(---[\s]*){2,}`)