Skip to content

Commit

Permalink
Merge pull request #36 from mach6/mach6_caching
Browse files Browse the repository at this point in the history
Add ability to leverage an in memory cache
  • Loading branch information
foosinn authored May 17, 2021
2 parents 5f9de16 + 5734575 commit 4805f78
Show file tree
Hide file tree
Showing 6 changed files with 251 additions and 26 deletions.
22 changes: 19 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ Currently supports
* `PLUGIN_ADDRESS`: Listen address for the plugins webserver. Defaults to `:3000`.
* `PLUGIN_SECRET`: Shared secret with drone. You can generate the token using `openssl rand -hex 16`.
* `PLUGIN_ALLOW_LIST_FILE`: (Optional) Path to regex pattern file. Matches the repo slug(s) against a list of regex patterns. Defaults to `""`, match everything.
* `PLUGIN_CACHE_TTL`: (Optional) Cache entry time to live value. When defined and greater than `0s`, enables in memory caching for request/response pairs.
* `PLUGIN_CONSIDER_FILE`: (Optional) Consider file name. Only consider the `.drone.yml` files listed in this file. When defined, all enabled repos must contain a consider file.

Backend specific options
Expand Down Expand Up @@ -130,9 +131,9 @@ File: drone-tree-config-matchfile:
If a `PLUGIN_CONSIDER_FILE` is defined, drone-tree-config will first read the content of the target file and will only consider
the `.drone.yml` files specified, when matching.

Depending on the size and the complexity of the repository, using a "consider file" can
significantly reduce the number of API calls made to the provider (github, bitbucket, other). The reduction in API calls
reduces the risk of being rate limited and can result in less processing time for drone-tree-config.
Depending on the size and the complexity of the repository, using a "consider file" can significantly reduce the number
of API calls made to the provider (github, bitbucket, other). The reduction in API calls reduces the risk of being rate
limited and can result in less processing time for drone-tree-config.

Given the config;

Expand Down Expand Up @@ -165,3 +166,18 @@ bar/.drone.yml
The downside of a "consider file" is that it has to be kept in sync. As a suggestion, to help with this, a step can be
added to each `.drone.yml` which verifies the "consider file" is in sync with the actual content of the repo. For
example, this can be accomplished by comparing the output of `find ./ -name .drone.yml` with the content of the "consider file".

#### Caching

If a `PLUGIN_CACHE_TTL` is defined, drone-tree-config will leverage an in memory cache to match the inbound requests
against ones that exist in the cache. When a match is found, the cached response is returned. Cached entries are
expired and removed when their per-entry TTL is reached.

Example (expire after 30 minutes);
```yaml
- PLUGIN_CACHE_TTL=30m
```

Depending on the size and the complexity of the repository, using a cache can significantly reduce the number of API
calls made to the provider (github, bitbucket, other). The reduction in API calls reduces the risk of being rate
limited and can result in less processing time for drone-tree-config.
33 changes: 18 additions & 15 deletions cmd/drone-tree-config/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"net/http"
"time"

"github.com/bitsbeats/drone-tree-config/plugin"

Expand All @@ -12,21 +13,22 @@ import (

type (
spec struct {
AllowListFile string `envconfig:"PLUGIN_ALLOW_LIST_FILE"`
Concat bool `envconfig:"PLUGIN_CONCAT"`
MaxDepth int `envconfig:"PLUGIN_MAXDEPTH" default:"2"`
Fallback bool `envconfig:"PLUGIN_FALLBACK"`
Debug bool `envconfig:"PLUGIN_DEBUG"`
Address string `envconfig:"PLUGIN_ADDRESS" default:":3000"`
Secret string `envconfig:"PLUGIN_SECRET"`
Server string `envconfig:"SERVER" default:"https://api.github.com"`
GitHubToken string `envconfig:"GITHUB_TOKEN"`
GitLabToken string `envconfig:"GITLAB_TOKEN"`
GitLabServer string `envconfig:"GITLAB_SERVER" default:"https://gitlab.com"`
BitBucketAuthServer string `envconfig:"BITBUCKET_AUTH_SERVER"`
BitBucketClient string `envconfig:"BITBUCKET_CLIENT"`
BitBucketSecret string `envconfig:"BITBUCKET_SECRET"`
ConsiderFile string `envconfig:"PLUGIN_CONSIDER_FILE"`
AllowListFile string `envconfig:"PLUGIN_ALLOW_LIST_FILE"`
Concat bool `envconfig:"PLUGIN_CONCAT"`
MaxDepth int `envconfig:"PLUGIN_MAXDEPTH" default:"2"`
Fallback bool `envconfig:"PLUGIN_FALLBACK"`
Debug bool `envconfig:"PLUGIN_DEBUG"`
Address string `envconfig:"PLUGIN_ADDRESS" default:":3000"`
Secret string `envconfig:"PLUGIN_SECRET"`
Server string `envconfig:"SERVER" default:"https://api.github.com"`
GitHubToken string `envconfig:"GITHUB_TOKEN"`
GitLabToken string `envconfig:"GITLAB_TOKEN"`
GitLabServer string `envconfig:"GITLAB_SERVER" default:"https://gitlab.com"`
BitBucketAuthServer string `envconfig:"BITBUCKET_AUTH_SERVER"`
BitBucketClient string `envconfig:"BITBUCKET_CLIENT"`
BitBucketSecret string `envconfig:"BITBUCKET_SECRET"`
ConsiderFile string `envconfig:"PLUGIN_CONSIDER_FILE"`
CacheTTL time.Duration `envconfig:"PLUGIN_CACHE_TTL"`
}
)

Expand Down Expand Up @@ -66,6 +68,7 @@ func main() {
plugin.WithGitlabToken(spec.GitLabToken),
plugin.WithGitlabServer(spec.GitLabServer),
plugin.WithConsiderFile(spec.ConsiderFile),
plugin.WithCacheTTL(spec.CacheTTL),
),
spec.Secret,
logrus.StandardLogger(),
Expand Down
112 changes: 112 additions & 0 deletions plugin/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
package plugin

import (
"fmt"
"sync"
"time"

"github.com/drone/drone-go/drone"
"github.com/google/uuid"
"github.com/sirupsen/logrus"
)

const (
msgCacheHit = "%s config-cache found entry for %s"
msgCacheExpire = "config-cache expired entry for %s"
msgCacheAdd = "%s config-cache added entry for %s"
)

// configCache is used to cache config responses on a per request basis
type configCache struct {
syncMap sync.Map
}

// cacheKey holds the unique key details which are associated with the config request
type cacheKey struct {
slug string
ref string
before string
after string
event string
trigger string
author string
}

// cacheEntry holds the response and ttl for a config request
type cacheEntry struct {
config string
error error
ttl *time.Timer
}

// newCacheEntry creates a new cacheEntry using the config string and error provided. The returned struct will have a
// nil ttl value -- it will be established when a entry is added to the cache via the add function.
func newCacheEntry(config string, error error) *cacheEntry {
entry := &cacheEntry{
config: config,
error: error,
}
return entry
}

// newCacheKey creates a new cacheKey for the provided request.
func newCacheKey(req *request) cacheKey {
ck := cacheKey{
slug: req.Repo.Slug,
ref: req.Build.Ref,
before: req.Build.Before,
after: req.Build.After,
event: req.Build.Event,
author: req.Build.Author,
trigger: req.Build.Trigger,
}
return ck
}

// add an entry to the cache
func (c *configCache) add(uuid uuid.UUID, key cacheKey, entry *cacheEntry, ttl time.Duration) {
logrus.Infof(msgCacheAdd, uuid, fmt.Sprintf("%+v", key))

entry.ttl = time.AfterFunc(ttl, func() {
c.expire(key)
})

c.syncMap.Store(key, entry)
}

// expire is typically called internally via a time.Afterfunc
func (c *configCache) expire(key cacheKey) {
logrus.Infof(msgCacheExpire, fmt.Sprintf("%+v", key))

if entry, _ := c.syncMap.Load(key); entry != nil {
entry.(*cacheEntry).ttl.Stop()
c.syncMap.Delete(key)
}
}

// retrieve an entry from the cache, if it exists
func (c *configCache) retrieve(uuid uuid.UUID, key cacheKey) (*cacheEntry, bool) {
entry, exists := c.syncMap.Load(key)
if exists {
logrus.Infof(msgCacheHit, uuid, fmt.Sprintf("%+v", key))
return entry.(*cacheEntry), true
}

return nil, false
}

// cacheAndReturn caches the result (if enabled) and returns the (drone.Config, error) that should be
// returned to the Find request.
func (p *Plugin) cacheAndReturn(uuid uuid.UUID, key cacheKey, entry *cacheEntry) (*drone.Config, error) {
var config *drone.Config
if entry.config != "" {
config = &drone.Config{Data: entry.config}
}

// cache the config before we return it, if enabled
if p.cacheTTL > 0 {
p.cache.add(uuid, key, entry, p.cacheTTL)
}

return config, entry.error
}
9 changes: 9 additions & 0 deletions plugin/options.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package plugin

import "time"

// WithServer configures with a custom SCM server
func WithServer(server string) func(*Plugin) {
return func(p *Plugin) {
Expand Down Expand Up @@ -84,3 +86,10 @@ func WithConsiderFile(considerFile string) func(*Plugin) {
p.considerFile = considerFile
}
}

// WithCacheTTL enables request/response caching and the specified TTL for each entry
func WithCacheTTL(ttl time.Duration) func(*Plugin) {
return func(p *Plugin) {
p.cacheTTL = ttl
}
}
39 changes: 31 additions & 8 deletions plugin/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"regexp"
"strings"
"time"

"github.com/bitsbeats/drone-tree-config/plugin/scm_clients"
"github.com/drone/drone-go/drone"
Expand All @@ -29,6 +30,8 @@ type (
maxDepth int
allowListFile string
considerFile string
cacheTTL time.Duration
cache *configCache
}

droneConfig struct {
Expand All @@ -46,7 +49,9 @@ type (

// New creates a drone plugin
func New(options ...func(*Plugin)) config.Plugin {
p := &Plugin{}
p := &Plugin{
cache: &configCache{},
}
for _, opt := range options {
opt(p)
}
Expand Down Expand Up @@ -83,15 +88,12 @@ func (p *Plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone
return nil, err
}

configData, err := p.getConfig(ctx, &req)
if err != nil {
return nil, err
}
return &drone.Config{Data: configData}, nil
return p.getConfig(ctx, &req)
}

// getConfig retrieves drone config data from the repo
func (p *Plugin) getConfig(ctx context.Context, req *request) (string, error) {
// getConfig retrieves drone config data. When the cache is enabled, this func will first check entries in
// the cache as well as add new entries.
func (p *Plugin) getConfig(ctx context.Context, req *request) (*drone.Config, error) {
logrus.WithFields(logrus.Fields{
"after": req.Build.After,
"before": req.Build.Before,
Expand All @@ -101,6 +103,27 @@ func (p *Plugin) getConfig(ctx context.Context, req *request) (string, error) {
"trigger": req.Build.Trigger,
}).Debugf("drone-tree-config environment")

// check cache first, when enabled
ck := newCacheKey(req)
if p.cacheTTL > 0 {
if cached, exists := p.cache.retrieve(req.UUID, ck); exists {
if cached != nil {
return &drone.Config{Data: cached.config}, cached.error
}
}
}

// fetch the config data. cache it, when enabled
return p.cacheAndReturn(
req.UUID, ck,
newCacheEntry(
p.getConfigData(ctx, req),
),
)
}

// getConfigData retrieves drone config data from the repo
func (p *Plugin) getConfigData(ctx context.Context, req *request) (string, error) {
// get changed files
changedFiles, err := p.getScmChanges(ctx, req)
if err != nil {
Expand Down
62 changes: 62 additions & 0 deletions plugin/plugin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ import (
"io"
"os"
"testing"
"time"

"net/http"
"net/http/httptest"

"github.com/drone/drone-go/drone"
"github.com/drone/drone-go/plugin/config"
"github.com/google/uuid"
"github.com/sirupsen/logrus"
)

Expand All @@ -21,6 +23,8 @@ const mockToken = "7535706b694c63526c6e4f5230374243"
var ts *httptest.Server

func TestMain(m *testing.M) {
logrus.SetLevel(logrus.DebugLevel)

ts = httptest.NewServer(testMux())
defer ts.Close()
os.Exit(m.Run())
Expand Down Expand Up @@ -252,6 +256,64 @@ func TestCron(t *testing.T) {
}
}

func TestCache(t *testing.T) {
req := &config.Request{
Build: drone.Build{
After: "8ecad91991d5da985a2a8dd97cc19029dc1c2899",
Trigger: "@cron",
},
Repo: drone.Repo{
Namespace: "foosinn",
Name: "dronetest",
Slug: "foosinn/dronetest",
Config: ".drone.yml",
},
}

// used to directly retrieve the cacheEntry, for verification
r := &request{
Request: req,
UUID: uuid.New(),
}
ck := newCacheKey(r)

p := &Plugin{
server: ts.URL,
gitHubToken: mockToken,
concat: true,
maxDepth: 2,
cacheTTL: time.Minute*1,
cache: &configCache{},
}

// test cache hit
for i := 0; i < 2; i++ {
droneConfig, err := p.Find(noContext, req)
if err != nil {
t.Error(err)
return
}

if entry, ok := p.cache.retrieve(r.UUID, ck); ok {
if want := droneConfig.Data; entry.config != want {
t.Errorf("Want %q got %q", droneConfig.Data, entry.config)
}
if want := err; entry.error != want {
t.Errorf("Want %q got %q", want, entry.error)
}
} else {
t.Error("entry not in cache")
}
}

// test cache expire
p.cache.expire(ck)
entry, ok := p.cache.retrieve(r.UUID, ck)
if entry != nil || ok {
t.Error("entry still in cache")
}
}

func TestMatchEnable(t *testing.T) {
req := &config.Request{
Build: drone.Build{
Expand Down

0 comments on commit 4805f78

Please sign in to comment.