diff --git a/cmd/task/task.go b/cmd/task/task.go index 0de9075054..05522f857f 100644 --- a/cmd/task/task.go +++ b/cmd/task/task.go @@ -108,6 +108,7 @@ func run() error { Download: flags.Download, Offline: flags.Offline, Timeout: flags.Timeout, + CacheTTL: flags.CacheTTL, Watch: flags.Watch, Verbose: flags.Verbose, Silent: flags.Silent, diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 2a1f9476fc..30d1c3b821 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/pflag" "github.com/go-task/task/v3/internal/experiments" + "github.com/go-task/task/v3/taskfile" "github.com/go-task/task/v3/taskfile/ast" ) @@ -67,6 +68,7 @@ var ( Offline bool ClearCache bool Timeout time.Duration + CacheTTL time.Duration ) func init() { @@ -115,12 +117,14 @@ func init() { pflag.BoolVarP(&ForceAll, "force", "f", false, "Forces execution even when the task is up-to-date.") } - // Remote Taskfiles experiment will adds the "download" and "offline" flags + // Remote Taskfiles experiment will add the following flags. if experiments.RemoteTaskfiles.Enabled { pflag.BoolVar(&Download, "download", false, "Downloads a cached version of a remote Taskfile.") pflag.BoolVar(&Offline, "offline", false, "Forces Task to only use local or cached Taskfiles.") pflag.DurationVar(&Timeout, "timeout", time.Second*10, "Timeout for downloading remote Taskfiles.") pflag.BoolVar(&ClearCache, "clear-cache", false, "Clear the remote cache.") + pflag.DurationVar(&CacheTTL, "remote-cache-ttl", taskfile.DefaultCacheTTL, + "TTL of remote Taskfiles downloaded into the local cache.") } pflag.Parse() diff --git a/setup.go b/setup.go index a6eb772d90..04d18a0288 100644 --- a/setup.go +++ b/setup.go @@ -69,6 +69,7 @@ func (e *Executor) readTaskfile(node taskfile.Node) error { e.Download, e.Offline, e.Timeout, + e.CacheTTL, e.TempDir.Remote, e.Logger, ) diff --git a/signals_test.go b/signals_test.go index acfacb51a1..bde4bf8cfd 100644 --- a/signals_test.go +++ b/signals_test.go @@ -20,9 +20,7 @@ import ( "time" ) -var ( - SLEEPIT, _ = filepath.Abs("./bin/sleepit") -) +var SLEEPIT, _ = filepath.Abs("./bin/sleepit") func TestSignalSentToProcessGroup(t *testing.T) { task, err := getTaskPath() diff --git a/task.go b/task.go index 516ee20450..a52f62b552 100644 --- a/task.go +++ b/task.go @@ -52,6 +52,7 @@ type Executor struct { Download bool Offline bool Timeout time.Duration + CacheTTL time.Duration Watch bool Verbose bool Silent bool diff --git a/taskfile/README.md b/taskfile/README.md new file mode 100644 index 0000000000..9b04190064 --- /dev/null +++ b/taskfile/README.md @@ -0,0 +1,43 @@ +# The `taskfile` package + +```mermaid +--- +title: taskfile.Cache behaviour +--- +flowchart LR + %% Beginning state + start([A remote Taskfile + is required]) + + %% Checks to decide + cached{Remote Taskfile + already cached?} + + subgraph checkTTL [Is the cached Taskfile still inside TTL?] + %% Beginning state + lastModified(Stat the cached + Taskfile and get last + modified timestamp) + + %% Check to decide + timestampPlusTTL{Timestamp + plus TTL is in + the future?} + + %% Flowlines + lastModified-->timestampPlusTTL + end + + %% End states + useCached([Use the + cached Taskfile]) + download(["(Re)download the + remote Taskfile"]) + + %% Flowlines + start-->cached + cached-- Yes -->lastModified + cached-- No -->download + timestampPlusTTL-- Yes -->useCached + timestampPlusTTL-- No -->download +``` diff --git a/taskfile/cache.go b/taskfile/cache.go index 2b57c17dd8..344c4ffe5a 100644 --- a/taskfile/cache.go +++ b/taskfile/cache.go @@ -2,24 +2,51 @@ package taskfile import ( "crypto/sha256" + "errors" "fmt" "os" "path/filepath" "strings" + "time" ) +// DefaultCacheTTL is used when a value is not explicitly given to a new Cache. +const DefaultCacheTTL = time.Duration(time.Hour * 24) + type Cache struct { dir string + ttl time.Duration } -func NewCache(dir string) (*Cache, error) { +// ErrExpired is returned when a cached file has expired. +var ErrExpired = errors.New("task: cache expired") + +type CacheOption func(*Cache) + +func NewCache(dir string, opts ...CacheOption) (*Cache, error) { dir = filepath.Join(dir, "remote") if err := os.MkdirAll(dir, 0o755); err != nil { return nil, err } - return &Cache{ + + cache := &Cache{ dir: dir, - }, nil + ttl: DefaultCacheTTL, + } + + // Apply options. + for _, opt := range opts { + opt(cache) + } + + return cache, nil +} + +// WithTTL will override the default TTL setting on a new Cache. +func WithTTL(ttl time.Duration) CacheOption { + return func(cache *Cache) { + cache.ttl = ttl + } } func checksum(b []byte) string { @@ -33,7 +60,19 @@ func (c *Cache) write(node Node, b []byte) error { } func (c *Cache) read(node Node) ([]byte, error) { - return os.ReadFile(c.cacheFilePath(node)) + cfp := c.cacheFilePath(node) + + fi, err := os.Stat(cfp) + if err != nil { + return nil, fmt.Errorf("could not stat cached file: %w", err) + } + + expiresAt := fi.ModTime().Add(c.ttl) + if expiresAt.Before(time.Now()) { + return nil, ErrExpired + } + + return os.ReadFile(cfp) } func (c *Cache) writeChecksum(node Node, checksum string) error { diff --git a/taskfile/cache_internal_test.go b/taskfile/cache_internal_test.go new file mode 100644 index 0000000000..63dd937eec --- /dev/null +++ b/taskfile/cache_internal_test.go @@ -0,0 +1,99 @@ +package taskfile + +import ( + "fmt" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/go-task/task/v3/internal/logger" +) + +func ExampleWithTTL() { + c, _ := NewCache(os.TempDir(), WithTTL(2*time.Minute+30*time.Second)) + + fmt.Println(c.ttl) + // Output: 2m30s +} + +var discardLogger = &logger.Logger{ + Stdout: io.Discard, + Stderr: io.Discard, +} + +func primeNewCache(t *testing.T, cacheOpts ...CacheOption) (*Cache, *FileNode) { + t.Helper() + + cache, err := NewCache(t.TempDir(), cacheOpts...) + require.NoErrorf(t, err, "creating new cache") + + // Prime the temporary cache directory with a basic Taskfile. + filename := "Taskfile.yaml" + srcTaskfile := filepath.Join("testdata", filename) + dstTaskfile := filepath.Join(cache.dir, filename) + + taskfileBytes, err := os.ReadFile(srcTaskfile) + require.NoErrorf(t, err, "reading from testdata Taskfile (%s)", srcTaskfile) + + err = os.WriteFile(dstTaskfile, taskfileBytes, 0o640) + require.NoErrorf(t, err, "writing to temporary Taskfile (%s)", dstTaskfile) + + // Create a new file node in the cache, with the entrypoint copied above. + fileNode, err := NewFileNode(discardLogger, dstTaskfile, cache.dir) + require.NoError(t, err, "creating new file node") + + return cache, fileNode +} + +func TestCache(t *testing.T) { + cache, fileNode := primeNewCache(t) + + // Attempt to read from cache, then write, then read again. + _, err := cache.read(fileNode) + require.ErrorIs(t, err, os.ErrNotExist, "reading from cache before writing should match error type") + + writeBytes := []byte("some bytes") + err = cache.write(fileNode, writeBytes) + require.NoError(t, err, "writing bytes to cache") + + readBytes, err := cache.read(fileNode) + require.NoError(t, err, "reading from cache after write should not error") + require.Equal(t, writeBytes, readBytes, "bytes read from cache should match bytes written") +} + +func TestCacheInsideTTL(t *testing.T) { + // Prime a new Cache with a TTL of one minute. + cache, fileNode := primeNewCache(t, WithTTL(time.Minute)) + + // Write some bytes for the cached file. + writeBytes := []byte("some bytes") + err := cache.write(fileNode, writeBytes) + require.NoError(t, err, "writing bytes to cache") + + // Reading from the cache while still inside the TTL should get the written bytes back. + readBytes, err := cache.read(fileNode) + require.NoError(t, err, "reading from cache inside TTL should not error") + require.Equal(t, writeBytes, readBytes, "bytes read from cache should match bytes written") +} + +func TestCacheOutsideTTL(t *testing.T) { + // Prime a new Cache with an extremely short TTL. + cache, fileNode := primeNewCache(t, WithTTL(time.Millisecond)) + + // Write some bytes for the cached file. + writeBytes := []byte("some bytes") + err := cache.write(fileNode, writeBytes) + require.NoError(t, err, "writing bytes to cache") + + // Sleep for 5ms so that the cached file is outside of TTL. + time.Sleep(5 * time.Millisecond) + + // Reading from the cache after sleeping past the end of TTL should get an error. + readBytes, err := cache.read(fileNode) + require.Empty(t, readBytes, "should not have read any bytes from cache") + require.ErrorIs(t, err, ErrExpired, "should get 'expired' error when attempting to read from cache outside of TTL") +} diff --git a/taskfile/cache_test.go b/taskfile/cache_test.go new file mode 100644 index 0000000000..0cb2044f18 --- /dev/null +++ b/taskfile/cache_test.go @@ -0,0 +1,29 @@ +package taskfile_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/go-task/task/v3/taskfile" +) + +func TestNewCache(t *testing.T) { + testCases := map[string]struct { + options []taskfile.CacheOption + }{ + "no options set": {}, + + "TTL option set": { + options: []taskfile.CacheOption{taskfile.WithTTL(time.Hour)}, + }, + } + + for desc, testCase := range testCases { + t.Run(desc, func(t *testing.T) { + _, err := taskfile.NewCache(t.TempDir(), testCase.options...) + require.NoError(t, err, "creating new cache") + }) + } +} diff --git a/taskfile/reader.go b/taskfile/reader.go index 7899ee0907..8a2dc2982b 100644 --- a/taskfile/reader.go +++ b/taskfile/reader.go @@ -36,6 +36,7 @@ type Reader struct { download bool offline bool timeout time.Duration + cacheTTL time.Duration tempDir string logger *logger.Logger } @@ -46,6 +47,7 @@ func NewReader( download bool, offline bool, timeout time.Duration, + cacheTTL time.Duration, tempDir string, logger *logger.Logger, ) *Reader { @@ -56,6 +58,7 @@ func NewReader( download: download, offline: offline, timeout: timeout, + cacheTTL: cacheTTL, tempDir: tempDir, logger: logger, } @@ -185,7 +188,7 @@ func (r *Reader) readNode(node Node) (*ast.Taskfile, error) { var cache *Cache if node.Remote() { - cache, err = NewCache(r.tempDir) + cache, err = NewCache(r.tempDir, WithTTL(r.cacheTTL)) if err != nil { return nil, err } diff --git a/taskfile/testdata/Taskfile.yaml b/taskfile/testdata/Taskfile.yaml new file mode 100644 index 0000000000..f19bd17e1c --- /dev/null +++ b/taskfile/testdata/Taskfile.yaml @@ -0,0 +1,3 @@ +version: '3' + +tasks: {} diff --git a/watch_test.go b/watch_test.go index f1fb6a218d..c581b4983a 100644 --- a/watch_test.go +++ b/watch_test.go @@ -41,10 +41,10 @@ Hello, World! require.NoError(t, e.Setup()) buff.Reset() - err := os.MkdirAll(filepathext.SmartJoin(dir, "src"), 0755) + err := os.MkdirAll(filepathext.SmartJoin(dir, "src"), 0o755) require.NoError(t, err) - err = os.WriteFile(filepathext.SmartJoin(dir, "src/a"), []byte("test"), 0644) + err = os.WriteFile(filepathext.SmartJoin(dir, "src/a"), []byte("test"), 0o644) if err != nil { t.Fatal(err) } @@ -67,7 +67,7 @@ Hello, World! }(ctx) time.Sleep(10 * time.Millisecond) - err = os.WriteFile(filepathext.SmartJoin(dir, "src/a"), []byte("test updated"), 0644) + err = os.WriteFile(filepathext.SmartJoin(dir, "src/a"), []byte("test updated"), 0o644) if err != nil { t.Fatal(err) } diff --git a/website/docs/experiments/remote_taskfiles.mdx b/website/docs/experiments/remote_taskfiles.mdx index 8a85ccd0b6..d5bb3a3c7b 100644 --- a/website/docs/experiments/remote_taskfiles.mdx +++ b/website/docs/experiments/remote_taskfiles.mdx @@ -104,21 +104,33 @@ Whenever you run a remote Taskfile, the latest copy will be downloaded from the internet and cached locally. If for whatever reason, you lose access to the internet, you will still be able to run your tasks by specifying the `--offline` flag. This will tell Task to use the latest cached version of the file instead -of trying to download it. You are able to use the `--download` flag to update -the cached version of the remote files without running any tasks. You are able -to use the `--clear-cache` flag to clear all cached version of the remote files -without running any tasks. +of trying to download it. -By default, Task will timeout requests to download remote files after 10 seconds -and look for a cached copy instead. This timeout can be configured by setting -the `--timeout` flag and specifying a duration. For example, `--timeout 5s` will -set the timeout to 5 seconds. +The flags to affect the caching behaviour are as follows: + +- `--download` (boolean): update the cached version of the remote files without + running any tasks. +- `--clear-cache` (boolean): clear all cached version of the remote files + without running any tasks. +- `--remote-cache-ttl` (duration): override the default time-to-live (TTL) of + remote Taskfiles in the cache, when there is a cached Taskfile present and + deciding whether to use the cached copy or download a newer version. + +By default, Task requests to download remote files will time out after 10 +seconds and look for a cached copy instead. This timeout can be configured by +setting the `--timeout` flag and specifying a duration. For example, +`--timeout 5s` will set the timeout to 5 seconds. By default, the cache is stored in the Task temp directory, represented by the `TASK_TEMP_DIR` [environment variable](../reference/environment.mdx) You can override the location of the cache by setting the `TASK_REMOTE_DIR` environment variable. This way, you can share the cache between different projects. +The default TTL for remote Taskfiles in the cache is 24 hours. This can be +overridden by the `--remote-cache-ttl` flag outlined above. The value must be +specified as a duration, e.g. `30s`, `15m`, `4h`. More information on how +[durations are parsed is available here](https://pkg.go.dev/time#ParseDuration). + {/* prettier-ignore-start */} [enabling-experiments]: ./experiments.mdx#enabling-experiments [man-in-the-middle-attacks]: https://en.wikipedia.org/wiki/Man-in-the-middle_attack