diff --git a/docs/cli/run.md b/docs/cli/run.md index c8107b74ae..be25e4cc4b 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -101,6 +101,8 @@ Change how tasks information is output when running tasks - `quiet` - Don't show extra output - `silent` - Don't show any output including stdout and stderr from the task except for errors +### `--no-cache` + Examples: ``` diff --git a/docs/cli/tasks/run.md b/docs/cli/tasks/run.md index 3ac1582fb7..7e71d67cc5 100644 --- a/docs/cli/tasks/run.md +++ b/docs/cli/tasks/run.md @@ -115,6 +115,8 @@ Change how tasks information is output when running tasks - `quiet` - Don't show extra output - `silent` - Don't show any output including stdout and stderr from the task except for errors +### `--no-cache` + Examples: ``` diff --git a/docs/tasks/toml-tasks.md b/docs/tasks/toml-tasks.md index 008833a34c..4200c15239 100644 --- a/docs/tasks/toml-tasks.md +++ b/docs/tasks/toml-tasks.md @@ -341,8 +341,13 @@ Task files can be fetched via http: file = "https://example.com/build.sh" ``` -Currently, they're fetched everytime they're executed, but we may add some cache support later. -This could be extended with other protocols like mentioned in [this ticket](https://github.com/jdx/mise/issues/2488) if there were interest. +Each task file is cached in the `MISE_CACHE_DIR` directory. If the file is updated, it will not be re-downloaded unless the cache is cleared. + +:::tip +You can reset the cache by running `mise cache clear`. +::: + +You can use the `MISE_TASK_REMOTE_NO_CACHE` environment variable to disable caching of remote tasks. ## Arguments diff --git a/mise.usage.kdl b/mise.usage.kdl index c851068455..4bd681a7df 100644 --- a/mise.usage.kdl +++ b/mise.usage.kdl @@ -604,6 +604,7 @@ cmd run help="Run task(s)" { long_help "Change how tasks information is output when running tasks\n\n- `prefix` - Print stdout/stderr by line, prefixed with the task's label\n- `interleave` - Print directly to stdout/stderr instead of by line\n- `replacing` - Stdout is replaced each time, stderr is printed as is\n- `timed` - Only show stdout lines if they are displayed for more than 1 second\n- `keep-order` - Print stdout/stderr by line, prefixed with the task's label, but keep the order of the output\n- `quiet` - Don't show extra output\n- `silent` - Don't show any output including stdout and stderr from the task except for errors" arg } + flag --no-cache mount run="mise tasks --usage" } cmd self-update help="Updates mise itself." { @@ -844,6 +845,7 @@ cmd tasks help="Manage tasks" { long_help "Change how tasks information is output when running tasks\n\n- `prefix` - Print stdout/stderr by line, prefixed with the task's label\n- `interleave` - Print directly to stdout/stderr instead of by line\n- `replacing` - Stdout is replaced each time, stderr is printed as is\n- `timed` - Only show stdout lines if they are displayed for more than 1 second\n- `keep-order` - Print stdout/stderr by line, prefixed with the task's label, but keep the order of the output\n- `quiet` - Don't show extra output\n- `silent` - Don't show any output including stdout and stderr from the task except for errors" arg } + flag --no-cache arg "[TASK]" help="Tasks to run\nCan specify multiple tasks by separating with `:::`\ne.g.: mise run task1 arg1 arg2 ::: task2 arg1 arg2" required=#false default=default arg "[ARGS]..." help="Arguments to pass to the tasks. Use \":::\" to separate tasks" required=#false var=#true arg "[-- ARGS_LAST]..." help="Arguments to pass to the tasks. Use \":::\" to separate tasks" required=#false var=#true hide=#true diff --git a/schema/mise.json b/schema/mise.json index a51ceddf73..a60a4e9861 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -875,6 +875,10 @@ "silent" ] }, + "task_remote_no_cache": { + "description": "Mise will always fetch the latest tasks from the remote, by default the cache is used.", + "type": "boolean" + }, "task_run_auto_install": { "default": true, "description": "Automatically install missing tools when executing tasks.", diff --git a/settings.toml b/settings.toml index 413a21072f..de48617d9b 100644 --- a/settings.toml +++ b/settings.toml @@ -1030,6 +1030,12 @@ docs = """ Change output style when executing tasks. This controls the output of `mise run`. """ +[task_remote_no_cache] +env = "MISE_TASK_REMOTE_NO_CACHE" +type = "Bool" +optional = true +description = "Mise will always fetch the latest tasks from the remote, by default the cache is used." + [task_run_auto_install] env = "MISE_TASK_RUN_AUTO_INSTALL" type = "Bool" diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 895c85e4a0..79114ec480 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -391,6 +391,7 @@ impl Cli { keep_order_output: Default::default(), task_prs: Default::default(), timed_outputs: Default::default(), + no_cache: Default::default(), })); } else if let Some(cmd) = external::COMMANDS.get(&task) { external::execute( diff --git a/src/cli/run.rs b/src/cli/run.rs index 8c36126a30..0ce15882fe 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -15,7 +15,7 @@ use crate::config::{Config, SETTINGS}; use crate::env_diff::EnvMap; use crate::errors::Error; use crate::file::display_path; -use crate::task::file_providers::TaskFileProviders; +use crate::task::task_file_providers::TaskFileProviders; use crate::task::{Deps, GetMatchingExt, Task}; use crate::toolset::{InstallOptions, ToolsetBuilder}; use crate::ui::multi_progress_report::MultiProgressReport; @@ -194,6 +194,10 @@ pub struct Run { #[clap(skip)] pub timed_outputs: Arc>>, + + // Do not use cache on remote tasks + #[clap(long, verbatim_doc_comment, env = "MISE_TASK_REMOTE_NO_CACHE")] + pub no_cache: bool, } type KeepOrderOutputs = (Vec<(String, String)>, Vec<(String, String)>); @@ -873,7 +877,8 @@ impl Run { } fn fetch_tasks(&self, tasks: &mut Vec) -> Result<()> { - let task_file_providers = TaskFileProviders::new(self.tmpdir.clone()); + let no_cache = self.no_cache || SETTINGS.task_remote_no_cache.unwrap_or(false); + let task_file_providers = TaskFileProviders::new(no_cache); for t in tasks { if let Some(file) = &t.file { diff --git a/src/task/file_providers/http_file_provider.rs b/src/task/file_providers/http_file_provider.rs deleted file mode 100644 index 3df8488bad..0000000000 --- a/src/task/file_providers/http_file_provider.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::path::PathBuf; - -use crate::{file, http::HTTP}; - -use super::TaskFileProvider; - -#[derive(Debug)] -pub struct HttpTaskFileProvider { - tmpdir: PathBuf, -} - -impl HttpTaskFileProvider { - pub fn new(tmpdir: PathBuf) -> Self { - Self { tmpdir } - } -} - -impl TaskFileProvider for HttpTaskFileProvider { - fn is_match(&self, file: &str) -> bool { - file.starts_with("http://") || file.starts_with("https://") - } - - fn get_local_path(&self, file: &str) -> Result> { - let url = url::Url::parse(file)?; - let filename = url - .path_segments() - .and_then(|segments| segments.last()) - .unwrap(); - let tmp_path = self.tmpdir.join(filename); - HTTP.download_file(file, &tmp_path, None)?; - file::make_executable(&tmp_path)?; - Ok(tmp_path) - } -} - -#[cfg(test)] -mod tests { - - use std::env; - - use super::*; - - #[test] - fn test_is_match() { - let provider = HttpTaskFileProvider::new(env::temp_dir()); - assert!(provider.is_match("http://test.txt")); - assert!(provider.is_match("https://test.txt")); - assert!(provider.is_match("https://mydomain.com/myfile.py")); - assert!(provider.is_match("https://subdomain.mydomain.com/myfile.sh")); - assert!(provider.is_match("https://subdomain.mydomain.com/myfile.sh?query=1")); - } - - #[test] - fn test_http_task_file_provider_get_local_path() { - let paths = vec![ - "/myfile.py", - "/subpath/myfile.sh", - "/myfile.sh?query=1&sdfsdf=2", - ]; - let mut server = mockito::Server::new(); - - for path in paths { - let mocked_server = server - .mock("GET", path) - .with_status(200) - .with_body("Random content") - .create(); - - let provider = HttpTaskFileProvider::new(env::temp_dir()); - let mock = format!("{}{}", server.url(), path); - let path = provider.get_local_path(&mock).unwrap(); - assert!(path.exists()); - assert!(path.is_file()); - - mocked_server.assert(); - } - } -} diff --git a/src/task/mod.rs b/src/task/mod.rs index b2c2a10d0c..57a6fcda4a 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -26,8 +26,8 @@ use std::{ffi, fmt, path}; use xx::regex; mod deps; -pub mod file_providers; mod task_dep; +pub mod task_file_providers; mod task_script_parser; pub mod task_sources; diff --git a/src/task/file_providers/local_file_provider.rs b/src/task/task_file_providers/local_task.rs similarity index 86% rename from src/task/file_providers/local_file_provider.rs rename to src/task/task_file_providers/local_task.rs index 834b79007d..59dcba6dc0 100644 --- a/src/task/file_providers/local_file_provider.rs +++ b/src/task/task_file_providers/local_task.rs @@ -3,9 +3,9 @@ use std::path::{Path, PathBuf}; use super::TaskFileProvider; #[derive(Debug)] -pub struct LocalTaskFileProvider; +pub struct LocalTask; -impl TaskFileProvider for LocalTaskFileProvider { +impl TaskFileProvider for LocalTask { fn is_match(&self, file: &str) -> bool { let path = Path::new(file); @@ -24,7 +24,7 @@ mod tests { #[test] fn test_is_match() { - let provider = LocalTaskFileProvider; + let provider = LocalTask; assert!(provider.is_match("filetask.bat")); assert!(provider.is_match("filetask")); assert!(provider.is_match("/test.txt")); @@ -34,7 +34,7 @@ mod tests { #[test] fn test_get_local_path() { - let provider = LocalTaskFileProvider; + let provider = LocalTask; assert_eq!( provider.get_local_path("/test.txt").unwrap(), PathBuf::from("/test.txt") diff --git a/src/task/file_providers/mod.rs b/src/task/task_file_providers/mod.rs similarity index 64% rename from src/task/file_providers/mod.rs rename to src/task/task_file_providers/mod.rs index cd5707d084..ee6d63c168 100644 --- a/src/task/file_providers/mod.rs +++ b/src/task/task_file_providers/mod.rs @@ -1,10 +1,15 @@ +use std::sync::LazyLock as Lazy; use std::{fmt::Debug, path::PathBuf}; -mod http_file_provider; -mod local_file_provider; +mod local_task; +mod remote_task_http; -pub use http_file_provider::HttpTaskFileProvider; -pub use local_file_provider::LocalTaskFileProvider; +pub use local_task::LocalTask; +pub use remote_task_http::RemoteTaskHttp; + +use crate::dirs; + +static REMOTE_TASK_CACHE_DIR: Lazy = Lazy::new(|| dirs::CACHE.join("remote-tasks-cache")); pub trait TaskFileProvider: Debug { fn is_match(&self, file: &str) -> bool; @@ -12,21 +17,24 @@ pub trait TaskFileProvider: Debug { } pub struct TaskFileProviders { - tmpdir: PathBuf, + no_cache: bool, } impl TaskFileProviders { + pub fn new(no_cache: bool) -> Self { + Self { no_cache } + } + fn get_providers(&self) -> Vec> { vec![ - Box::new(HttpTaskFileProvider::new(self.tmpdir.clone())), - Box::new(LocalTaskFileProvider), // Must be the last provider + Box::new(RemoteTaskHttp::new( + REMOTE_TASK_CACHE_DIR.clone(), + self.no_cache, + )), + Box::new(LocalTask), // Must be the last provider ] } - pub fn new(tmpdir: PathBuf) -> Self { - Self { tmpdir } - } - pub fn get_provider(&self, file: &str) -> Option> { self.get_providers().into_iter().find(|p| p.is_match(file)) } @@ -34,32 +42,31 @@ impl TaskFileProviders { #[cfg(test)] mod tests { - use std::env; use super::*; #[test] fn test_get_providers() { - let task_file_providers = TaskFileProviders::new(env::temp_dir()); + let task_file_providers = TaskFileProviders::new(false); let providers = task_file_providers.get_providers(); assert_eq!(providers.len(), 2); } #[test] fn test_local_file_match_local_provider() { - let task_file_providers = TaskFileProviders::new(env::temp_dir()); + let task_file_providers = TaskFileProviders::new(false); let cases = vec!["file.txt", "./file.txt", "../file.txt", "/file.txt"]; for file in cases { let provider = task_file_providers.get_provider(file); assert!(provider.is_some()); - assert!(format!("{:?}", provider.unwrap()).contains("LocalTaskFileProvider")); + assert!(format!("{:?}", provider.unwrap()).contains("LocalTask")); } } #[test] - fn test_http_file_match_http_provider() { - let task_file_providers = TaskFileProviders::new(env::temp_dir()); + fn test_http_file_match_http_remote_task_provider() { + let task_file_providers = TaskFileProviders::new(false); let cases = vec![ "http://example.com/file.txt", "https://example.com/file.txt", @@ -69,7 +76,7 @@ mod tests { for file in cases { let provider = task_file_providers.get_provider(file); assert!(provider.is_some()); - assert!(format!("{:?}", provider.unwrap()).contains("HttpTaskFileProvider")); + assert!(format!("{:?}", provider.unwrap()).contains("RemoteTaskHttp")); } } } diff --git a/src/task/task_file_providers/remote_task_http.rs b/src/task/task_file_providers/remote_task_http.rs new file mode 100644 index 0000000000..1b015a4dff --- /dev/null +++ b/src/task/task_file_providers/remote_task_http.rs @@ -0,0 +1,172 @@ +use std::path::PathBuf; + +use crate::{env, file, hash, http::HTTP}; + +use super::TaskFileProvider; + +#[derive(Debug)] +pub struct RemoteTaskHttp { + cache_path: PathBuf, + no_cache: bool, +} + +impl RemoteTaskHttp { + pub fn new(cache_path: PathBuf, no_cache: bool) -> Self { + Self { + cache_path, + no_cache, + } + } +} + +impl RemoteTaskHttp { + fn get_cache_key(&self, file: &str) -> String { + hash::hash_sha256_to_str(file) + } + + fn download_file( + &self, + file: &str, + destination: &PathBuf, + ) -> Result<(), Box> { + HTTP.download_file(file, destination, None)?; + file::make_executable(destination)?; + Ok(()) + } +} + +impl TaskFileProvider for RemoteTaskHttp { + fn is_match(&self, file: &str) -> bool { + let url = url::Url::parse(file); + + // Check if the URL is valid and the scheme is http or https + // and the path is not empty + // and the path is not a directory + url.is_ok_and(|url| { + (url.scheme() == "http" || url.scheme() == "https") + && url.path().len() > 1 + && !url.path().ends_with('/') + }) + } + + fn get_local_path(&self, file: &str) -> Result> { + match self.no_cache { + false => { + trace!("Cache mode enabled"); + let cache_key = self.get_cache_key(file); + let destination = self.cache_path.join(&cache_key); + + if destination.exists() { + debug!("Using cached file: {:?}", destination); + return Ok(destination); + } + + debug!("Downloading file: {}", file); + self.download_file(file, &destination)?; + Ok(destination) + } + true => { + trace!("Cache mode disabled"); + let url = url::Url::parse(file)?; + let filename = url + .path_segments() + .and_then(|segments| segments.last()) + .unwrap(); + + let destination = env::temp_dir().join(filename); + if destination.exists() { + file::remove_file(&destination)?; + } + self.download_file(file, &destination)?; + Ok(destination) + } + } + } +} + +#[cfg(test)] +mod tests { + + use std::env; + + use super::*; + + #[test] + fn test_is_match() { + let provider = RemoteTaskHttp::new(env::temp_dir(), true); + + // Positive cases + assert!(provider.is_match("http://myhost.com/test.txt")); + assert!(provider.is_match("https://myhost.com/test.txt")); + assert!(provider.is_match("https://mydomain.com/myfile.py")); + assert!(provider.is_match("https://subdomain.mydomain.com/myfile.sh")); + assert!(provider.is_match("https://subdomain.mydomain.com/myfile.sh?query=1")); + + // Negative cases + assert!(!provider.is_match("https://myhost.com/js/")); + assert!(!provider.is_match("https://myhost.com")); + assert!(!provider.is_match("https://myhost.com/")); + } + + #[test] + fn test_http_remote_task_get_local_path_without_cache() { + let paths = vec![ + ("/myfile.py", "myfile.py"), + ("/subpath/myfile.sh", "myfile.sh"), + ("/myfile.sh?query=1&sdfsdf=2", "myfile.sh"), + ]; + let mut server = mockito::Server::new(); + + for (request_path, expected_file_name) in paths { + let mocked_server: mockito::Mock = server + .mock("GET", request_path) + .with_status(200) + .with_body("Random content") + .expect(2) + .create(); + + let provider = RemoteTaskHttp::new(env::temp_dir(), true); + let mock = format!("{}{}", server.url(), request_path); + + for _ in 0..2 { + let local_path = provider.get_local_path(&mock).unwrap(); + assert!(local_path.exists()); + assert!(local_path.is_file()); + assert!(local_path.ends_with(expected_file_name)); + } + + mocked_server.assert(); + } + } + + #[test] + fn test_http_remote_task_get_local_path_with_cache() { + let paths = vec![ + ("/myfile.py", "myfile.py"), + ("/subpath/myfile.sh", "myfile.sh"), + ("/myfile.sh?query=1&sdfsdf=2", "myfile.sh"), + ]; + let mut server = mockito::Server::new(); + + for (request_path, not_expected_file_name) in paths { + let mocked_server = server + .mock("GET", request_path) + .with_status(200) + .with_body("Random content") + .expect(1) + .create(); + + let provider = RemoteTaskHttp::new(env::temp_dir(), false); + let mock = format!("{}{}", server.url(), request_path); + + for _ in 0..2 { + let path = provider.get_local_path(&mock).unwrap(); + assert!(path.exists()); + assert!(path.is_file()); + assert!(!path.ends_with(not_expected_file_name)); + } + + mocked_server.assert(); + } + } +} diff --git a/xtasks/fig/src/mise.ts b/xtasks/fig/src/mise.ts index 78f8488aca..0cdb0c8e88 100644 --- a/xtasks/fig/src/mise.ts +++ b/xtasks/fig/src/mise.ts @@ -1762,6 +1762,10 @@ const completionSpec: Fig.Spec = { name: "output", }, }, + { + name: "--no-cache", + isRepeatable: false, + }, ], generateSpec: usageGenerateSpec(["mise tasks --usage"]), cache: false, @@ -2402,6 +2406,10 @@ const completionSpec: Fig.Spec = { name: "output", }, }, + { + name: "--no-cache", + isRepeatable: false, + }, ], args: [ {