diff --git a/src/actions.rs b/src/actions.rs index 5beb952..808d5d6 100644 --- a/src/actions.rs +++ b/src/actions.rs @@ -1,5 +1,4 @@ use std::{ - collections::HashMap, io::Read, path::Path, process::{Command, Stdio}, @@ -8,11 +7,9 @@ use std::{ time::Instant, }; -use serde::Deserialize; - use crate::{ + config::ActionConfig, errors::GitOpsError, - opts::CliOptions, receiver::{SourceType, WorkloadEvent}, utils::POLL_INTERVAL, }; @@ -23,60 +20,35 @@ pub enum ActionResult { Failure, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone)] pub struct Action { - name: String, - entrypoint: String, - #[serde(default)] - args: Vec, - #[serde(default)] - environment: HashMap, - #[serde(default)] - inherit_environment: bool, + config: ActionConfig, } impl Action { - pub fn id(&self) -> String { - self.name.clone() + pub fn new(config: ActionConfig) -> Self { + Action { config } } - pub fn set_env(&mut self, key: String, val: String) { - self.environment.insert(key, val); + pub fn id(&self) -> String { + self.config.name.clone() } -} - -impl TryFrom<&CliOptions> for Action { - type Error = GitOpsError; - fn try_from(opts: &CliOptions) -> Result { - let mut environment = HashMap::new(); - for env in &opts.environment { - let (key, val) = env - .split_once('=') - .ok_or_else(|| GitOpsError::InvalidEnvVar(env.clone()))?; - environment.insert(key.to_owned(), val.to_owned()); - } - Ok(Self { - name: opts.action.clone().unwrap(), - // TODO --action won't work on Windows - entrypoint: "/bin/sh".to_string(), - args: vec!["-c".to_string(), opts.action.clone().unwrap()], - environment, - inherit_environment: false, - }) + pub fn set_env(&mut self, key: String, val: String) { + self.config.environment.insert(key, val); } } -fn build_command(action: &Action, cwd: &Path) -> Command { - let mut command = Command::new(action.entrypoint.clone()); - command.args(action.args.clone()); - if !action.inherit_environment { +fn build_command(config: &ActionConfig, cwd: &Path) -> Command { + let mut command = Command::new(config.entrypoint.clone()); + command.args(config.args.clone()); + if !config.inherit_environment { command.env_clear(); if let Ok(path) = std::env::var("PATH") { command.env("PATH", path); } } - command.envs(action.environment.iter()); + command.envs(config.environment.iter()); command.current_dir(cwd); command.stdout(Stdio::piped()); command.stderr(Stdio::piped()); @@ -121,7 +93,7 @@ pub fn run_action( where F: Fn(WorkloadEvent) -> Result<(), GitOpsError> + Send + 'static, { - let mut command = build_command(action, cwd); + let mut command = build_command(&action.config, cwd); let mut child = command.spawn().map_err(GitOpsError::ActionError)?; let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); @@ -152,6 +124,7 @@ where #[cfg(test)] mod tests { use std::{ + collections::HashMap, process::ExitStatus, sync::{Arc, Mutex}, time::Duration, @@ -162,11 +135,13 @@ mod tests { fn shell_action(cmd: &str) -> Action { Action { - name: "test".to_owned(), - entrypoint: "/bin/sh".to_owned(), - args: vec!["-c".to_owned(), cmd.to_owned()], - environment: HashMap::new(), - inherit_environment: false, + config: ActionConfig { + name: "test".to_owned(), + entrypoint: "/bin/sh".to_owned(), + args: vec!["-c".to_owned(), cmd.to_owned()], + environment: HashMap::new(), + inherit_environment: false, + }, } } diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..e2223a9 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,256 @@ +use std::{collections::HashMap, io::Read, path::PathBuf, time::Duration}; + +use gix::Url; +use serde::{Deserialize, Deserializer}; + +use crate::{errors::GitOpsError, opts::CliOptions}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ConfigFile { + pub tasks: Vec, +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GitTaskConfig { + pub name: String, + pub github: Option, + pub git: GitConfig, + pub actions: Vec, + #[serde( + default = "GitTaskConfig::default_interval", + deserialize_with = "human_readable_duration" + )] + pub interval: Duration, + #[serde( + default = "GitTaskConfig::default_timeout", + deserialize_with = "human_readable_duration" + )] + pub timeout: Duration, +} + +impl GitTaskConfig { + pub fn default_interval() -> Duration { + Duration::from_secs(60) + } + + pub fn default_timeout() -> Duration { + Duration::from_secs(3600) + } +} + +impl TryFrom<&CliOptions> for GitTaskConfig { + type Error = GitOpsError; + + fn try_from(opts: &CliOptions) -> Result { + let url = Url::try_from(opts.url.clone().unwrap()).map_err(GitOpsError::InvalidUrl)?; + let action: ActionConfig = TryFrom::try_from(opts)?; + Ok(Self { + name: url.path.to_string(), + github: TryFrom::try_from(opts)?, + git: TryFrom::try_from(opts)?, + actions: vec![action], + interval: opts.interval.unwrap_or(Self::default_interval()), + timeout: opts.timeout.unwrap_or(Self::default_timeout()), + }) + } +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GithubConfig { + pub app_id: String, + pub private_key_file: PathBuf, + #[serde(default = "GithubConfig::default_context")] + pub status_context: Option, +} + +impl GithubConfig { + pub fn default_context() -> Option { + Some("kitops".to_owned()) + } +} + +impl TryFrom<&CliOptions> for Option { + type Error = GitOpsError; + + fn try_from(opts: &CliOptions) -> Result { + match (&opts.github_app_id, &opts.github_private_key_file) { + (None, None) => Ok(None), + (Some(app_id), Some(private_key_file)) => Ok(Some(GithubConfig { + app_id: app_id.clone(), + private_key_file: private_key_file.clone(), + status_context: opts.github_status_context.clone(), + })), + _ => Err(GitOpsError::InvalidNotifyConfig), + } + } +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GitConfig { + #[serde(deserialize_with = "url_from_string")] + pub url: Url, + #[serde(default = "GitConfig::default_branch")] + pub branch: String, +} + +impl GitConfig { + pub fn default_branch() -> String { + "main".to_owned() + } +} + +impl TryFrom<&CliOptions> for GitConfig { + type Error = GitOpsError; + + fn try_from(opts: &CliOptions) -> Result { + let url = Url::try_from(opts.url.clone().unwrap()).map_err(GitOpsError::InvalidUrl)?; + Ok(GitConfig { + url, + branch: opts.branch.clone(), + }) + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ActionConfig { + pub name: String, + pub entrypoint: String, + #[serde(default)] + pub args: Vec, + #[serde(default)] + pub environment: HashMap, + #[serde(default)] + pub inherit_environment: bool, +} + +impl TryFrom<&CliOptions> for ActionConfig { + type Error = GitOpsError; + + fn try_from(opts: &CliOptions) -> Result { + let mut environment = HashMap::new(); + for env in &opts.environment { + let (key, val) = env + .split_once('=') + .ok_or_else(|| GitOpsError::InvalidEnvVar(env.clone()))?; + environment.insert(key.to_owned(), val.to_owned()); + } + Ok(ActionConfig { + name: opts.action.clone().unwrap(), + // TODO --action won't work on Windows + entrypoint: "/bin/sh".to_string(), + args: vec!["-c".to_string(), opts.action.clone().unwrap()], + environment, + inherit_environment: false, + }) + } +} + +fn human_readable_duration<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s: String = Deserialize::deserialize(deserializer)?; + humantime::parse_duration(&s).map_err(serde::de::Error::custom) +} + +fn url_from_string<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s: String = Deserialize::deserialize(deserializer)?; + Url::try_from(s).map_err(serde::de::Error::custom) +} + +pub fn read_config(reader: impl Read) -> Result { + serde_yaml::from_reader(reader).map_err(GitOpsError::MalformedConfig) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use crate::{config::GitTaskConfig, errors::GitOpsError}; + + use super::read_config; + + #[test] + fn minimum_config() { + let config = r#"tasks: + - name: testo + git: + url: https://github.com/bittrance/kitops + actions: + - name: list files + entrypoint: /bin/ls +"#; + read_config(config.as_bytes()).unwrap(); + } + + #[test] + fn fail_on_unknown_git_config() { + let config = r#"tasks: + - name: testo + git: + url: https://github.com/bittrance/kitops + non: sense + actions: + - name: list files + entrypoint: /bin/ls +"#; + assert!(matches!( + read_config(config.as_bytes()), + Err(GitOpsError::MalformedConfig(_)) + )); + } + + #[test] + fn fail_on_unknown_action_config() { + let config = r#"tasks: + - name: testo + git: + url: https://github.com/bittrance/kitops + actions: + - name: list files + non: sense + entrypoint: /bin/ls +"#; + assert!(matches!( + read_config(config.as_bytes()), + Err(GitOpsError::MalformedConfig(_)) + )); + } + + #[test] + fn action_environment_config() { + let config = r#"tasks: + - name: testo + git: + url: https://github.com/bittrance/kitops + actions: + - name: list files + entrypoint: /bin/ls + environment: + FOO: bar +"#; + read_config(config.as_bytes()).unwrap(); + } + + #[test] + fn parse_gittaskconfig() { + let raw_config = r#"name: testo +git: + url: https://github.com/bittrance/kitops +timeout: 3s +interval: 1m 2s +actions: [] +"#; + let config = serde_yaml::from_str::(raw_config).unwrap(); + assert_eq!(config.timeout, Duration::from_secs(3)); + assert_eq!(config.interval, Duration::from_secs(62)); + } +} diff --git a/src/errors.rs b/src/errors.rs index 6fb1c18..e39144f 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -44,6 +44,8 @@ pub enum GitOpsError { NotifyError(String), #[error("Failed to launch action: {0}")] ActionError(std::io::Error), + #[error("Auth only on HTTPS URLs: {0}")] + GitHubAuthNonHttpsUrl(String), #[error("Missing private key file: {0}")] GitHubMissingPrivateKeyFile(std::io::Error), #[error("Malformed private RS256 key: {0}")] diff --git a/src/task/github.rs b/src/github.rs similarity index 80% rename from src/task/github.rs rename to src/github.rs index afc12fa..5d0bde1 100644 --- a/src/task/github.rs +++ b/src/github.rs @@ -5,46 +5,16 @@ use std::{ time::Duration, }; -use gix::{ObjectId, Url}; +use gix::{url::Scheme, ObjectId, Url}; use jwt_simple::prelude::{Claims, RS256KeyPair, RSAKeyPairLike}; use reqwest::{ blocking::ClientBuilder, header::{ACCEPT, AUTHORIZATION, USER_AGENT}, }; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use serde_json::Value; -use crate::{errors::GitOpsError, git::UrlProvider, opts::CliOptions, receiver::WorkloadEvent}; - -#[derive(Clone, Deserialize)] -pub struct GithubConfig { - app_id: String, - private_key_file: PathBuf, - #[serde(default = "GithubConfig::default_context")] - pub status_context: Option, -} - -impl GithubConfig { - pub fn default_context() -> Option { - Some("kitops".to_owned()) - } -} - -impl TryFrom<&CliOptions> for Option { - type Error = GitOpsError; - - fn try_from(opts: &CliOptions) -> Result { - match (&opts.github_app_id, &opts.github_private_key_file) { - (None, None) => Ok(None), - (Some(app_id), Some(private_key_file)) => Ok(Some(GithubConfig { - app_id: app_id.clone(), - private_key_file: private_key_file.clone(), - status_context: opts.github_status_context.clone(), - })), - _ => Err(GitOpsError::InvalidNotifyConfig), - } - } -} +use crate::{config::GithubConfig, errors::GitOpsError, gix::UrlProvider, receiver::WorkloadEvent}; #[derive(Clone)] pub struct GithubUrlProvider { @@ -73,21 +43,20 @@ impl UrlProvider for GithubUrlProvider { } fn auth_url(&self) -> Result { + if self.url.scheme != Scheme::Https { + let mut buf = Vec::new(); + self.url.write_to(&mut buf).unwrap(); + let url_str = String::from_utf8(buf).unwrap_or_else(|_| "".to_owned()); + return Err(GitOpsError::GitHubAuthNonHttpsUrl(url_str)); + } let client = http_client(); let jwt_token = generate_jwt(&self.app_id, &self.private_key_file)?; let installation_id = get_installation_id(&self.repo_slug(), &client, &jwt_token)?; let access_token = get_access_token(installation_id, &client, &jwt_token)?; - // TODO Newer version of gix-url has set_username/set_password - Ok(Url::from_parts( - self.url.scheme.clone(), - Some("x-access-token".to_owned()), - Some(access_token), - self.url.host().map(str::to_owned), - self.url.port, - self.url.path.clone(), - false, - ) - .unwrap()) + let mut auth_url = self.url.clone(); + auth_url.set_user(Some("x-access-token".to_owned())); + auth_url.set_password(Some(access_token)); + Ok(auth_url) } } @@ -190,7 +159,6 @@ pub fn update_commit_status( status: GitHubStatus, message: &str, ) -> Result<(), GitOpsError> { - let config = config.clone(); let client = http_client(); let jwt_token = generate_jwt(&config.app_id, &config.private_key_file)?; let installation_id = get_installation_id(repo_slug, &client, &jwt_token)?; @@ -202,7 +170,7 @@ pub fn update_commit_status( ); let body = serde_json::json!({ "state": status, - "context": config.status_context.unwrap(), + "context": config.status_context, "description": message, }); let res = client @@ -271,3 +239,36 @@ pub fn github_watcher( Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn github_url_provider_slug() { + let url = Url::try_from("https://github.com/bittrance/kitops.git".to_owned()).unwrap(); + let config = GithubConfig { + app_id: "1234".to_owned(), + private_key_file: PathBuf::from("ze-key"), + status_context: Some("ze-context".to_owned()), + }; + let provider = GithubUrlProvider::new(url, &config); + assert_eq!(provider.repo_slug(), "bittrance/kitops"); + } + + #[test] + fn github_url_provider_refuses_http_on_auth() { + let url = Url::try_from("http://some.site/bittrance/kitops".to_owned()).unwrap(); + let config = GithubConfig { + app_id: "1234".to_owned(), + private_key_file: PathBuf::from("ze-key"), + status_context: Some("ze-context".to_owned()), + }; + let provider = GithubUrlProvider::new(url, &config); + assert!(matches!( + provider.auth_url(), + Err(GitOpsError::GitHubAuthNonHttpsUrl(_)) + )); + } +} diff --git a/src/git.rs b/src/gix.rs similarity index 64% rename from src/git.rs rename to src/gix.rs index f7fa6ac..06e4aa2 100644 --- a/src/git.rs +++ b/src/gix.rs @@ -23,45 +23,8 @@ use gix::{ remote::{fetch::Outcome, ref_map::Options, Direction}, ObjectId, Repository, Url, }; -use serde::{Deserialize, Deserializer}; -use crate::{errors::GitOpsError, opts::CliOptions, utils::Watchdog}; - -#[derive(Clone, Deserialize)] -pub struct GitConfig { - #[serde(deserialize_with = "url_from_string")] - pub url: Arc>, - #[serde(default = "GitConfig::default_branch")] - branch: String, -} - -impl GitConfig { - pub fn default_branch() -> String { - "main".to_owned() - } -} - -impl TryFrom<&CliOptions> for GitConfig { - type Error = GitOpsError; - - fn try_from(opts: &CliOptions) -> Result { - let url = Url::try_from(opts.url.clone().unwrap()).map_err(GitOpsError::InvalidUrl)?; - Ok(GitConfig { - url: Arc::new(Box::new(DefaultUrlProvider { url })), - branch: opts.branch.clone(), - }) - } -} - -fn url_from_string<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - let s: String = Deserialize::deserialize(deserializer)?; - Ok(Arc::new(Box::new(DefaultUrlProvider { - url: Url::try_from(s).map_err(serde::de::Error::custom)?, - }))) -} +use crate::{errors::GitOpsError, utils::Watchdog}; pub trait UrlProvider: Send + Sync { fn url(&self) -> &Url; @@ -78,6 +41,12 @@ pub struct DefaultUrlProvider { url: Url, } +impl DefaultUrlProvider { + pub fn new(url: Url) -> Self { + DefaultUrlProvider { url } + } +} + impl UrlProvider for DefaultUrlProvider { fn url(&self) -> &Url { &self.url @@ -88,24 +57,19 @@ impl UrlProvider for DefaultUrlProvider { } } -fn clone_repo( - config: &GitConfig, - deadline: Instant, - target: &Path, -) -> Result { +// TODO What about branch?! +fn clone_repo(url: Url, deadline: Instant, target: &Path) -> Result { let watchdog = Watchdog::new(deadline); scope(|s| { s.spawn(watchdog.runner()); - let maybe_repo = config.url.auth_url().and_then(|url| { - gix::prepare_clone(url, target) - .unwrap() - .with_in_memory_config_overrides(vec![gitoxide::Credentials::TERMINAL_PROMPT - .validated_assignment_fmt(&false) - .unwrap()]) - .fetch_only(Discard, &watchdog) - .map(|(r, _)| r) - .map_err(GitOpsError::InitRepo) - }); + let maybe_repo = gix::prepare_clone(url, target) + .unwrap() + .with_in_memory_config_overrides(vec![gitoxide::Credentials::TERMINAL_PROMPT + .validated_assignment_fmt(&false) + .unwrap()]) + .fetch_only(Discard, &watchdog) + .map(|(r, _)| r) + .map_err(GitOpsError::InitRepo); watchdog.cancel(); maybe_repo }) @@ -113,12 +77,13 @@ fn clone_repo( fn perform_fetch( repo: &Repository, - config: &GitConfig, + url: Url, + branch: &str, cancel: &AtomicBool, ) -> Result> { - repo.remote_at(config.url.auth_url()?) + repo.remote_at(url) .unwrap() - .with_refspecs([BString::from(config.branch.clone())], Direction::Fetch) + .with_refspecs([BString::from(branch)], Direction::Fetch) .unwrap() .connect(Direction::Fetch)? .prepare_fetch(Discard, Options::default())? @@ -126,15 +91,20 @@ fn perform_fetch( .map_err(Into::into) } -fn fetch_repo(repo: &Repository, config: &GitConfig, deadline: Instant) -> Result<(), GitOpsError> { +fn fetch_repo( + repo: &Repository, + url: Url, + branch: &str, + deadline: Instant, +) -> Result<(), GitOpsError> { let watchdog = Watchdog::new(deadline); let outcome = scope(|s| { s.spawn(watchdog.runner()); - let outcome = perform_fetch(repo, config, &watchdog).map_err(GitOpsError::FetchError); + let outcome = perform_fetch(repo, url, branch, &watchdog).map_err(GitOpsError::FetchError); watchdog.cancel(); outcome })?; - let needle = BString::from("refs/heads/".to_owned() + &config.branch); + let needle = BString::from("refs/heads/".to_owned() + branch); let target = outcome .ref_map .remote_refs @@ -230,7 +200,8 @@ fn checkout_worktree( } pub fn ensure_worktree( - config: &GitConfig, + url: Url, + branch: &str, deadline: Instant, repodir: P, workdir: Q, @@ -251,48 +222,35 @@ where .set_value(&Credentials::TERMINAL_PROMPT, "false") .unwrap(); gitconfig.commit().unwrap(); - fetch_repo(&repo, config, deadline)?; + fetch_repo(&repo, url, branch, deadline)?; repo } else { - clone_repo(config, deadline, repodir)? + clone_repo(url, deadline, repodir)? }; - checkout_worktree(&repo, &config.branch, workdir) + checkout_worktree(&repo, branch, workdir) } #[cfg(test)] mod tests { - use std::{ - sync::Arc, - time::{Duration, Instant}, - }; + use std::time::{Duration, Instant}; - use crate::{ - errors::GitOpsError, - git::{clone_repo, fetch_repo, GitConfig}, - testutils::TestUrl, - }; + use crate::gix::{clone_repo, fetch_repo}; + + const TEST_URL: &str = "https://example.com"; #[test] fn clone_with_bad_url() { - let config = GitConfig { - url: Arc::new(Box::new(TestUrl::new(Some(GitOpsError::TestError)))), - branch: "main".into(), - }; let deadline = Instant::now() + Duration::from_secs(61); // Fail tests that time out let target = tempfile::tempdir().unwrap(); - let result = clone_repo(&config, deadline, target.path()); - assert!(matches!(result, Err(GitOpsError::TestError))); + let result = clone_repo(TEST_URL.try_into().unwrap(), deadline, target.path()); + assert!(result.is_err()); } #[test] fn fetch_with_bad_url() { let repo = gix::open(".").unwrap(); - let config = GitConfig { - url: Arc::new(Box::new(TestUrl::new(Some(GitOpsError::TestError)))), - branch: "main".into(), - }; let deadline = Instant::now() + Duration::from_secs(61); // Fail tests that time out - let result = fetch_repo(&repo, &config, deadline); + let result = fetch_repo(&repo, TEST_URL.try_into().unwrap(), "main", deadline); assert!(result.is_err()); } } diff --git a/src/lib.rs b/src/lib.rs index 10f6842..8a87327 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,17 +1,21 @@ use std::{thread::sleep, time::Duration}; -use task::{scheduled::ScheduledTask, Workload}; +use crate::{task::ScheduledTask, workload::Workload}; pub mod actions; +pub mod config; pub mod errors; -pub mod git; +pub mod github; +pub mod gix; pub mod opts; pub mod receiver; +pub mod state; pub mod store; pub mod task; #[cfg(test)] pub(crate) mod testutils; pub(crate) mod utils; +pub mod workload; #[derive(Debug, PartialEq)] pub enum Progress { @@ -75,11 +79,7 @@ mod lib { use gix::{hash::Kind, ObjectId}; - use crate::{ - errors::GitOpsError, - task::{scheduled::ScheduledTask, State}, - testutils::TestWorkload, - }; + use crate::{errors::GitOpsError, state::State, task::ScheduledTask, testutils::TestWorkload}; #[test] fn run_eligible_task() { diff --git a/src/main.rs b/src/main.rs index 206f426..af00cc1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,8 +5,8 @@ use kitops::errors::GitOpsError; use kitops::opts::{load_store, load_tasks, CliOptions}; use kitops::run_tasks; use kitops::store::Store; -use kitops::task::gixworkload::GitWorkload; -use kitops::task::scheduled::ScheduledTask; +use kitops::task::ScheduledTask; +use kitops::workload::GitWorkload; use std::collections::HashSet; use std::time::Duration; diff --git a/src/opts.rs b/src/opts.rs index 4f46859..5d1944a 100644 --- a/src/opts.rs +++ b/src/opts.rs @@ -1,18 +1,16 @@ use std::{fs::File, path::PathBuf, sync::mpsc::channel, thread::spawn, time::Duration}; use clap::Parser; -use serde::Deserialize; use crate::{ + config::{read_config, GitTaskConfig}, errors::GitOpsError, + github::{github_watcher, GithubUrlProvider}, + gix::DefaultUrlProvider, receiver::logging_receiver, store::{FileStore, Store}, - task::{ - github::{github_watcher, GithubUrlProvider}, - gixworkload::GitWorkload, - scheduled::ScheduledTask, - GitTaskConfig, - }, + task::ScheduledTask, + workload::GitWorkload, }; const DEFAULT_BRANCH: &str = "main"; @@ -91,25 +89,21 @@ impl CliOptions { } } -#[derive(Deserialize)] -struct ConfigFile { - tasks: Vec, -} - fn into_task(mut config: GitTaskConfig, opts: &CliOptions) -> ScheduledTask { + let repo_dir = opts.repo_dir.clone().unwrap(); let github = config.github.take(); - let mut slug = None; // TODO Yuck! - if let Some(ref github) = github { - let provider = GithubUrlProvider::new(config.git.url.url().clone(), github); - slug = Some(provider.repo_slug()); - config.upgrade_url_provider(|_| provider); - } - let mut work = GitWorkload::from_config(config, opts); - if let Some(github) = github { + let mut work = if let Some(github) = github { + let provider = GithubUrlProvider::new(config.git.url.clone(), &github); + let slug = Some(provider.repo_slug()); + let mut work = GitWorkload::new(config, provider, &repo_dir); if github.status_context.is_some() { work.watch(github_watcher(slug.unwrap(), github)); } - } + work + } else { + let provider = DefaultUrlProvider::new(config.git.url.clone()); + GitWorkload::new(config, provider, &repo_dir) + }; let (tx, rx) = channel(); work.watch(move |event| { tx.send(event) @@ -125,8 +119,7 @@ fn into_task(mut config: GitTaskConfig, opts: &CliOptions) -> ScheduledTask Result>, GitOpsError> { let config = File::open(opts.config_file.clone().unwrap()).map_err(GitOpsError::MissingConfig)?; - let config_file: ConfigFile = - serde_yaml::from_reader(config).map_err(GitOpsError::MalformedConfig)?; + let config_file = read_config(config)?; Ok(config_file .tasks .into_iter() @@ -177,16 +170,3 @@ fn complete_cli_options_conflicting_args() { let res = opts.complete(); assert!(matches!(res, Err(GitOpsError::ConfigMethodConflict))); } - -#[test] -fn minimum_config() { - let config = r#"tasks: - - name: testo - git: - url: https://github.com/bittrance/kitops - actions: - - name: list files - entrypoint: /bin/ls -"#; - serde_yaml::from_str::(config).unwrap(); -} diff --git a/src/state.rs b/src/state.rs new file mode 100644 index 0000000..bbc1154 --- /dev/null +++ b/src/state.rs @@ -0,0 +1,19 @@ +use std::time::SystemTime; + +use gix::{hash::Kind, ObjectId}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct State { + pub next_run: SystemTime, + pub current_sha: ObjectId, +} + +impl Default for State { + fn default() -> Self { + Self { + current_sha: ObjectId::null(Kind::Sha1), + next_run: SystemTime::now(), + } + } +} diff --git a/src/store.rs b/src/store.rs index 1879c69..81be62f 100644 --- a/src/store.rs +++ b/src/store.rs @@ -4,10 +4,7 @@ use std::{ path::{Path, PathBuf}, }; -use crate::{ - errors::GitOpsError, - task::{scheduled::ScheduledTask, State, Workload}, -}; +use crate::{errors::GitOpsError, state::State, task::ScheduledTask, workload::Workload}; pub trait Store { fn get(&self, id: &str) -> Option<&State>; diff --git a/src/task/scheduled.rs b/src/task.rs similarity index 96% rename from src/task/scheduled.rs rename to src/task.rs index c6a97f1..25e97cf 100644 --- a/src/task/scheduled.rs +++ b/src/task.rs @@ -6,9 +6,7 @@ use std::{ use gix::ObjectId; -use crate::errors::GitOpsError; - -use super::{State, Workload}; +use crate::{errors::GitOpsError, state::State, workload::Workload}; pub struct ScheduledTask { work: W, @@ -89,10 +87,7 @@ mod tests { use gix::ObjectId; - use crate::{ - task::{scheduled::ScheduledTask, State}, - testutils::TestWorkload, - }; + use crate::{state::State, task::ScheduledTask, testutils::TestWorkload}; #[test] fn scheduled_task_flow() { diff --git a/src/task/mod.rs b/src/task/mod.rs deleted file mode 100644 index e7ba226..0000000 --- a/src/task/mod.rs +++ /dev/null @@ -1,129 +0,0 @@ -use std::{ - path::PathBuf, - sync::Arc, - time::{Duration, SystemTime}, -}; - -use gix::{hash::Kind, ObjectId, Url}; -use serde::{Deserialize, Deserializer, Serialize}; - -use crate::{ - actions::Action, - errors::GitOpsError, - git::{GitConfig, UrlProvider}, - opts::CliOptions, -}; - -pub mod github; -pub mod gixworkload; -pub mod scheduled; - -pub trait Workload { - fn id(&self) -> String; - fn interval(&self) -> Duration; - fn perform(self, workdir: PathBuf, current_sha: ObjectId) -> Result; -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct State { - pub next_run: SystemTime, - pub current_sha: ObjectId, -} - -impl Default for State { - fn default() -> Self { - Self { - current_sha: ObjectId::null(Kind::Sha1), - next_run: SystemTime::now(), - } - } -} - -fn human_readable_duration<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let s: String = Deserialize::deserialize(deserializer)?; - humantime::parse_duration(&s).map_err(serde::de::Error::custom) -} - -#[derive(Clone, Deserialize)] -pub struct GitTaskConfig { - name: String, - pub github: Option, - pub git: GitConfig, - actions: Vec, - #[serde( - default = "GitTaskConfig::default_interval", - deserialize_with = "human_readable_duration" - )] - interval: Duration, - #[serde( - default = "GitTaskConfig::default_timeout", - deserialize_with = "human_readable_duration" - )] - timeout: Duration, -} - -impl GitTaskConfig { - pub fn upgrade_url_provider(&mut self, upgrader: U) - where - U: FnOnce(&Arc>) -> Q, - Q: UrlProvider + 'static, - { - let new_provider = upgrader(&self.git.url); - self.git.url = Arc::new(Box::new(new_provider)); - } - - pub fn add_action(&mut self, action: Action) { - self.actions.push(action); - } -} - -impl TryFrom<&CliOptions> for GitTaskConfig { - type Error = GitOpsError; - - fn try_from(opts: &CliOptions) -> Result { - let url = Url::try_from(opts.url.clone().unwrap()).map_err(GitOpsError::InvalidUrl)?; - let action: Action = TryFrom::try_from(opts)?; - Ok(Self { - name: url.path.to_string(), - github: TryFrom::try_from(opts)?, - git: TryFrom::try_from(opts)?, - actions: vec![action], - interval: opts.interval.unwrap_or(Self::default_interval()), - timeout: opts.timeout.unwrap_or(Self::default_timeout()), - }) - } -} - -impl GitTaskConfig { - pub fn default_interval() -> Duration { - Duration::from_secs(60) - } - - pub fn default_timeout() -> Duration { - Duration::from_secs(3600) - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use super::GitTaskConfig; - - #[test] - fn parse_gittaskconfig() { - let raw_config = r#"name: testo -git: - url: https://github.com/bittrance/kitops -timeout: 3s -interval: 1m 2s -actions: [] - "#; - let config = serde_yaml::from_str::(raw_config).unwrap(); - assert_eq!(config.timeout, Duration::from_secs(3)); - assert_eq!(config.interval, Duration::from_secs(62)); - } -} diff --git a/src/testutils.rs b/src/testutils.rs index 08ba587..3dcff37 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -1,17 +1,8 @@ -use std::{ - path::PathBuf, - sync::{Arc, Mutex}, - thread::sleep, - time::Duration, -}; +use std::{path::PathBuf, sync::Arc, thread::sleep, time::Duration}; -use gix::{ObjectId, Url}; +use gix::ObjectId; -use crate::{ - errors::GitOpsError, - git::UrlProvider, - task::{scheduled::ScheduledTask, Workload}, -}; +use crate::{errors::GitOpsError, task::ScheduledTask, workload::Workload}; impl ScheduledTask { pub fn await_finished(&self) { @@ -27,35 +18,6 @@ impl ScheduledTask { } } -pub struct TestUrl { - url: Url, - auth_url_error: Mutex>, -} - -impl TestUrl { - pub fn new(auth_url_error: Option) -> Self { - let url = gix::url::parse("https://example.com".into()).unwrap(); - TestUrl { - url, - auth_url_error: Mutex::new(auth_url_error), - } - } -} - -impl UrlProvider for TestUrl { - fn url(&self) -> &Url { - &self.url - } - - fn auth_url(&self) -> Result { - if let Some(err) = self.auth_url_error.lock().unwrap().take() { - Err(err) - } else { - Ok(self.url().clone()) - } - } -} - #[derive(Clone, Default)] pub struct TestWorkload { errfunc: Option GitOpsError + Send + Sync>>>, diff --git a/src/task/gixworkload.rs b/src/workload.rs similarity index 77% rename from src/task/gixworkload.rs rename to src/workload.rs index 0e12875..a638cad 100644 --- a/src/task/gixworkload.rs +++ b/src/workload.rs @@ -7,33 +7,46 @@ use std::{ use gix::ObjectId; use crate::{ - actions::{run_action, ActionResult}, + actions::{run_action, Action, ActionResult}, + config::GitTaskConfig, errors::GitOpsError, - git::ensure_worktree, - opts::CliOptions, + gix::{ensure_worktree, UrlProvider}, receiver::WorkloadEvent, }; -use super::{GitTaskConfig, Workload}; +pub trait Workload { + fn id(&self) -> String; + fn interval(&self) -> Duration; + fn perform(self, workdir: PathBuf, current_sha: ObjectId) -> Result; +} #[allow(clippy::type_complexity)] #[derive(Clone)] pub struct GitWorkload { + actions: Vec, config: GitTaskConfig, + url_provider: Arc>, repo_dir: PathBuf, watchers: Vec Result<(), GitOpsError> + Send + 'static>>>>, } impl GitWorkload { - pub fn from_config(config: GitTaskConfig, opts: &CliOptions) -> Self { - let repo_dir = opts - .repo_dir - .as_ref() - .map(|dir| dir.join(config.git.url.safe_url())) - .unwrap(); + pub fn new( + config: GitTaskConfig, + url_provider: impl UrlProvider + 'static, + repo_dir: &Path, + ) -> Self { + let repo_dir = repo_dir.join(url_provider.safe_url()); + let actions = config + .actions + .iter() + .map(|config| Action::new(config.clone())) + .collect(); GitWorkload { + actions, config, + url_provider: Arc::new(Box::new(url_provider)), repo_dir, watchers: Vec::new(), } @@ -52,7 +65,7 @@ impl GitWorkload { deadline: Instant, sink: &Arc Result<(), GitOpsError> + Send + 'static>>, ) -> Result, GitOpsError> { - for action in &self.config.actions { + for action in &self.actions { let name = format!("{}|{}", self.config.name, action.id()); let res = run_action(&name, action, workdir, deadline, sink)?; if res != ActionResult::Success { @@ -81,9 +94,11 @@ impl Workload for GitWorkload { } Ok::<_, GitOpsError>(()) })); - let new_sha = ensure_worktree(&self.config.git, deadline, &self.repo_dir, &workdir)?; + let url = self.url_provider.auth_url()?; + let branch = self.config.git.branch.clone(); + let new_sha = ensure_worktree(url, &branch, deadline, &self.repo_dir, &workdir)?; if current_sha != new_sha { - self.config.actions.iter_mut().for_each(|action| { + self.actions.iter_mut().for_each(|action| { action.set_env( "KITOPS_LAST_SUCCESSFUL_SHA".to_string(), current_sha.to_string(), diff --git a/tests/git.rs b/tests/git.rs index dacab1f..31bc0c0 100644 --- a/tests/git.rs +++ b/tests/git.rs @@ -2,7 +2,7 @@ use xshell::cmd; use std::time::{Duration, Instant}; -use kitops::git::{ensure_worktree, GitConfig}; +use kitops::{config::GitConfig, gix::ensure_worktree}; use utils::{clone_repo, commit_file, empty_repo, reset_branch, shell, TEST_CONFIG}; @@ -15,7 +15,7 @@ fn clone_repo_from_github_https() { let deadline = Instant::now() + Duration::from_secs(60); let repodir = tempfile::tempdir().unwrap(); let workdir = tempfile::tempdir().unwrap(); - ensure_worktree(&config, deadline, &repodir, &workdir).unwrap(); + ensure_worktree(config.url, &config.branch, deadline, &repodir, &workdir).unwrap(); sh.change_dir(&workdir); let files = cmd!(sh, "ls").read().unwrap(); assert!(files.contains("Cargo.toml")); @@ -35,14 +35,28 @@ fn fetch_repo_from_file_url() { .unwrap(); let deadline = Instant::now() + Duration::from_secs(60); let workdir = tempfile::tempdir().unwrap(); - ensure_worktree(&config, deadline, &repodir, &workdir).unwrap(); + ensure_worktree( + config.url.clone(), + &config.branch, + deadline, + &repodir, + &workdir, + ) + .unwrap(); assert_eq!( sh.read_file(workdir.path().join("ze-file")).unwrap(), "revision 1" ); commit_file(&upstream, "revision 2"); let workdir = tempfile::tempdir().unwrap(); - ensure_worktree(&config, deadline, &repodir, &workdir).unwrap(); + ensure_worktree( + config.url.clone(), + &config.branch, + deadline, + &repodir, + &workdir, + ) + .unwrap(); assert_eq!( sh.read_file(workdir.path().join("ze-file")).unwrap(), "revision 2" @@ -62,7 +76,7 @@ fn fetch_repo_with_force_push() { .unwrap(); let deadline = Instant::now() + Duration::from_secs(60); let workdir = tempfile::tempdir().unwrap(); - ensure_worktree(&config, deadline, &repodir, &workdir).unwrap(); + ensure_worktree(config.url, &config.branch, deadline, &repodir, &workdir).unwrap(); assert_eq!( sh.read_file(workdir.path().join("ze-file")).unwrap(), "revision 1" diff --git a/tests/workload.rs b/tests/workload.rs index 207a1da..bd4fcce 100644 --- a/tests/workload.rs +++ b/tests/workload.rs @@ -1,21 +1,17 @@ use std::sync::{Arc, Mutex}; -use clap::Parser; use gix::{hash::Kind, ObjectId}; use kitops::{ + config::GitTaskConfig, errors::GitOpsError, - opts::CliOptions, + gix::DefaultUrlProvider, receiver::{SourceType, WorkloadEvent}, - task::{gixworkload::GitWorkload, GitTaskConfig, Workload}, + workload::{GitWorkload, Workload}, }; use utils::*; mod utils; -fn cli_options(repodir: &tempfile::TempDir) -> CliOptions { - CliOptions::parse_from(&["kitops", "--repo-dir", &repodir.path().to_str().unwrap()]) -} - fn config(upstream: &tempfile::TempDir, entrypoint: &str, args: &[&str]) -> GitTaskConfig { serde_yaml::from_str(&format!( r#" @@ -58,9 +54,9 @@ fn watch_successful_workload() { let repodir = tempfile::tempdir().unwrap(); let next_sha = ObjectId::from_hex(next_sha.as_bytes()).unwrap(); let workdir = tempfile::tempdir().unwrap(); - let opts = cli_options(&repodir); let config = config(&upstream, "/bin/ls", &[]); - let mut workload = GitWorkload::from_config(config, &opts); + let provider = DefaultUrlProvider::new(config.git.url.clone()); + let mut workload = GitWorkload::new(config, provider, &repodir.path()); let events = Arc::new(Mutex::new(Vec::new())); let events2 = events.clone(); workload.watch(move |event| { @@ -86,9 +82,9 @@ fn watch_failing_workload() { commit_file(&upstream, "revision 1"); let repodir = tempfile::tempdir().unwrap(); let workdir = tempfile::tempdir().unwrap(); - let opts = cli_options(&repodir); let config = config(&upstream, "/usr/bin/false", &[]); - let mut workload = GitWorkload::from_config(config, &opts); + let provider = DefaultUrlProvider::new(config.git.url.clone()); + let mut workload = GitWorkload::new(config, provider, &repodir.path()); let events = Arc::new(Mutex::new(Vec::new())); let events2 = events.clone(); workload.watch(move |event| { @@ -112,9 +108,9 @@ fn watch_erroring_workload() { commit_file(&upstream, "revision 1"); let repodir = tempfile::tempdir().unwrap(); let workdir = tempfile::tempdir().unwrap(); - let opts = cli_options(&repodir); let config = config(&upstream, "/no/such/file", &[]); - let mut workload = GitWorkload::from_config(config, &opts); + let provider = DefaultUrlProvider::new(config.git.url.clone()); + let mut workload = GitWorkload::new(config, provider, &repodir.path()); let events = Arc::new(Mutex::new(Vec::new())); let events2 = events.clone(); workload.watch(move |event| { @@ -139,9 +135,9 @@ fn woarkload_gets_sha_env() { let repodir = tempfile::tempdir().unwrap(); let next_sha = ObjectId::from_hex(next_sha.as_bytes()).unwrap(); let workdir = tempfile::tempdir().unwrap(); - let opts = cli_options(&repodir); let config = config(&upstream, "/bin/sh", &["-c", "echo $KITOPS_SHA"]); - let mut workload = GitWorkload::from_config(config, &opts); + let provider = DefaultUrlProvider::new(config.git.url.clone()); + let mut workload = GitWorkload::new(config, provider, &repodir.path()); let events = Arc::new(Mutex::new(Vec::new())); let events2 = events.clone(); workload.watch(move |event| {