From b31a04fc81286533bab97e37fac47d912abbabe8 Mon Sep 17 00:00:00 2001 From: Heulitig Date: Mon, 13 Jun 2022 17:40:08 +0530 Subject: [PATCH 01/13] shifted serve block up --- src/main.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index 6475395b..c36249b4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,17 @@ async fn main() -> fpm::Result<()> { return Ok(()); } + // Serve block moved up + if let Some(mark) = matches.subcommand_matches("serve") { + let port = mark.value_of("port").unwrap_or("8000").to_string(); + tokio::task::spawn_blocking(move || { + fpm::serve(port.as_str()).expect("http service error"); + }) + .await + .expect("Thread spawn error"); + return Ok(()); + } + let mut config = fpm::Config::read(None).await?; if matches.subcommand_matches("update").is_some() { @@ -72,14 +83,6 @@ async fn main() -> fpm::Result<()> { let target = mark.value_of("target"); fpm::stop_tracking(&config, source, target).await?; } - if let Some(mark) = matches.subcommand_matches("serve") { - let port = mark.value_of("port").unwrap_or("8000").to_string(); - tokio::task::spawn_blocking(move || { - fpm::serve(port.as_str()).expect("http service error"); - }) - .await - .expect("Thread spawn error"); - } Ok(()) } From 7b928f72cb9fa59ca9f10c35c2299a2915ee7a20 Mon Sep 17 00:00:00 2001 From: Heulitig Date: Mon, 13 Jun 2022 17:41:29 +0530 Subject: [PATCH 02/13] made http public to access it from serve.rs --- src/library/http.rs | 2 +- src/library/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library/http.rs b/src/library/http.rs index 9ca25012..44f2d98c 100644 --- a/src/library/http.rs +++ b/src/library/http.rs @@ -78,7 +78,7 @@ async fn get( } } -async fn _get(url: url::Url) -> reqwest::Result { +pub async fn _get(url: url::Url) -> reqwest::Result { let mut headers = reqwest::header::HeaderMap::new(); headers.insert( reqwest::header::USER_AGENT, diff --git a/src/library/mod.rs b/src/library/mod.rs index f4a9d564..edfc1833 100644 --- a/src/library/mod.rs +++ b/src/library/mod.rs @@ -1,7 +1,7 @@ mod fpm_dot_ftd; mod get_data; mod get_version_data; -mod http; +pub(crate) mod http; mod include; mod sitemap; mod sqlite; From 2da725333c67bdca48cb2a12662ce15248d0a057 Mon Sep 17 00:00:00 2001 From: Heulitig Date: Mon, 13 Jun 2022 17:44:24 +0530 Subject: [PATCH 03/13] added controller API functions (initial) --- src/commands/serve.rs | 154 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/src/commands/serve.rs b/src/commands/serve.rs index cbf1018c..f834fe12 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -145,6 +145,19 @@ async fn serve_static(req: actix_web::HttpRequest) -> actix_web::HttpResponse { #[actix_web::main] pub async fn serve(port: &str) -> std::io::Result<()> { + let use_controller = false; + + if use_controller { + // fpm-controller base path and ec2 instance id + let fpm_controller: String = "https:///controller.fifthtry.com".to_string(); + let fpm_instance: String = "".to_string(); + + match controller::resolve_dependencies(fpm_instance, fpm_controller).await { + Ok(_) => println!("Dependencies resolved"), + Err(_) => panic!("Error resolving dependencies using controller!!"), + } + } + println!("### Server Started ###"); println!("Go to: http://127.0.0.1:{}", port); actix_web::HttpServer::new(|| { @@ -154,3 +167,144 @@ pub async fn serve(port: &str) -> std::io::Result<()> { .run() .await } + +// FPM Controller Support +// FPM cli supports communication with fpm controller. This is an optional feature, and is only available when controller feature is enabled, which is not enabled by default. +// Controller Communication +// When controller feature is enabled, fpm serve will first communicate with the FPM controller service’s /get-package/ API. + +// FPM Controller Service Endpoint +// The FPM Controller Service’s endpoint is computed by using environment variable FPM_CONTROLLER, which will look something like this: https://controller.fifthtry.com, with the API path. +// FPM Controller Service has more than one APIs: /get-package/ and /fpm-ready/. + +// get-package: +// Through an environment variable FPM_INSTANCE_ID, the fpm serve will learn it’s instance id, and it will pass the instance id to the get-package API. +// The API returns the URL of the package to be downloaded, git repository URL and the package name. +// FPM will clone the git repository in the current directory. The current directory will contain FPM.ftd and other files of the package. +// FPM will then calls fpm install on it. + +// fpm-ready: +// Once dependencies are ready fpm calls /fpm-ready/ API on the controller. We will pass the FPM_INSTANCE_ID and the git commit hash as input to the API +// The API will return with success, and once it is done fpm will start receiving HTTP traffic from the controller service. + +mod controller { + pub async fn resolve_dependencies( + fpm_instance: String, + fpm_controller: String, + ) -> fpm::Result<()> { + // First call get_package API to get package details and resolve dependencies + // input: fpm_instance + // output: package_name and git repo URL + // format: { + // "success": true, + // "result": { + // "package": "" + // "git": "" + // } + // } + let package_response = get_package(fpm_instance.as_str(), fpm_controller.as_str()).await?; + let gp_status = match package_response["success"].as_bool() { + Some(res) => res, + None => panic!("success parameter doesn't exist in Json or isn't valid boolean type"), + }; + + if gp_status == true { + // package name and git repo url + let package_name = match package_response["result"]["package"].as_str() { + Some(valid_name) => valid_name, + None => panic!("received invalid package name from get_package API"), + }; + if let Some(git_url) = package_response["result"]["git"].as_str() { + // Clone the git package into the current directory + // Need to execute shell commands from rust + let out = std::process::Command::new("git") + .arg("clone") + .arg(git_url) + .output() + .expect("unable to execute git clone command"); + + if out.status.success() { + // By this time the cloned repo should be available in the current directory + println!("Git cloning successful for the package {}", package_name); + // Resolve dependencies by reading the FPM.ftd using config.read() + let _config = fpm::Config::read(Some(package_name.to_string())).await?; + } + } else { + panic!("Invalid git url for the package {}", package_name); + } + } else { + panic!("get-package api success status returned false!!"); + } + + // Once the dependencies are resolved for the package + // then call fpm_ready API to ensure that the controller now is ready for receiving further HTTP traffic + // input: fpm_instance, *(git commit hash) + // output: success: true/false + // format: lang: json + // { + // "success": true + // } + + // response from fpm_ready API + let ready_response = fpm_ready(fpm_instance.as_str(), fpm_controller.as_str()).await?; + let fr_status = match ready_response["success"].as_bool() { + Some(res) => res, + None => panic!("success parameter doesn't exist in Json or isn't valid boolean type"), + }; + + match fr_status { + true => println!("FPM controller ready!!"), + false => panic!("FPM controller isn't ready yet"), + } + + Ok(()) + } + + async fn make_request(url: url::Url) -> fpm::Result { + use fpm::library::http; + let response = match http::_get(url).await { + Ok(some_response) => some_response, + Err(e) => { + panic!("failed to fetch data, error: {}", e.to_string()) + } + }; + + match serde_json::from_str(response.as_str()) { + Ok(v) => Ok(v), + Err(e) => panic!( + "failed parsing json from response text, error: {}", + e.to_string() + ), + } + } + + async fn get_package( + fpm_instance: &str, + fpm_controller: &str, + ) -> fpm::Result { + let query_string = format!("instance={}", fpm_instance); + let controller_api = format!("{}/get-package?{}", fpm_controller, query_string); + + let url = match url::Url::parse(controller_api.as_str()) { + Ok(safe_url) => safe_url, + Err(_) => panic!("Invalid url for get-package api"), + }; + + make_request(url).await + } + + async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result { + // Git commit hash needs to be computed before making a call to the fpm_ready API + let git_commit = ""; + + let query_string = format!("instance={}&git-commit={}", fpm_instance, git_commit); + let controller_api = format!("{}/fpm-ready?{}", fpm_controller, query_string); + + let url = match url::Url::parse(controller_api.as_str()) { + Ok(safe_url) => safe_url, + Err(_) => panic!("Invalid url for fpm_ready api"), + }; + + make_request(url).await + } +} From 8db8892485b46c50f208daf84571962c8b7b4c3b Mon Sep 17 00:00:00 2001 From: Heulitig Date: Tue, 14 Jun 2022 12:00:01 +0530 Subject: [PATCH 04/13] minor changes, commenting in serve.rs --- src/commands/serve.rs | 108 +++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/src/commands/serve.rs b/src/commands/serve.rs index f834fe12..715e0e9a 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -145,10 +145,11 @@ async fn serve_static(req: actix_web::HttpRequest) -> actix_web::HttpResponse { #[actix_web::main] pub async fn serve(port: &str) -> std::io::Result<()> { + // For debugging (will be refactored later) let use_controller = false; if use_controller { - // fpm-controller base path and ec2 instance id + // fpm-controller base path and ec2 instance id (hardcoded for now) let fpm_controller: String = "https:///controller.fifthtry.com".to_string(); let fpm_instance: String = "".to_string(); @@ -193,57 +194,47 @@ mod controller { fpm_controller: String, ) -> fpm::Result<()> { // First call get_package API to get package details and resolve dependencies - // input: fpm_instance - // output: package_name and git repo URL - // format: { - // "success": true, - // "result": { - // "package": "" - // "git": "" - // } - // } + + // response from get-package API let package_response = get_package(fpm_instance.as_str(), fpm_controller.as_str()).await?; let gp_status = match package_response["success"].as_bool() { Some(res) => res, None => panic!("success parameter doesn't exist in Json or isn't valid boolean type"), }; - if gp_status == true { - // package name and git repo url - let package_name = match package_response["result"]["package"].as_str() { - Some(valid_name) => valid_name, - None => panic!("received invalid package name from get_package API"), - }; - if let Some(git_url) = package_response["result"]["git"].as_str() { - // Clone the git package into the current directory - // Need to execute shell commands from rust - let out = std::process::Command::new("git") - .arg("clone") - .arg(git_url) - .output() - .expect("unable to execute git clone command"); - - if out.status.success() { - // By this time the cloned repo should be available in the current directory - println!("Git cloning successful for the package {}", package_name); - // Resolve dependencies by reading the FPM.ftd using config.read() - let _config = fpm::Config::read(Some(package_name.to_string())).await?; + match gp_status { + true => { + // package name and git repo url + let package_name = match package_response["result"]["package"].as_str() { + Some(valid_name) => valid_name, + None => panic!("received invalid package name from get_package API"), + }; + if let Some(git_url) = package_response["result"]["git"].as_str() { + // Clone the git package into the current directory + // Need to execute shell commands from rust + // git_url https format: https://github.com//.git + let out = std::process::Command::new("git") + .arg("clone") + .arg(git_url) + .output() + .expect("unable to execute git clone command"); + + if out.status.success() { + // By this time the cloned repo should be available in the current directory + println!("Git cloning successful for the package {}", package_name); + // Resolve dependencies by reading the FPM.ftd using config.read() + // Assuming package_name and repo name are identical + let _config = fpm::Config::read(Some(package_name.to_string())).await?; + } + } else { + panic!("Invalid git url for the package {}", package_name); } - } else { - panic!("Invalid git url for the package {}", package_name); - } - } else { - panic!("get-package api success status returned false!!"); + }, + false => panic!("get-package api success status returned false!!") } // Once the dependencies are resolved for the package - // then call fpm_ready API to ensure that the controller now is ready for receiving further HTTP traffic - // input: fpm_instance, *(git commit hash) - // output: success: true/false - // format: lang: json - // { - // "success": true - // } + // then call fpm_ready API to ensure that the controller service is now ready // response from fpm_ready API let ready_response = fpm_ready(fpm_instance.as_str(), fpm_controller.as_str()).await?; @@ -254,13 +245,13 @@ mod controller { match fr_status { true => println!("FPM controller ready!!"), - false => panic!("FPM controller isn't ready yet"), + false => panic!("FPM controller isn't ready!!"), } Ok(()) } - async fn make_request(url: url::Url) -> fpm::Result { + async fn make_get_request(url: url::Url) -> fpm::Result { use fpm::library::http; let response = match http::_get(url).await { Ok(some_response) => some_response, @@ -282,18 +273,37 @@ mod controller { fpm_instance: &str, fpm_controller: &str, ) -> fpm::Result { + // get-package API + // input: fpm_instance + // output: package_name and git repo URL + // format: { + // "success": true, + // "result": { + // "package": "" + // "git": "" + // } + // } + let query_string = format!("instance={}", fpm_instance); let controller_api = format!("{}/get-package?{}", fpm_controller, query_string); let url = match url::Url::parse(controller_api.as_str()) { Ok(safe_url) => safe_url, - Err(_) => panic!("Invalid url for get-package api"), + Err(e) => panic!("Invalid get-package API endpoint, Parse error: {}",e.to_string()), }; - make_request(url).await + make_get_request(url).await } async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result { + // fpm-ready API + // input: fpm_instance, *(git commit hash) + // output: success: true/false + // format: lang: json + // { + // "success": true + // } + // Git commit hash needs to be computed before making a call to the fpm_ready API let git_commit = ""; @@ -302,9 +312,11 @@ mod controller { let url = match url::Url::parse(controller_api.as_str()) { Ok(safe_url) => safe_url, - Err(_) => panic!("Invalid url for fpm_ready api"), + Err(e) => panic!("Invalid fpm_ready API endpoint, Parse error: {}",e.to_string()), }; - make_request(url).await + // This request should be put request for fpm_ready API to update the instance status to ready + // Using http::_get() function to make request to this API for now + make_get_request(url).await } } From 8b821a5fc34a5b847519cf5c88a98151d3556c47 Mon Sep 17 00:00:00 2001 From: Arpita-Jaiswal Date: Tue, 14 Jun 2022 15:02:09 +0530 Subject: [PATCH 05/13] wip --- src/commands/serve.rs | 185 ++++++++++++++++++++++++------------------ src/dependency.rs | 68 ++++++++++++++++ src/library/http.rs | 4 +- 3 files changed, 174 insertions(+), 83 deletions(-) diff --git a/src/commands/serve.rs b/src/commands/serve.rs index 715e0e9a..89925c85 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -145,13 +145,22 @@ async fn serve_static(req: actix_web::HttpRequest) -> actix_web::HttpResponse { #[actix_web::main] pub async fn serve(port: &str) -> std::io::Result<()> { - // For debugging (will be refactored later) - let use_controller = false; + let use_controller = { + let mut use_controller = false; + if let Ok(val) = std::env::var("CONTROLLER") { + if let Ok(val) = val.parse::() { + use_controller = val; + } + } + use_controller + }; if use_controller { // fpm-controller base path and ec2 instance id (hardcoded for now) - let fpm_controller: String = "https:///controller.fifthtry.com".to_string(); - let fpm_instance: String = "".to_string(); + let fpm_controller: String = std::env::var("FPM_CONTROLLER") + .unwrap_or("https:///controller.fifthtry.com".to_string()); + let fpm_instance: String = + std::env::var("FPM_INSTANCE_ID").unwrap_or("".to_string()); match controller::resolve_dependencies(fpm_instance, fpm_controller).await { Ok(_) => println!("Dependencies resolved"), @@ -159,6 +168,8 @@ pub async fn serve(port: &str) -> std::io::Result<()> { } } + let mut config = fpm::Config::read(None).await.unwrap(); + println!("### Server Started ###"); println!("Go to: http://127.0.0.1:{}", port); actix_web::HttpServer::new(|| { @@ -199,38 +210,61 @@ mod controller { let package_response = get_package(fpm_instance.as_str(), fpm_controller.as_str()).await?; let gp_status = match package_response["success"].as_bool() { Some(res) => res, - None => panic!("success parameter doesn't exist in Json or isn't valid boolean type"), + None => { + return Err(fpm::Error::UsageError { + message: "success parameter doesn't exist in Json or isn't valid boolean type" + .to_string(), + }) + } }; - match gp_status { - true => { - // package name and git repo url - let package_name = match package_response["result"]["package"].as_str() { - Some(valid_name) => valid_name, - None => panic!("received invalid package name from get_package API"), - }; - if let Some(git_url) = package_response["result"]["git"].as_str() { - // Clone the git package into the current directory - // Need to execute shell commands from rust - // git_url https format: https://github.com//.git - let out = std::process::Command::new("git") - .arg("clone") - .arg(git_url) - .output() - .expect("unable to execute git clone command"); - - if out.status.success() { - // By this time the cloned repo should be available in the current directory - println!("Git cloning successful for the package {}", package_name); - // Resolve dependencies by reading the FPM.ftd using config.read() - // Assuming package_name and repo name are identical - let _config = fpm::Config::read(Some(package_name.to_string())).await?; - } - } else { - panic!("Invalid git url for the package {}", package_name); - } - }, - false => panic!("get-package api success status returned false!!") + if !gp_status { + return Err(fpm::Error::UsageError { + message: "get-package api success status returned false!!".to_string(), + }); + } + + // package name and git repo url + let package_name = match package_response["result"]["package"].as_str() { + Some(valid_name) => valid_name, + None => { + return Err(fpm::Error::UsageError { + message: "received invalid package name from get_package API".to_string(), + }) + } + }; + + if let Some(git_url) = package_response["result"]["git"].as_str() { + // Clone the git package into the current directory + // Need to execute shell commands from rust + // git_url https format: https://github.com//.git + + let package = { + let mut package = fpm::Package::new(package_name); + package.zip = Some(git_url.to_string()); + package + }; + + package.unzip_package().await?; + fpm::Config::read(None).await?; + + /*let out = std::process::Command::new("git") + .arg("clone") + .arg(git_url) + .output() + .expect("unable to execute git clone command"); + + if out.status.success() { + // By this time the cloned repo should be available in the current directory + println!("Git cloning successful for the package {}", package_name); + // Resolve dependencies by reading the FPM.ftd using config.read() + // Assuming package_name and repo name are identical + let _config = fpm::Config::read(Some(package_name.to_string())).await?; + }*/ + } else { + return Err(fpm::Error::UsageError { + message: "received invalid package name from get_package API".to_string(), + }); } // Once the dependencies are resolved for the package @@ -251,72 +285,61 @@ mod controller { Ok(()) } - async fn make_get_request(url: url::Url) -> fpm::Result { - use fpm::library::http; - let response = match http::_get(url).await { - Ok(some_response) => some_response, - Err(e) => { - panic!("failed to fetch data, error: {}", e.to_string()) - } - }; - - match serde_json::from_str(response.as_str()) { - Ok(v) => Ok(v), - Err(e) => panic!( - "failed parsing json from response text, error: {}", - e.to_string() - ), - } - } - + /// get-package API + /// input: fpm_instance + /// output: package_name and git repo URL + /// format: { + /// "success": true, + /// "result": { + /// "package": "" + /// "git": "" + /// } + /// } async fn get_package( fpm_instance: &str, fpm_controller: &str, ) -> fpm::Result { - // get-package API - // input: fpm_instance - // output: package_name and git repo URL - // format: { - // "success": true, - // "result": { - // "package": "" - // "git": "" - // } - // } - - let query_string = format!("instance={}", fpm_instance); - let controller_api = format!("{}/get-package?{}", fpm_controller, query_string); - + let controller_api = format!("{}/get-package?instance={}", fpm_controller, fpm_instance); let url = match url::Url::parse(controller_api.as_str()) { Ok(safe_url) => safe_url, - Err(e) => panic!("Invalid get-package API endpoint, Parse error: {}",e.to_string()), + Err(e) => panic!( + "Invalid get-package API endpoint, Parse error: {}", + e.to_string() + ), }; - make_get_request(url).await + let val = fpm::library::http::get(url, "", 0).await?; + Ok(val) } + /// fpm-ready API + /// input: fpm_instance, *(git commit hash) + /// output: success: true/false + /// format: lang: json + /// { + /// "success": true + /// } + + /// Git commit hash needs to be computed before making a call to the fpm_ready API async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result { - // fpm-ready API - // input: fpm_instance, *(git commit hash) - // output: success: true/false - // format: lang: json - // { - // "success": true - // } - - // Git commit hash needs to be computed before making a call to the fpm_ready API let git_commit = ""; - let query_string = format!("instance={}&git-commit={}", fpm_instance, git_commit); - let controller_api = format!("{}/fpm-ready?{}", fpm_controller, query_string); + let controller_api = format!( + "{}/fpm-ready?instance={}&git-commit={}", + fpm_controller, fpm_instance, git_commit + ); let url = match url::Url::parse(controller_api.as_str()) { Ok(safe_url) => safe_url, - Err(e) => panic!("Invalid fpm_ready API endpoint, Parse error: {}",e.to_string()), + Err(e) => panic!( + "Invalid fpm_ready API endpoint, Parse error: {}", + e.to_string() + ), }; // This request should be put request for fpm_ready API to update the instance status to ready // Using http::_get() function to make request to this API for now - make_get_request(url).await + let val = fpm::library::http::get(url, "", 0).await?; + Ok(val) } } diff --git a/src/dependency.rs b/src/dependency.rs index 52e265a6..65448253 100644 --- a/src/dependency.rs +++ b/src/dependency.rs @@ -311,6 +311,74 @@ impl fpm::Package { } } + pub(crate) async fn unzip_package(&self) -> fpm::Result<()> { + use std::convert::TryInto; + use std::io::Write; + + let download_url = if let Some(ref url) = self.zip { + url + } else { + return Ok(()); + }; + + let path = std::env::temp_dir().join(format!("{}.zip", self.name.replace("/", "__"))); + + let start = std::time::Instant::now(); + print!("Downloading {} ... ", self.name.as_str()); + std::io::stdout().flush()?; + // Download the zip folder + { + let mut response = if download_url[1..].contains("://") + || download_url.starts_with("//") + { + reqwest::get(download_url.as_str())? + } else if let Ok(response) = reqwest::get(format!("https://{}", download_url).as_str()) + { + response + } else { + reqwest::get(format!("http://{}", download_url).as_str())? + }; + let mut file = std::fs::File::create(&path)?; + // TODO: instead of reading the whole thing in memory use tokio::io::copy() somehow? + let mut buf: Vec = vec![]; + response.copy_to(&mut buf)?; + file.write_all(&buf)?; + // file.write_all(response.text().await?.as_bytes())?; + } + + let file = std::fs::File::open(&path)?; + // TODO: switch to async_zip crate + let mut archive = zip::ZipArchive::new(file)?; + for i in 0..archive.len() { + let mut c_file = archive.by_index(i).unwrap(); + let out_path = match c_file.enclosed_name() { + Some(path) => path.to_owned(), + None => continue, + }; + let out_path_without_folder = out_path.to_str().unwrap().split_once("/").unwrap().1; + let file_extract_path = { + let mut file_extract_path: camino::Utf8PathBuf = + std::env::current_dir()?.canonicalize()?.try_into()?; + file_extract_path = file_extract_path.join(out_path_without_folder); + file_extract_path + }; + if (&*c_file.name()).ends_with('/') { + std::fs::create_dir_all(&file_extract_path)?; + } else { + if let Some(p) = file_extract_path.parent() { + if !p.exists() { + std::fs::create_dir_all(p)?; + } + } + // Note: we will be able to use tokio::io::copy() with async_zip + let mut outfile = std::fs::File::create(file_extract_path)?; + std::io::copy(&mut c_file, &mut outfile)?; + } + } + fpm::utils::print_end(format!("Downloaded {}", self.name.as_str()).as_str(), start); + Ok(()) + } + /// This function is called by `process()` or recursively called by itself. /// It checks the `FPM.ftd` file of dependent package and find out all the dependency packages. /// If dependent package is not available, it calls `process()` to download it inside `.packages` directory diff --git a/src/library/http.rs b/src/library/http.rs index 44f2d98c..b43283cd 100644 --- a/src/library/http.rs +++ b/src/library/http.rs @@ -53,7 +53,7 @@ pub async fn processor<'a>( doc.from_json(&json, section) } -async fn get( +pub(crate) async fn get( url: url::Url, doc_id: &str, line_number: usize, @@ -78,7 +78,7 @@ async fn get( } } -pub async fn _get(url: url::Url) -> reqwest::Result { +async fn _get(url: url::Url) -> reqwest::Result { let mut headers = reqwest::header::HeaderMap::new(); headers.insert( reqwest::header::USER_AGENT, From 9e65b2949926a50b8461b7793080cec802ef6b94 Mon Sep 17 00:00:00 2001 From: Arpita-Jaiswal Date: Tue, 14 Jun 2022 15:09:07 +0530 Subject: [PATCH 06/13] Clippy fixes --- src/commands/serve.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/commands/serve.rs b/src/commands/serve.rs index 89925c85..a7ac7196 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -158,9 +158,9 @@ pub async fn serve(port: &str) -> std::io::Result<()> { if use_controller { // fpm-controller base path and ec2 instance id (hardcoded for now) let fpm_controller: String = std::env::var("FPM_CONTROLLER") - .unwrap_or("https:///controller.fifthtry.com".to_string()); + .unwrap_or_else(|_| "https:///controller.fifthtry.com".to_string()); let fpm_instance: String = - std::env::var("FPM_INSTANCE_ID").unwrap_or("".to_string()); + std::env::var("FPM_INSTANCE_ID").unwrap_or_else(|_| "".to_string()); match controller::resolve_dependencies(fpm_instance, fpm_controller).await { Ok(_) => println!("Dependencies resolved"), @@ -168,7 +168,7 @@ pub async fn serve(port: &str) -> std::io::Result<()> { } } - let mut config = fpm::Config::read(None).await.unwrap(); + fpm::Config::read(None).await.unwrap(); println!("### Server Started ###"); println!("Go to: http://127.0.0.1:{}", port); @@ -302,10 +302,7 @@ mod controller { let controller_api = format!("{}/get-package?instance={}", fpm_controller, fpm_instance); let url = match url::Url::parse(controller_api.as_str()) { Ok(safe_url) => safe_url, - Err(e) => panic!( - "Invalid get-package API endpoint, Parse error: {}", - e.to_string() - ), + Err(e) => panic!("Invalid get-package API endpoint, Parse error: {}", e), }; let val = fpm::library::http::get(url, "", 0).await?; @@ -331,10 +328,7 @@ mod controller { let url = match url::Url::parse(controller_api.as_str()) { Ok(safe_url) => safe_url, - Err(e) => panic!( - "Invalid fpm_ready API endpoint, Parse error: {}", - e.to_string() - ), + Err(e) => panic!("Invalid fpm_ready API endpoint, Parse error: {}", e), }; // This request should be put request for fpm_ready API to update the instance status to ready From 32b83f69edc3f2473d7672654e1c0a136b2a7078 Mon Sep 17 00:00:00 2001 From: wilderbit Date: Tue, 14 Jun 2022 16:01:25 +0530 Subject: [PATCH 07/13] refactor the code --- src/commands/serve.rs | 56 ++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/commands/serve.rs b/src/commands/serve.rs index a7ac7196..8f771b37 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -70,7 +70,7 @@ async fn handle_ftd(config: &mut fpm::Config, path: std::path::PathBuf) -> actix .await { Ok(r) => actix_web::HttpResponse::Ok().body(r), - Err(_e) => actix_web::HttpResponse::InternalServerError().body("TODO".as_bytes()), + Err(e) => actix_web::HttpResponse::InternalServerError().body(e.to_string()), }; } _ => actix_web::HttpResponse::InternalServerError().body("".as_bytes()), @@ -158,7 +158,7 @@ pub async fn serve(port: &str) -> std::io::Result<()> { if use_controller { // fpm-controller base path and ec2 instance id (hardcoded for now) let fpm_controller: String = std::env::var("FPM_CONTROLLER") - .unwrap_or_else(|_| "https:///controller.fifthtry.com".to_string()); + .unwrap_or_else(|_| "https://controller.fifthtry.com".to_string()); let fpm_instance: String = std::env::var("FPM_INSTANCE_ID").unwrap_or_else(|_| "".to_string()); @@ -180,24 +180,31 @@ pub async fn serve(port: &str) -> std::io::Result<()> { .await } -// FPM Controller Support -// FPM cli supports communication with fpm controller. This is an optional feature, and is only available when controller feature is enabled, which is not enabled by default. -// Controller Communication -// When controller feature is enabled, fpm serve will first communicate with the FPM controller service’s /get-package/ API. - -// FPM Controller Service Endpoint -// The FPM Controller Service’s endpoint is computed by using environment variable FPM_CONTROLLER, which will look something like this: https://controller.fifthtry.com, with the API path. -// FPM Controller Service has more than one APIs: /get-package/ and /fpm-ready/. - -// get-package: -// Through an environment variable FPM_INSTANCE_ID, the fpm serve will learn it’s instance id, and it will pass the instance id to the get-package API. -// The API returns the URL of the package to be downloaded, git repository URL and the package name. -// FPM will clone the git repository in the current directory. The current directory will contain FPM.ftd and other files of the package. -// FPM will then calls fpm install on it. - -// fpm-ready: -// Once dependencies are ready fpm calls /fpm-ready/ API on the controller. We will pass the FPM_INSTANCE_ID and the git commit hash as input to the API -// The API will return with success, and once it is done fpm will start receiving HTTP traffic from the controller service. +/// FPM Controller Support +/// FPM cli supports communication with fpm controller. This is an optional feature, and is only +/// available when controller feature is enabled, which is not enabled by default. +/// Controller Communication +/// When controller feature is enabled, fpm serve will first communicate with the FPM controller +/// service’s /get-package/ API. + +/// FPM Controller Service Endpoint +/// The FPM Controller Service’s endpoint is computed by using environment variable FPM_CONTROLLER, +/// which will look something like this: https://controller.fifthtry.com, with the API path. +/// FPM Controller Service has more than one APIs: /get-package/ and /fpm-ready/. + +/// get-package: +/// Through an environment variable FPM_INSTANCE_ID, the fpm serve will learn it’s instance id, and +/// it will pass the instance id to the get-package API. +/// The API returns the URL of the package to be downloaded, git repository URL and the package name. +/// FPM will clone the git repository in the current directory. The current directory will contain +/// FPM.ftd and other files of the package. +/// FPM will then calls fpm install on it. + +/// fpm-ready: +/// Once dependencies are ready fpm calls /fpm-ready/ API on the controller. We will pass the +/// FPM_INSTANCE_ID and the git commit hash as input to the API +/// The API will return with success, and once it is done fpm will start receiving HTTP traffic +/// from the controller service. mod controller { pub async fn resolve_dependencies( @@ -299,7 +306,11 @@ mod controller { fpm_instance: &str, fpm_controller: &str, ) -> fpm::Result { - let controller_api = format!("{}/get-package?instance={}", fpm_controller, fpm_instance); + let controller_api = format!( + "{}/v1/fpm/get-package?ec2_reservation={}", + fpm_controller, fpm_instance + ); + let url = match url::Url::parse(controller_api.as_str()) { Ok(safe_url) => safe_url, Err(e) => panic!("Invalid get-package API endpoint, Parse error: {}", e), @@ -322,7 +333,7 @@ mod controller { let git_commit = ""; let controller_api = format!( - "{}/fpm-ready?instance={}&git-commit={}", + "{}/v1/fpm/fpm-ready?ec2_reservation={}&hash={}", fpm_controller, fpm_instance, git_commit ); @@ -334,6 +345,7 @@ mod controller { // This request should be put request for fpm_ready API to update the instance status to ready // Using http::_get() function to make request to this API for now let val = fpm::library::http::get(url, "", 0).await?; + dbg!(&val); Ok(val) } } From 2ff9cd339e3fb9124f8f8bc3667392c09cbee97e Mon Sep 17 00:00:00 2001 From: wilderbit Date: Tue, 14 Jun 2022 18:41:12 +0530 Subject: [PATCH 08/13] http with type --- src/lib.rs | 3 +++ src/library/http.rs | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 15d198a0..035d2c8f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -390,6 +390,9 @@ pub enum Error { #[error("HttpError: {}", _0)] HttpError(#[from] reqwest::Error), + #[error("APIResponseError: {}", _0)] + APIResponseError(String), + #[error("IoError: {}", _0)] IoError(#[from] std::io::Error), diff --git a/src/library/http.rs b/src/library/http.rs index b43283cd..4ee58e57 100644 --- a/src/library/http.rs +++ b/src/library/http.rs @@ -89,3 +89,23 @@ async fn _get(url: url::Url) -> reqwest::Result { .build()?; c.get(url.to_string().as_str()).send()?.text() } + +pub async fn get_with_type( + url: url::Url, + headers: reqwest::header::HeaderMap, +) -> fpm::Result { + let c = reqwest::Client::builder() + .default_headers(headers) + .build()?; + + let mut resp = c.get(url.to_string().as_str()).send()?; + if !resp.status().eq(&reqwest::StatusCode::OK) { + return Err(fpm::Error::APIResponseError(format!( + "url: {}, response_status: {}, response: {:?}", + url, + resp.status(), + resp.text() + ))); + } + return resp.json().map_err(|x| x.into()); +} From 9de93814fe3734794149f93d95eecb03211a82bb Mon Sep 17 00:00:00 2001 From: wilderbit Date: Tue, 14 Jun 2022 18:58:00 +0530 Subject: [PATCH 09/13] feature flag for controller --- Cargo.toml | 3 +++ src/commands/serve.rs | 12 +----------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0c468c65..41a1642b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,9 @@ repository = "https://github.com/FifthTry/fpm" homepage = "https://fpm.dev" build = "build.rs" +[features] +controller = [] + [dependencies] async-recursion = "0.3.2" camino = "1.0.5" diff --git a/src/commands/serve.rs b/src/commands/serve.rs index 8f771b37..b18be419 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -145,17 +145,7 @@ async fn serve_static(req: actix_web::HttpRequest) -> actix_web::HttpResponse { #[actix_web::main] pub async fn serve(port: &str) -> std::io::Result<()> { - let use_controller = { - let mut use_controller = false; - if let Ok(val) = std::env::var("CONTROLLER") { - if let Ok(val) = val.parse::() { - use_controller = val; - } - } - use_controller - }; - - if use_controller { + if cfg!(feature = "controller") { // fpm-controller base path and ec2 instance id (hardcoded for now) let fpm_controller: String = std::env::var("FPM_CONTROLLER") .unwrap_or_else(|_| "https://controller.fifthtry.com".to_string()); From ac1ebc3bca273e47749fdcce2ee2d95022ce71f4 Mon Sep 17 00:00:00 2001 From: wilderbit Date: Tue, 14 Jun 2022 18:59:11 +0530 Subject: [PATCH 10/13] clippy happy --- src/library/http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library/http.rs b/src/library/http.rs index 4ee58e57..63a08b9f 100644 --- a/src/library/http.rs +++ b/src/library/http.rs @@ -107,5 +107,5 @@ pub async fn get_with_type( resp.text() ))); } - return resp.json().map_err(|x| x.into()); + resp.json().map_err(|x| x.into()) } From 1c9f361726e23e2709a810ec6413c5503df8c0a0 Mon Sep 17 00:00:00 2001 From: Heulitig Date: Wed, 15 Jun 2022 10:52:55 +0530 Subject: [PATCH 11/13] used get_with_type() for making requests --- src/commands/serve.rs | 119 ++++++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 56 deletions(-) diff --git a/src/commands/serve.rs b/src/commands/serve.rs index b18be419..2ec0e6d8 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -204,17 +204,10 @@ mod controller { // First call get_package API to get package details and resolve dependencies // response from get-package API - let package_response = get_package(fpm_instance.as_str(), fpm_controller.as_str()).await?; - let gp_status = match package_response["success"].as_bool() { - Some(res) => res, - None => { - return Err(fpm::Error::UsageError { - message: "success parameter doesn't exist in Json or isn't valid boolean type" - .to_string(), - }) - } - }; + let package_response: GetPackageResponse = + get_package(fpm_instance.as_str(), fpm_controller.as_str()).await?; + let gp_status = package_response.success; if !gp_status { return Err(fpm::Error::UsageError { message: "get-package api success status returned false!!".to_string(), @@ -222,58 +215,44 @@ mod controller { } // package name and git repo url - let package_name = match package_response["result"]["package"].as_str() { - Some(valid_name) => valid_name, - None => { - return Err(fpm::Error::UsageError { - message: "received invalid package name from get_package API".to_string(), - }) - } + let package_name = package_response.result.package; + let git_url = package_response.result.git; + + // Clone the git package into the current directory + // Need to execute shell commands from rust + // git_url https format: https://github.com//.git + + let package = { + let mut package = fpm::Package::new(package_name.as_str()); + package.zip = Some(git_url); + package }; - if let Some(git_url) = package_response["result"]["git"].as_str() { - // Clone the git package into the current directory - // Need to execute shell commands from rust - // git_url https format: https://github.com//.git + package.unzip_package().await?; + fpm::Config::read(None).await?; - let package = { - let mut package = fpm::Package::new(package_name); - package.zip = Some(git_url.to_string()); - package - }; + /*let out = std::process::Command::new("git") + .arg("clone") + .arg(git_url) + .output() + .expect("unable to execute git clone command"); - package.unzip_package().await?; - fpm::Config::read(None).await?; - - /*let out = std::process::Command::new("git") - .arg("clone") - .arg(git_url) - .output() - .expect("unable to execute git clone command"); - - if out.status.success() { - // By this time the cloned repo should be available in the current directory - println!("Git cloning successful for the package {}", package_name); - // Resolve dependencies by reading the FPM.ftd using config.read() - // Assuming package_name and repo name are identical - let _config = fpm::Config::read(Some(package_name.to_string())).await?; - }*/ - } else { - return Err(fpm::Error::UsageError { - message: "received invalid package name from get_package API".to_string(), - }); - } + if out.status.success() { + // By this time the cloned repo should be available in the current directory + println!("Git cloning successful for the package {}", package_name); + // Resolve dependencies by reading the FPM.ftd using config.read() + // Assuming package_name and repo name are identical + let _config = fpm::Config::read(Some(package_name.to_string())).await?; + }*/ // Once the dependencies are resolved for the package // then call fpm_ready API to ensure that the controller service is now ready // response from fpm_ready API - let ready_response = fpm_ready(fpm_instance.as_str(), fpm_controller.as_str()).await?; - let fr_status = match ready_response["success"].as_bool() { - Some(res) => res, - None => panic!("success parameter doesn't exist in Json or isn't valid boolean type"), - }; + let ready_response: FpmReadyResponse = + fpm_ready(fpm_instance.as_str(), fpm_controller.as_str()).await?; + let fr_status = ready_response.success; match fr_status { true => println!("FPM controller ready!!"), false => panic!("FPM controller isn't ready!!"), @@ -281,6 +260,17 @@ mod controller { Ok(()) } + #[derive(serde::Deserialize, Debug)] + struct PackageResult { + package: String, + git: String, + } + + #[derive(serde::Deserialize, Debug)] + struct GetPackageResponse { + success: bool, + result: PackageResult, + } /// get-package API /// input: fpm_instance @@ -295,7 +285,7 @@ mod controller { async fn get_package( fpm_instance: &str, fpm_controller: &str, - ) -> fpm::Result { + ) -> fpm::Result { let controller_api = format!( "{}/v1/fpm/get-package?ec2_reservation={}", fpm_controller, fpm_instance @@ -306,10 +296,21 @@ mod controller { Err(e) => panic!("Invalid get-package API endpoint, Parse error: {}", e), }; - let val = fpm::library::http::get(url, "", 0).await?; + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::USER_AGENT, + reqwest::header::HeaderValue::from_static("fpm"), + ); + + let val: GetPackageResponse = fpm::library::http::get_with_type(url, headers).await?; + dbg!(&val); Ok(val) } + #[derive(serde::Deserialize, Debug)] + struct FpmReadyResponse { + success: bool, + } /// fpm-ready API /// input: fpm_instance, *(git commit hash) /// output: success: true/false @@ -319,7 +320,7 @@ mod controller { /// } /// Git commit hash needs to be computed before making a call to the fpm_ready API - async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result { + async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result { let git_commit = ""; let controller_api = format!( @@ -334,7 +335,13 @@ mod controller { // This request should be put request for fpm_ready API to update the instance status to ready // Using http::_get() function to make request to this API for now - let val = fpm::library::http::get(url, "", 0).await?; + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::USER_AGENT, + reqwest::header::HeaderValue::from_static("fpm"), + ); + + let val: FpmReadyResponse = fpm::library::http::get_with_type(url, headers).await?; dbg!(&val); Ok(val) } From 690aa133add4fc8b590b86b2d6da3c2bef1421bc Mon Sep 17 00:00:00 2001 From: wilderbit Date: Wed, 15 Jun 2022 12:12:29 +0530 Subject: [PATCH 12/13] refactor --- src/commands/serve.rs | 179 +----------------------------------------- src/config.rs | 5 ++ src/controller.rs | 154 ++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 + src/library/http.rs | 2 +- 5 files changed, 165 insertions(+), 179 deletions(-) create mode 100644 src/controller.rs diff --git a/src/commands/serve.rs b/src/commands/serve.rs index 2ec0e6d8..ebe0d1a3 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -152,7 +152,7 @@ pub async fn serve(port: &str) -> std::io::Result<()> { let fpm_instance: String = std::env::var("FPM_INSTANCE_ID").unwrap_or_else(|_| "".to_string()); - match controller::resolve_dependencies(fpm_instance, fpm_controller).await { + match crate::controller::resolve_dependencies(fpm_instance, fpm_controller).await { Ok(_) => println!("Dependencies resolved"), Err(_) => panic!("Error resolving dependencies using controller!!"), } @@ -169,180 +169,3 @@ pub async fn serve(port: &str) -> std::io::Result<()> { .run() .await } - -/// FPM Controller Support -/// FPM cli supports communication with fpm controller. This is an optional feature, and is only -/// available when controller feature is enabled, which is not enabled by default. -/// Controller Communication -/// When controller feature is enabled, fpm serve will first communicate with the FPM controller -/// service’s /get-package/ API. - -/// FPM Controller Service Endpoint -/// The FPM Controller Service’s endpoint is computed by using environment variable FPM_CONTROLLER, -/// which will look something like this: https://controller.fifthtry.com, with the API path. -/// FPM Controller Service has more than one APIs: /get-package/ and /fpm-ready/. - -/// get-package: -/// Through an environment variable FPM_INSTANCE_ID, the fpm serve will learn it’s instance id, and -/// it will pass the instance id to the get-package API. -/// The API returns the URL of the package to be downloaded, git repository URL and the package name. -/// FPM will clone the git repository in the current directory. The current directory will contain -/// FPM.ftd and other files of the package. -/// FPM will then calls fpm install on it. - -/// fpm-ready: -/// Once dependencies are ready fpm calls /fpm-ready/ API on the controller. We will pass the -/// FPM_INSTANCE_ID and the git commit hash as input to the API -/// The API will return with success, and once it is done fpm will start receiving HTTP traffic -/// from the controller service. - -mod controller { - pub async fn resolve_dependencies( - fpm_instance: String, - fpm_controller: String, - ) -> fpm::Result<()> { - // First call get_package API to get package details and resolve dependencies - - // response from get-package API - let package_response: GetPackageResponse = - get_package(fpm_instance.as_str(), fpm_controller.as_str()).await?; - - let gp_status = package_response.success; - if !gp_status { - return Err(fpm::Error::UsageError { - message: "get-package api success status returned false!!".to_string(), - }); - } - - // package name and git repo url - let package_name = package_response.result.package; - let git_url = package_response.result.git; - - // Clone the git package into the current directory - // Need to execute shell commands from rust - // git_url https format: https://github.com//.git - - let package = { - let mut package = fpm::Package::new(package_name.as_str()); - package.zip = Some(git_url); - package - }; - - package.unzip_package().await?; - fpm::Config::read(None).await?; - - /*let out = std::process::Command::new("git") - .arg("clone") - .arg(git_url) - .output() - .expect("unable to execute git clone command"); - - if out.status.success() { - // By this time the cloned repo should be available in the current directory - println!("Git cloning successful for the package {}", package_name); - // Resolve dependencies by reading the FPM.ftd using config.read() - // Assuming package_name and repo name are identical - let _config = fpm::Config::read(Some(package_name.to_string())).await?; - }*/ - - // Once the dependencies are resolved for the package - // then call fpm_ready API to ensure that the controller service is now ready - - // response from fpm_ready API - let ready_response: FpmReadyResponse = - fpm_ready(fpm_instance.as_str(), fpm_controller.as_str()).await?; - - let fr_status = ready_response.success; - match fr_status { - true => println!("FPM controller ready!!"), - false => panic!("FPM controller isn't ready!!"), - } - - Ok(()) - } - #[derive(serde::Deserialize, Debug)] - struct PackageResult { - package: String, - git: String, - } - - #[derive(serde::Deserialize, Debug)] - struct GetPackageResponse { - success: bool, - result: PackageResult, - } - - /// get-package API - /// input: fpm_instance - /// output: package_name and git repo URL - /// format: { - /// "success": true, - /// "result": { - /// "package": "" - /// "git": "" - /// } - /// } - async fn get_package( - fpm_instance: &str, - fpm_controller: &str, - ) -> fpm::Result { - let controller_api = format!( - "{}/v1/fpm/get-package?ec2_reservation={}", - fpm_controller, fpm_instance - ); - - let url = match url::Url::parse(controller_api.as_str()) { - Ok(safe_url) => safe_url, - Err(e) => panic!("Invalid get-package API endpoint, Parse error: {}", e), - }; - - let mut headers = reqwest::header::HeaderMap::new(); - headers.insert( - reqwest::header::USER_AGENT, - reqwest::header::HeaderValue::from_static("fpm"), - ); - - let val: GetPackageResponse = fpm::library::http::get_with_type(url, headers).await?; - dbg!(&val); - Ok(val) - } - - #[derive(serde::Deserialize, Debug)] - struct FpmReadyResponse { - success: bool, - } - /// fpm-ready API - /// input: fpm_instance, *(git commit hash) - /// output: success: true/false - /// format: lang: json - /// { - /// "success": true - /// } - - /// Git commit hash needs to be computed before making a call to the fpm_ready API - async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result { - let git_commit = ""; - - let controller_api = format!( - "{}/v1/fpm/fpm-ready?ec2_reservation={}&hash={}", - fpm_controller, fpm_instance, git_commit - ); - - let url = match url::Url::parse(controller_api.as_str()) { - Ok(safe_url) => safe_url, - Err(e) => panic!("Invalid fpm_ready API endpoint, Parse error: {}", e), - }; - - // This request should be put request for fpm_ready API to update the instance status to ready - // Using http::_get() function to make request to this API for now - let mut headers = reqwest::header::HeaderMap::new(); - headers.insert( - reqwest::header::USER_AGENT, - reqwest::header::HeaderValue::from_static("fpm"), - ); - - let val: FpmReadyResponse = fpm::library::http::get_with_type(url, headers).await?; - dbg!(&val); - Ok(val) - } -} diff --git a/src/config.rs b/src/config.rs index 5b262a66..012d1cfd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -705,6 +705,11 @@ impl Package { } } + pub fn with_zip(mut self, zip: String) -> fpm::Package { + self.zip = Some(zip); + self + } + pub fn get_dependency_for_interface(&self, interface: &str) -> Option<&fpm::Dependency> { self.dependencies .iter() diff --git a/src/controller.rs b/src/controller.rs new file mode 100644 index 00000000..460e6469 --- /dev/null +++ b/src/controller.rs @@ -0,0 +1,154 @@ +/// FPM Controller Support +/// FPM cli supports communication with fpm controller. This is an optional feature, and is only +/// available when controller feature is enabled, which is not enabled by default. +/// Controller Communication +/// When controller feature is enabled, fpm serve will first communicate with the FPM controller +/// service’s /get-package/ API. + +/// FPM Controller Service Endpoint +/// The FPM Controller Service’s endpoint is computed by using environment variable FPM_CONTROLLER, +/// which will look something like this: https://controller.fifthtry.com, with the API path. +/// FPM Controller Service has more than one APIs: /get-package/ and /fpm-ready/. + +/// get-package: +/// Through an environment variable FPM_INSTANCE_ID, the fpm serve will learn it’s instance id, and +/// it will pass the instance id to the get-package API. +/// The API returns the URL of the package to be downloaded, git repository URL and the package name. +/// FPM will clone the git repository in the current directory. The current directory will contain +/// FPM.ftd and other files of the package. +/// FPM will then calls fpm install on it. + +/// fpm-ready: +/// Once dependencies are ready fpm calls /fpm-ready/ API on the controller. We will pass the +/// FPM_INSTANCE_ID and the git commit hash as input to the API +/// The API will return with success, and once it is done fpm will start receiving HTTP traffic +/// from the controller service. + +#[derive(serde::Deserialize, Debug)] +struct ApiResponse { + success: bool, + result: Option, + message: Option, +} + +#[derive(serde::Deserialize, Debug)] +struct PackageResult { + package: String, + git: String, +} + +pub async fn resolve_dependencies(fpm_instance: String, fpm_controller: String) -> fpm::Result<()> { + // First call get_package API to get package details and resolve dependencies + + // response from get-package API + let package_response = get_package(fpm_instance.as_str(), fpm_controller.as_str()).await?; + + // Clone the git package into the current directory + // Need to execute shell commands from rust + // git_url https format: https://github.com//.git + + let package = + fpm::Package::new(package_response.package.as_str()).with_zip(package_response.git); + + package.unzip_package().await?; + fpm::Config::read(None).await?; + + /*let out = std::process::Command::new("git") + .arg("clone") + .arg(git_url) + .output() + .expect("unable to execute git clone command"); + + if out.status.success() { + // By this time the cloned repo should be available in the current directory + println!("Git cloning successful for the package {}", package_name); + // Resolve dependencies by reading the FPM.ftd using config.read() + // Assuming package_name and repo name are identical + let _config = fpm::Config::read(Some(package_name.to_string())).await?; + }*/ + + // Once the dependencies are resolved for the package + // then call fpm_ready API to ensure that the controller service is now ready + + // response from fpm_ready API + + fpm_ready(fpm_instance.as_str(), fpm_controller.as_str()).await?; + + Ok(()) +} + +/// get-package API +/// input: fpm_instance +/// output: package_name and git repo URL +/// format: { +/// "success": true, +/// "result": { +/// "package": "" +/// "git": "" +/// } +/// } +async fn get_package(fpm_instance: &str, fpm_controller: &str) -> fpm::Result { + let controller_api = format!( + "{}/v1/fpm/get-package?ec2_reservation={}", + fpm_controller, fpm_instance + ); + + let url = url::Url::parse(controller_api.as_str())?; + + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::USER_AGENT, + reqwest::header::HeaderValue::from_static("fpm"), + ); + + let resp: ApiResponse = fpm::library::http::get_with_type(url, headers).await?; + + if !resp.success { + return Err(fpm::Error::APIResponseError(format!( + "get_package api error: {:?}", + resp.message + ))); + } + + resp.result.ok_or({ + fpm::Error::APIResponseError(format!("get_package api error: {:?}", &resp.message)) + }) +} + +/// fpm-ready API +/// input: fpm_instance, *(git commit hash) +/// output: success: true/false +/// format: lang: json +/// { +/// "success": true +/// } + +/// Git commit hash needs to be computed before making a call to the fpm_ready API +async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result<()> { + let git_commit = ""; + + let controller_api = format!( + "{}/v1/fpm/fpm-ready?ec2_reservation={}&hash={}", + fpm_controller, fpm_instance, git_commit + ); + + let url = url::Url::parse(controller_api.as_str())?; + + // This request should be put request for fpm_ready API to update the instance status to ready + // Using http::_get() function to make request to this API for now + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::USER_AGENT, + reqwest::header::HeaderValue::from_static("fpm"), + ); + + // TODO: here i32 is wrong, + let resp: ApiResponse = fpm::library::http::get_with_type(url, headers).await?; + if !resp.success { + return Err(fpm::Error::APIResponseError(format!( + "fpm_ready api error: {:?}", + resp.message + ))); + } + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 035d2c8f..8a1aa59f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub(crate) mod utils; mod auto_import; mod commands; mod config; +mod controller; mod dependency; mod doc; mod file; @@ -419,6 +420,9 @@ pub enum Error { #[error("SitemapParseError: {}", _0)] SitemapParseError(#[from] fpm::sitemap::ParseError), + + #[error("URLParseError: {}", _0)] + UrlParseError(#[from] url::ParseError), } pub type Result = std::result::Result; diff --git a/src/library/http.rs b/src/library/http.rs index 63a08b9f..46a03851 100644 --- a/src/library/http.rs +++ b/src/library/http.rs @@ -107,5 +107,5 @@ pub async fn get_with_type( resp.text() ))); } - resp.json().map_err(|x| x.into()) + Ok(resp.json()?) } From fc1b22dd50eb745516f9f01de1a6e5109a0dba24 Mon Sep 17 00:00:00 2001 From: wilderbit Date: Wed, 15 Jun 2022 12:38:53 +0530 Subject: [PATCH 13/13] refactor --- src/controller.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/controller.rs b/src/controller.rs index 460e6469..a4ed2c38 100644 --- a/src/controller.rs +++ b/src/controller.rs @@ -142,8 +142,9 @@ async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result<()> reqwest::header::HeaderValue::from_static("fpm"), ); - // TODO: here i32 is wrong, - let resp: ApiResponse = fpm::library::http::get_with_type(url, headers).await?; + // TODO: here Map is wrong, + let resp: ApiResponse> = + fpm::library::http::get_with_type(url, headers).await?; if !resp.success { return Err(fpm::Error::APIResponseError(format!( "fpm_ready api error: {:?}",