From 2434bd3be409bb81b5da13d03f3da10a7669699f Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 30 Apr 2024 19:48:26 +0700 Subject: [PATCH 01/39] impl PayloadData which has JsonRpcPayload and GetUrlPayload; provide general generate_payload_from_req func; doc comms; add url crate --- .gitignore | 2 +- Cargo.lock | 2 + Cargo.toml | 1 + assets/.config_test | 61 ++++++++++++--------- src/ctx.rs | 26 +++++++++ src/net/http.rs | 94 +++++++++++++++++++++++++++----- src/net/server.rs | 7 ++- src/net/websocket.rs | 4 +- src/security/proof_of_funding.rs | 4 +- src/security/sign.rs | 7 ++- 10 files changed, 160 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 30cca12..48d2393 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # Generated by Cargo # will have compiled files and executables /target/ - +.idea # These are backup files generated by rustfmt **/*.rs.bk diff --git a/Cargo.lock b/Cargo.lock index f20b020..8d86739 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,6 +717,7 @@ dependencies = [ "simple_logger", "tokio", "tokio-tungstenite", + "url", ] [[package]] @@ -1700,6 +1701,7 @@ dependencies = [ "idna", "matches", "percent-encoding", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 92172dd..c9ff2d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ tokio-tungstenite = { version = "0.20.0", features = ["native-tls"] } bitcrypto = { git = "https://github.com/KomodoPlatform/atomicDEX-API", branch = "dev" } ethkey = { git = "https://github.com/artemii235/parity-ethereum.git" } serialization = { git = "https://github.com/KomodoPlatform/atomicDEX-API", branch = "dev" } +url = { version = "2.2.2", features = ["serde"] } [target.x86_64-unknown-linux-gnu.dependencies] jemallocator = "0.5.0" diff --git a/assets/.config_test b/assets/.config_test index 95e9892..6f1691a 100644 --- a/assets/.config_test +++ b/assets/.config_test @@ -1,28 +1,37 @@ { - "port": 5000, - "redis_connection_string": "dummy-value", - "pubkey_path": "dummy-value", - "privkey_path": "dummy-value", - "token_expiration_time": 300, - "proxy_routes": [ - { - "inbound_route": "/test", - "outbound_route": "https://komodoplatform.com", - "authorized": false, - "allowed_methods": [] - }, - { - "inbound_route": "/test-2", - "outbound_route": "https://atomicdex.io", - "authorized": false, - "allowed_methods": [] - } - ], - "rate_limiter": { - "rp_1_min": 555, - "rp_5_min": 555, - "rp_15_min": 555, - "rp_30_min": 555, - "rp_60_min": 555 - } + "port": 5000, + "redis_connection_string": "dummy-value", + "pubkey_path": "dummy-value", + "privkey_path": "dummy-value", + "token_expiration_time": 300, + "proxy_routes": [ + { + "inbound_route": "/test", + "outbound_route": "https://komodoplatform.com", + "proxy_type": "json_rpc", + "authorized": false, + "allowed_methods": [] + }, + { + "inbound_route": "/test-2", + "outbound_route": "https://atomicdex.io", + "proxy_type": "json_rpc", + "authorized": false, + "allowed_methods": [] + }, + { + "inbound_route": "/nft-test", + "outbound_route": "https://nft.proxy", + "proxy_type": "get_url", + "authorized": false, + "allowed_methods": [] + } + ], + "rate_limiter": { + "rp_1_min": 555, + "rp_5_min": 555, + "rp_15_min": 555, + "rp_30_min": 555, + "rp_60_min": 555 + } } \ No newline at end of file diff --git a/src/ctx.rs b/src/ctx.rs index 316666a..dfc2fad 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -29,12 +29,20 @@ pub(crate) struct AppConfig { pub(crate) struct ProxyRoute { pub(crate) inbound_route: String, pub(crate) outbound_route: String, + pub(crate) proxy_type: ProxyType, #[serde(default)] pub(crate) authorized: bool, #[serde(default)] pub(crate) allowed_methods: Vec, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ProxyType { + JsonRpc, + GetUrl, +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct RateLimiter { pub(crate) rp_1_min: u16, @@ -82,12 +90,21 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { ProxyRoute { inbound_route: String::from("/test"), outbound_route: String::from("https://komodoplatform.com"), + proxy_type: ProxyType::JsonRpc, authorized: false, allowed_methods: Vec::default(), }, ProxyRoute { inbound_route: String::from("/test-2"), outbound_route: String::from("https://atomicdex.io"), + proxy_type: ProxyType::JsonRpc, + authorized: false, + allowed_methods: Vec::default(), + }, + ProxyRoute { + inbound_route: String::from("/nft-test"), + outbound_route: String::from("https://nft.proxy"), + proxy_type: ProxyType::GetUrl, authorized: false, allowed_methods: Vec::default(), }, @@ -114,12 +131,21 @@ fn test_app_config_serialzation_and_deserialization() { { "inbound_route": "/test", "outbound_route": "https://komodoplatform.com", + "proxy_type":"json_rpc", "authorized": false, "allowed_methods": [] }, { "inbound_route": "/test-2", "outbound_route": "https://atomicdex.io", + "proxy_type":"json_rpc", + "authorized": false, + "allowed_methods": [] + }, + { + "inbound_route": "/nft-test", + "outbound_route": "https://nft.proxy", + "proxy_type":"get_url", "authorized": false, "allowed_methods": [] } diff --git a/src/net/http.rs b/src/net/http.rs index c71e898..c30c356 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -1,7 +1,7 @@ use std::net::SocketAddr; use address_status::{get_address_status_list, post_address_status}; -use ctx::{AppConfig, ProxyRoute}; +use ctx::{AppConfig, ProxyRoute, ProxyType}; use hyper::header::HeaderName; use hyper::{ header::{self, HeaderValue}, @@ -12,6 +12,7 @@ use jwt::{get_cached_token_or_generate_one, JwtClaims}; use serde::{Deserialize, Serialize}; use serde_json::json; use sign::SignedMessage; +use url::Url; use super::*; use crate::server::{is_private_ip, validation_middleware}; @@ -62,17 +63,35 @@ pub(crate) async fn insert_jwt_to_http_header( Ok(()) } -async fn parse_payload(req: Request) -> GenericResult<(Request, RpcPayload)> { +/// Asynchronously parses an HTTP request's body into a specified type `T`, modifying the request +/// to have an empty body if the method is `GET`, and returning the original body otherwise. +/// Ensures that the body is not empty before attempting deserialization into the non-optional type `T`. +async fn parse_payload(req: Request) -> GenericResult<(Request, T)> +where + T: serde::de::DeserializeOwned, +{ let (parts, body) = req.into_parts(); let body_bytes = hyper::body::to_bytes(body).await?; - let payload: RpcPayload = serde_json::from_slice(&body_bytes)?; + if body_bytes.is_empty() { + return Err("Empty body cannot be deserialized into non-optional type T".into()); + } + + let payload: T = serde_json::from_slice(&body_bytes)?; - Ok((Request::from_parts(parts, Body::from(body_bytes)), payload)) + let new_req = if parts.method == Method::GET { + Request::from_parts(parts, Body::empty()) + } else { + Request::from_parts(parts, Body::from(body_bytes)) + }; + + Ok((new_req, payload)) } -#[derive(Debug, Serialize, Deserialize, PartialEq)] -pub(crate) struct RpcPayload { +/// Represents a JSON RPC payload parsed from a proxy request. It combines standard JSON RPC method call +/// fields with a `SignedMessage` for authentication and validation by the proxy. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub(crate) struct JsonRpcPayload { pub(crate) method: String, pub(crate) params: serde_json::value::Value, pub(crate) id: usize, @@ -80,11 +99,47 @@ pub(crate) struct RpcPayload { pub(crate) signed_message: SignedMessage, } +/// Represents a payload for a GET URL request parsed from a proxy request. This struct contains the URL +/// that the proxy will forward the GET request to, along with a `SignedMessage` for authentication and validation. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub(crate) struct GetUrlPayload { + pub(crate) url: Url, + pub(crate) signed_message: SignedMessage, +} + +/// Enumerates the types of payloads that can be processed by the proxy. +/// Each variant holds a specific payload type relevant to the proxy operation being performed. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum PayloadData { + JsonRpc(JsonRpcPayload), + GetUrl(GetUrlPayload), +} + +/// Asynchronously generates and parses payload data from an HTTP request based on the specified proxy type. +/// Returns a tuple containing the modified (if necessary) request and the parsed payload from req Body as `PayloadData`. +#[allow(dead_code)] +async fn generate_payload_from_req( + req: Request, + proxy_type: &ProxyType, +) -> GenericResult<(Request, PayloadData)> { + match proxy_type { + ProxyType::JsonRpc => { + let (req, payload) = parse_payload::(req).await?; + Ok((req, PayloadData::JsonRpc(payload))) + } + ProxyType::GetUrl => { + let (req, payload) = parse_payload::(req).await?; + Ok((req, PayloadData::GetUrl(payload))) + } + } +} + +// TODO change name to proxy_eth, as it handles eth api specific logic from JsonRpcPayload async fn proxy( cfg: &AppConfig, mut req: Request, remote_addr: &SocketAddr, - payload: RpcPayload, + payload: JsonRpcPayload, x_forwarded_for: HeaderValue, proxy_route: &ProxyRoute, ) -> GenericResult> { @@ -217,7 +272,8 @@ pub(crate) async fn http_handler( return handle_preflight(); } - let (req, payload) = match parse_payload(req).await { + // TODO use generate_payload_from_req() instead + let (req, payload): (Request, JsonRpcPayload) = match parse_payload(req).await { Ok(t) => t, Err(_) => { log::warn!( @@ -259,7 +315,7 @@ pub(crate) async fn http_handler( } }; - let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().to_string()) { + let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().path().to_string()) { Some(proxy_route) => proxy_route, None => { log::warn!( @@ -275,6 +331,8 @@ pub(crate) async fn http_handler( } }; + // TODO here we can get ProxyType from inbound and only then call parse_payload with "Request received." logging for eth/nft etc proxy + if is_private_ip { return proxy( cfg, @@ -319,9 +377,9 @@ fn test_rpc_payload_serialzation_and_deserialization() { } }); - let actual_payload: RpcPayload = serde_json::from_str(&json_payload.to_string()).unwrap(); + let actual_payload: JsonRpcPayload = serde_json::from_str(&json_payload.to_string()).unwrap(); - let expected_payload = RpcPayload { + let expected_payload = JsonRpcPayload { method: String::from("dummy-value"), params: json!([]), id: 1, @@ -344,8 +402,13 @@ fn test_rpc_payload_serialzation_and_deserialization() { #[test] fn test_get_proxy_route_by_inbound() { + use std::str::FromStr; + let cfg = ctx::get_app_config_test_instance(); + // If we leave this code line `let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().to_string()) {` + // inbound_route cant be "/test", as it's not uri. I suppose inbound actually should be a Path. + // Two options: in `req.uri().to_string()` path() is missing or "/test" in test is wrong and the whole url should be. let proxy_route = cfg .get_proxy_route_by_inbound(String::from("/test")) .unwrap(); @@ -357,6 +420,11 @@ fn test_get_proxy_route_by_inbound() { .unwrap(); assert_eq!(proxy_route.outbound_route, "https://atomicdex.io"); + + let url = Url::from_str("https://komodo.proxy:5535/nft-test").unwrap(); + let path = url.path().to_string(); + let proxy_route = cfg.get_proxy_route_by_inbound(path).unwrap(); + assert_eq!(proxy_route.outbound_route, "https://nft.proxy"); } #[test] @@ -397,10 +465,10 @@ async fn test_parse_payload() { "dummy-value".parse().unwrap(), ); - let (req, payload) = parse_payload(req).await.unwrap(); + let (req, payload): (Request, JsonRpcPayload) = parse_payload(req).await.unwrap(); let header_value = req.headers().get("dummy-header").unwrap(); - let expected_payload = RpcPayload { + let expected_payload = JsonRpcPayload { method: String::from("dummy-value"), params: json!([]), id: 1, diff --git a/src/net/server.rs b/src/net/server.rs index 863a802..7d47291 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -8,7 +8,7 @@ use hyper::{Body, Request, Response, Server, StatusCode, Uri}; use crate::address_status::AddressStatusOperations; use crate::ctx::ProxyRoute; use crate::db::Db; -use crate::http::{http_handler, response_by_status, RpcPayload}; +use crate::http::{http_handler, response_by_status, JsonRpcPayload}; use crate::log_format; use crate::proof_of_funding::{verify_message_and_balance, ProofOfFundingError}; use crate::rate_limiter::RateLimitOperations; @@ -67,9 +67,10 @@ async fn connection_handler( } } +// TODO right now this is only an ethereum API specific validation pub(crate) async fn validation_middleware( cfg: &AppConfig, - payload: &RpcPayload, + payload: &JsonRpcPayload, proxy_route: &ProxyRoute, req_uri: &Uri, remote_addr: &SocketAddr, @@ -185,7 +186,7 @@ pub(crate) async fn serve(cfg: &'static AppConfig) -> GenericResult<()> { let server = Server::bind(&addr).serve(handler); - log::info!("AtomicDEX Auth API serving on http://{}", addr); + log::info!("Komodo-DeFi-Poxy API serving on http://{}", addr); Ok(server.await?) } diff --git a/src/net/websocket.rs b/src/net/websocket.rs index b5ec7d9..c211972 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -10,7 +10,7 @@ use tokio_tungstenite::{ use crate::{ ctx::AppConfig, - http::{response_by_status, RpcPayload}, + http::{response_by_status, JsonRpcPayload}, log_format, server::validation_middleware, GenericResult, @@ -137,7 +137,7 @@ pub(crate) async fn socket_handler( match msg { Some(Ok(msg)) => { if let Message::Text(msg) = msg { - let payload: RpcPayload = match serde_json::from_str(&msg) { + let payload: JsonRpcPayload = match serde_json::from_str(&msg) { Ok(t) => t, Err(e) => { if let Err(e) = inbound_socket.send(format!("Invalid payload. {e}").into()).await { diff --git a/src/security/proof_of_funding.rs b/src/security/proof_of_funding.rs index fde8c25..890de94 100644 --- a/src/security/proof_of_funding.rs +++ b/src/security/proof_of_funding.rs @@ -1,6 +1,6 @@ use ctx::{AppConfig, ProxyRoute}; use db::Db; -use http::RpcPayload; +use http::JsonRpcPayload; use rpc::Json; use serde_json::json; use sign::SignOps; @@ -18,7 +18,7 @@ pub(crate) enum ProofOfFundingError { pub(crate) async fn verify_message_and_balance( cfg: &AppConfig, - payload: &RpcPayload, + payload: &JsonRpcPayload, proxy_route: &ProxyRoute, ) -> Result<(), ProofOfFundingError> { if let Ok(true) = payload.signed_message.verify_message() { diff --git a/src/security/sign.rs b/src/security/sign.rs index 76d79bb..ce9c34a 100644 --- a/src/security/sign.rs +++ b/src/security/sign.rs @@ -20,7 +20,12 @@ pub(crate) trait SignOps { fn verify_message(&self) -> GenericResult; } -#[derive(Debug, Serialize, Deserialize, PartialEq)] +/// Represents a signed message used for authenticating and validating requests processed by the proxy. +/// +/// This structure contains cryptographic elements (`signature`) and metadata (`coin_ticker`, `address`, `timestamp_message`) +/// that are used to verify the authenticity and integrity of a request to the proxy. This is essential for securing +/// the proxy operations and preventing unauthorized access. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct SignedMessage { pub(crate) coin_ticker: String, pub(crate) address: String, From 91aa895b4a82334a2add105cd2108e1f2e69dc21 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 30 Apr 2024 19:51:27 +0700 Subject: [PATCH 02/39] remove Unnecessary trailing semicolon --- src/net/websocket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/websocket.rs b/src/net/websocket.rs index c211972..e0a3a5a 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -236,7 +236,7 @@ pub(crate) async fn socket_handler( _ => break }; } - }; + } } } e => { From f4fab5aaff76d603e4f018297cdc610a8c0d9dc9 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 1 May 2024 15:30:45 +0700 Subject: [PATCH 03/39] use get_proxy_route_by_inbound func in http_handler, add test_parse_get_url_payload --- src/net/http.rs | 150 +++++++++++++++++++++++++++++++--------------- src/net/server.rs | 2 +- 2 files changed, 104 insertions(+), 48 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index c30c356..63e2f3c 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -115,9 +115,18 @@ pub(crate) enum PayloadData { GetUrl(GetUrlPayload), } +impl PayloadData { + /// Returns a reference to the `SignedMessage` contained within the payload. + fn signed_message(&self) -> &SignedMessage { + match self { + PayloadData::JsonRpc(json_rpc_payload) => &json_rpc_payload.signed_message, + PayloadData::GetUrl(get_url_payload) => &get_url_payload.signed_message, + } + } +} + /// Asynchronously generates and parses payload data from an HTTP request based on the specified proxy type. /// Returns a tuple containing the modified (if necessary) request and the parsed payload from req Body as `PayloadData`. -#[allow(dead_code)] async fn generate_payload_from_req( req: Request, proxy_type: &ProxyType, @@ -134,7 +143,7 @@ async fn generate_payload_from_req( } } -// TODO change name to proxy_eth, as it handles eth api specific logic from JsonRpcPayload +// TODO handle eth and nft features async fn proxy( cfg: &AppConfig, mut req: Request, @@ -272,8 +281,23 @@ pub(crate) async fn http_handler( return handle_preflight(); } - // TODO use generate_payload_from_req() instead - let (req, payload): (Request, JsonRpcPayload) = match parse_payload(req).await { + let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().path().to_string()) { + Some(proxy_route) => proxy_route, + None => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + String::from("-"), + req.uri(), + "Proxy route not found, returning 404." + ) + ); + return response_by_status(StatusCode::NOT_FOUND); + } + }; + + let (req, payload) = match generate_payload_from_req(req, &proxy_route.proxy_type).await { Ok(t) => t, Err(_) => { log::warn!( @@ -282,7 +306,7 @@ pub(crate) async fn http_handler( remote_addr.ip(), String::from("-"), req_uri, - "Recieved invalid http payload, returning 401." + "Received invalid http payload, returning 401." ) ); return response_by_status(StatusCode::UNAUTHORIZED); @@ -293,9 +317,9 @@ pub(crate) async fn http_handler( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + payload.signed_message().address, req_uri, - "Request received." + "Request and payload data received." ) ); @@ -306,7 +330,7 @@ pub(crate) async fn http_handler( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + payload.signed_message().address, req_uri, "Error type casting of IpAddr into HeaderValue, returning 500." ) @@ -315,51 +339,29 @@ pub(crate) async fn http_handler( } }; - let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().path().to_string()) { - Some(proxy_route) => proxy_route, - None => { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req.uri(), - "Proxy route not found, returning 404." - ) - ); - return response_by_status(StatusCode::NOT_FOUND); - } + let temp = JsonRpcPayload { + method: "".to_string(), + params: Default::default(), + id: 0, + jsonrpc: "".to_string(), + signed_message: SignedMessage { + coin_ticker: "".to_string(), + address: "".to_string(), + timestamp_message: 0, + signature: "".to_string(), + }, }; - - // TODO here we can get ProxyType from inbound and only then call parse_payload with "Request received." logging for eth/nft etc proxy - if is_private_ip { - return proxy( - cfg, - req, - &remote_addr, - payload, - x_forwarded_for, - proxy_route, - ) - .await; + return proxy(cfg, req, &remote_addr, temp, x_forwarded_for, proxy_route).await; } if let Err(status_code) = - validation_middleware(cfg, &payload, proxy_route, req.uri(), &remote_addr).await + validation_middleware(cfg, &temp, proxy_route, req.uri(), &remote_addr).await { return response_by_status(status_code); } - proxy( - cfg, - req, - &remote_addr, - payload, - x_forwarded_for, - proxy_route, - ) - .await + proxy(cfg, req, &remote_addr, temp, x_forwarded_for, proxy_route).await } #[test] @@ -444,7 +446,7 @@ fn test_respond_by_status() { } #[tokio::test] -async fn test_parse_payload() { +async fn test_parse_json_rpc_payload() { let serialized_payload = json!({ "method": "dummy-value", "params": [], @@ -459,13 +461,23 @@ async fn test_parse_payload() { }) .to_string(); - let mut req = Request::new(Body::from(serialized_payload)); + let mut req = Request::builder() + .method(Method::POST) + .body(Body::from(serialized_payload)) + .unwrap(); req.headers_mut().insert( HeaderName::from_static("dummy-header"), "dummy-value".parse().unwrap(), ); - let (req, payload): (Request, JsonRpcPayload) = parse_payload(req).await.unwrap(); + let (mut req, payload): (Request, JsonRpcPayload) = parse_payload(req).await.unwrap(); + + let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); + assert!( + !body_bytes.is_empty(), + "Body should not be empty for non-GET methods" + ); + let header_value = req.headers().get("dummy-header").unwrap(); let expected_payload = JsonRpcPayload { @@ -484,3 +496,47 @@ async fn test_parse_payload() { assert_eq!(payload, expected_payload); assert_eq!(header_value, "dummy-value"); } + +#[tokio::test] +async fn test_parse_get_url_payload() { + let url_string = "http://example.com"; + let serialized_payload = json!({ + "url": url_string, + "signed_message": { + "coin_ticker": "BTC", + "address": "dummy-value", + "timestamp_message": 1655320000, + "signature": "dummy-value", + } + }) + .to_string(); + + let mut req = Request::new(Body::from(serialized_payload)); + req.headers_mut().insert( + HeaderName::from_static("accept"), + "application/json".parse().unwrap(), + ); + + let (mut req, payload): (Request, GetUrlPayload) = parse_payload(req).await.unwrap(); + + let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); + assert!( + body_bytes.is_empty(), + "Body should be empty for GET methods" + ); + + let header_value = req.headers().get("accept").unwrap(); + + let expected_payload = GetUrlPayload { + url: Url::parse(url_string).unwrap(), + signed_message: SignedMessage { + coin_ticker: String::from("BTC"), + address: String::from("dummy-value"), + timestamp_message: 1655320000, + signature: String::from("dummy-value"), + }, + }; + + assert_eq!(payload, expected_payload); + assert_eq!(header_value, "application/json"); +} diff --git a/src/net/server.rs b/src/net/server.rs index 7d47291..ff30ae4 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -67,7 +67,7 @@ async fn connection_handler( } } -// TODO right now this is only an ethereum API specific validation +// TODO handle eth and nft features pub(crate) async fn validation_middleware( cfg: &AppConfig, payload: &JsonRpcPayload, From 7abd17c13c6773eec13e94cf35b55bec5c704a46 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 1 May 2024 20:51:33 +0700 Subject: [PATCH 04/39] rename GetUrl to HttpGet and GetUrlPayload to HttpGetPayload, some more doc comms --- assets/.config_test | 2 +- src/ctx.rs | 25 ++++++++++++++++++++++--- src/main.rs | 5 +++++ src/net/http.rs | 21 +++++++++++---------- src/net/server.rs | 8 ++++++++ 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/assets/.config_test b/assets/.config_test index 6f1691a..657fddb 100644 --- a/assets/.config_test +++ b/assets/.config_test @@ -22,7 +22,7 @@ { "inbound_route": "/nft-test", "outbound_route": "https://nft.proxy", - "proxy_type": "get_url", + "proxy_type": "http_get", "authorized": false, "allowed_methods": [] } diff --git a/src/ctx.rs b/src/ctx.rs index dfc2fad..7900d0d 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -14,35 +14,54 @@ pub(crate) fn get_app_config() -> &'static AppConfig { }) } +/// Configuration settings for the application, loaded typically from a JSON configuration file. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct AppConfig { + // Optional server port to listen on. If None in config file, then 5000 is default. pub(crate) port: Option, + // Redis database connection string. pub(crate) redis_connection_string: String, + // File path to the public key used for cryptographic operations. pub(crate) pubkey_path: String, + // File path to the private key used for cryptographic operations. pub(crate) privkey_path: String, + // Optional token expiration time in seconds. If None then 3600 is default. pub(crate) token_expiration_time: Option, + // Routing configurations for proxying requests. pub(crate) proxy_routes: Vec, + // Rate limiting settings for request handling. pub(crate) rate_limiter: RateLimiter, } +/// Defines a routing rule for proxying requests from an inbound route to an outbound URL +/// based on a specified proxy type and additional authorization and method filtering criteria. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct ProxyRoute { + // The incoming route pattern. pub(crate) inbound_route: String, + // The target URL to which requests are forwarded. pub(crate) outbound_route: String, + // The type of proxying to perform (e.g., JSON RPC, HTTP GET). pub(crate) proxy_type: ProxyType, + // Whether authorization is required for this route. #[serde(default)] pub(crate) authorized: bool, + // Specific HTTP methods allowed for this route. #[serde(default)] pub(crate) allowed_methods: Vec, } +/// Enumerates different types of proxy operations supported, such as JSON RPC and HTTP GET. +/// This helps in applying specific handling logic based on the proxy type. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] pub(crate) enum ProxyType { JsonRpc, - GetUrl, + HttpGet, } +/// Configuration for rate limiting to manage the number of requests allowed over specified time intervals. +/// This prevents abuse and ensures fair usage of resources among all clients. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct RateLimiter { pub(crate) rp_1_min: u16, @@ -104,7 +123,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { ProxyRoute { inbound_route: String::from("/nft-test"), outbound_route: String::from("https://nft.proxy"), - proxy_type: ProxyType::GetUrl, + proxy_type: ProxyType::HttpGet, authorized: false, allowed_methods: Vec::default(), }, @@ -145,7 +164,7 @@ fn test_app_config_serialzation_and_deserialization() { { "inbound_route": "/nft-test", "outbound_route": "https://nft.proxy", - "proxy_type":"get_url", + "proxy_type":"http_get", "authorized": false, "allowed_methods": [] } diff --git a/src/main.rs b/src/main.rs index d4528eb..6d20493 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,7 +27,12 @@ mod websocket; #[global_allocator] static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; +/// A type alias for a generic error, encompassing any error that implements the `Error` trait, +/// along with traits for thread-safe error handling (`Send` and `Sync`). +/// This type is typically used across the application to handle errors uniformly. type GenericError = Box; +/// A type alias for a generic result, used throughout the application to encapsulate the +/// outcome of operations that might fail with a `GenericError`. type GenericResult = std::result::Result; #[tokio::main] diff --git a/src/net/http.rs b/src/net/http.rs index 63e2f3c..8a5af01 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -99,10 +99,10 @@ pub(crate) struct JsonRpcPayload { pub(crate) signed_message: SignedMessage, } -/// Represents a payload for a GET URL request parsed from a proxy request. This struct contains the URL +/// Represents a payload for HTTP GET request parsed from a proxy request. This struct contains the URL /// that the proxy will forward the GET request to, along with a `SignedMessage` for authentication and validation. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub(crate) struct GetUrlPayload { +pub(crate) struct HttpGetPayload { pub(crate) url: Url, pub(crate) signed_message: SignedMessage, } @@ -112,7 +112,7 @@ pub(crate) struct GetUrlPayload { #[derive(Clone, Debug, PartialEq)] pub(crate) enum PayloadData { JsonRpc(JsonRpcPayload), - GetUrl(GetUrlPayload), + HttpGet(HttpGetPayload), } impl PayloadData { @@ -120,7 +120,7 @@ impl PayloadData { fn signed_message(&self) -> &SignedMessage { match self { PayloadData::JsonRpc(json_rpc_payload) => &json_rpc_payload.signed_message, - PayloadData::GetUrl(get_url_payload) => &get_url_payload.signed_message, + PayloadData::HttpGet(http_get_payload) => &http_get_payload.signed_message, } } } @@ -136,9 +136,9 @@ async fn generate_payload_from_req( let (req, payload) = parse_payload::(req).await?; Ok((req, PayloadData::JsonRpc(payload))) } - ProxyType::GetUrl => { - let (req, payload) = parse_payload::(req).await?; - Ok((req, PayloadData::GetUrl(payload))) + ProxyType::HttpGet => { + let (req, payload) = parse_payload::(req).await?; + Ok((req, PayloadData::HttpGet(payload))) } } } @@ -281,6 +281,7 @@ pub(crate) async fn http_handler( return handle_preflight(); } + // create proxy_route before payload, as we need proxy_type from it for payload generation let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().path().to_string()) { Some(proxy_route) => proxy_route, None => { @@ -498,7 +499,7 @@ async fn test_parse_json_rpc_payload() { } #[tokio::test] -async fn test_parse_get_url_payload() { +async fn test_parse_http_get_payload() { let url_string = "http://example.com"; let serialized_payload = json!({ "url": url_string, @@ -517,7 +518,7 @@ async fn test_parse_get_url_payload() { "application/json".parse().unwrap(), ); - let (mut req, payload): (Request, GetUrlPayload) = parse_payload(req).await.unwrap(); + let (mut req, payload): (Request, HttpGetPayload) = parse_payload(req).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( @@ -527,7 +528,7 @@ async fn test_parse_get_url_payload() { let header_value = req.headers().get("accept").unwrap(); - let expected_payload = GetUrlPayload { + let expected_payload = HttpGetPayload { url: Url::parse(url_string).unwrap(), signed_message: SignedMessage { coin_ticker: String::from("BTC"), diff --git a/src/net/server.rs b/src/net/server.rs index ff30ae4..6602d75 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -39,6 +39,12 @@ fn get_real_address(req: &Request, remote_addr: &SocketAddr) -> GenericRes Ok(*remote_addr) } +/// Handles incoming HTTP requests based on their content and whether they need to be upgraded +/// to a socket connection. +/// +/// This function first resolves the real client address from the request, considering forwarded headers. +/// It then decides whether to handle the request as a regular HTTP request or upgrade it to a +/// socket-based connection based on its headers and content. async fn connection_handler( cfg: &AppConfig, req: Request, @@ -172,6 +178,8 @@ pub(crate) async fn validation_middleware( } } +/// Starts serving the proxy API on the configured port. This function sets up the HTTP server, +/// binds it to the specified address, and listens for incoming requests. pub(crate) async fn serve(cfg: &'static AppConfig) -> GenericResult<()> { let addr = format!("0.0.0.0:{}", cfg.port.unwrap_or(5000)).parse()?; From ac1c9e2d84b46d493a567d075e73082c17d97359 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 2 May 2024 15:22:44 +0700 Subject: [PATCH 05/39] provide PayloadData type in proxy and validation_middleware funcs --- src/net/http.rs | 52 +++++++++++++++++++++++++++++++------------- src/net/server.rs | 20 ++++++++++++++++- src/net/websocket.rs | 5 +++-- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index 8a5af01..1e910e4 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -145,6 +145,24 @@ async fn generate_payload_from_req( // TODO handle eth and nft features async fn proxy( + cfg: &AppConfig, + req: Request, + remote_addr: &SocketAddr, + payload: PayloadData, + x_forwarded_for: HeaderValue, + proxy_route: &ProxyRoute, +) -> GenericResult> { + match payload { + PayloadData::JsonRpc(payload) => { + proxy_eth(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await + } + PayloadData::HttpGet(_payload) => { + todo!() + } + } +} + +async fn proxy_eth( cfg: &AppConfig, mut req: Request, remote_addr: &SocketAddr, @@ -340,29 +358,33 @@ pub(crate) async fn http_handler( } }; - let temp = JsonRpcPayload { - method: "".to_string(), - params: Default::default(), - id: 0, - jsonrpc: "".to_string(), - signed_message: SignedMessage { - coin_ticker: "".to_string(), - address: "".to_string(), - timestamp_message: 0, - signature: "".to_string(), - }, - }; if is_private_ip { - return proxy(cfg, req, &remote_addr, temp, x_forwarded_for, proxy_route).await; + return proxy( + cfg, + req, + &remote_addr, + payload, + x_forwarded_for, + proxy_route, + ) + .await; } if let Err(status_code) = - validation_middleware(cfg, &temp, proxy_route, req.uri(), &remote_addr).await + validation_middleware(cfg, &payload, proxy_route, req.uri(), &remote_addr).await { return response_by_status(status_code); } - proxy(cfg, req, &remote_addr, temp, x_forwarded_for, proxy_route).await + proxy( + cfg, + req, + &remote_addr, + payload, + x_forwarded_for, + proxy_route, + ) + .await } #[test] diff --git a/src/net/server.rs b/src/net/server.rs index 6602d75..d2a6bbd 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -8,7 +8,7 @@ use hyper::{Body, Request, Response, Server, StatusCode, Uri}; use crate::address_status::AddressStatusOperations; use crate::ctx::ProxyRoute; use crate::db::Db; -use crate::http::{http_handler, response_by_status, JsonRpcPayload}; +use crate::http::{http_handler, response_by_status, JsonRpcPayload, PayloadData}; use crate::log_format; use crate::proof_of_funding::{verify_message_and_balance, ProofOfFundingError}; use crate::rate_limiter::RateLimitOperations; @@ -75,6 +75,24 @@ async fn connection_handler( // TODO handle eth and nft features pub(crate) async fn validation_middleware( + cfg: &AppConfig, + payload: &PayloadData, + proxy_route: &ProxyRoute, + req_uri: &Uri, + remote_addr: &SocketAddr, +) -> Result<(), StatusCode> { + match payload { + PayloadData::JsonRpc(payload) => { + validation_middleware_eth(cfg, payload, proxy_route, req_uri, remote_addr).await + } + PayloadData::HttpGet(_payload) => { + // TODO add validation_middleware for nft + Ok(()) + } + } +} + +pub(crate) async fn validation_middleware_eth( cfg: &AppConfig, payload: &JsonRpcPayload, proxy_route: &ProxyRoute, diff --git a/src/net/websocket.rs b/src/net/websocket.rs index e0a3a5a..cc670ec 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -12,7 +12,7 @@ use crate::{ ctx::AppConfig, http::{response_by_status, JsonRpcPayload}, log_format, - server::validation_middleware, + server::validation_middleware_eth, GenericResult, }; @@ -173,7 +173,8 @@ pub(crate) async fn socket_handler( continue; } - match validation_middleware( + // TODO add general validation_middleware support + match validation_middleware_eth( &cfg, &payload, &proxy_route, From 93d55637d25b0abfc7648e3cd6a788c8052530b7 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 2 May 2024 17:56:24 +0700 Subject: [PATCH 06/39] impl proxy_http_get --- src/net/http.rs | 139 +++++++++++++++++++++++++++++++-- src/net/rpc.rs | 4 +- src/net/server.rs | 6 +- src/security/address_status.rs | 5 +- 4 files changed, 140 insertions(+), 14 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index 1e910e4..7f618fc 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -17,6 +17,10 @@ use url::Url; use super::*; use crate::server::{is_private_ip, validation_middleware}; +/// Value +pub(crate) const APPLICATION_JSON: &str = "application/json"; +/// Header +pub(crate) const X_FORWARDED_FOR: &str = "x-forwarded-for"; async fn get_healthcheck() -> GenericResult> { let json = json!({ "health": "ok", @@ -27,7 +31,7 @@ async fn get_healthcheck() -> GenericResult> { .header("Access-Control-Allow-Origin", "*") .header("Access-Control-Allow-Headers", "*") .header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") - .header(header::CONTENT_TYPE, "application/json") + .header(header::CONTENT_TYPE, APPLICATION_JSON) .body(Body::from(json.to_string()))?) } @@ -156,8 +160,8 @@ async fn proxy( PayloadData::JsonRpc(payload) => { proxy_eth(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await } - PayloadData::HttpGet(_payload) => { - todo!() + PayloadData::HttpGet(payload) => { + proxy_http_get(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await } } } @@ -239,9 +243,130 @@ async fn proxy_eth( } req.headers_mut() - .insert(HeaderName::from_static("x-forwarded-for"), x_forwarded_for); + .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); req.headers_mut() - .insert(header::CONTENT_TYPE, "application/json".parse()?); + .insert(header::CONTENT_TYPE, APPLICATION_JSON.parse()?); + + let https = HttpsConnector::new(); + let client = hyper::Client::builder().build(https); + + let target_uri = req.uri().clone(); + let res = match client.request(req).await { + Ok(t) => t, + Err(_) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + original_req_uri, + "Couldn't reach {}, returning 503.", + target_uri + ) + ); + return response_by_status(StatusCode::SERVICE_UNAVAILABLE); + } + }; + + Ok(res) +} + +async fn proxy_http_get( + cfg: &AppConfig, + mut req: Request, + remote_addr: &SocketAddr, + payload: HttpGetPayload, + x_forwarded_for: HeaderValue, + proxy_route: &ProxyRoute, +) -> GenericResult> { + if proxy_route.authorized { + if let Err(e) = insert_jwt_to_http_header(cfg, req.headers_mut()).await { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req.uri(), + "Error inserting JWT into HTTP header: {}", + e + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + } + + let original_req_uri = req.uri().clone(); + + // Parse the intended outbound URL from the ProxyRoute configuration + let proxy_outbound_uri = match proxy_route.outbound_route.parse::() { + Ok(r) => r, + Err(_) => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + original_req_uri, + "Failed to parse outbound_route URL, returning 500." + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + }; + + // Check that the payload URL's Host and Port match the proxy route's outbound URL + if payload.url.host_str() == proxy_outbound_uri.host_str() + && payload.url.port_or_known_default() == proxy_outbound_uri.port_or_known_default() + { + match payload.url.as_str().parse() { + Ok(uri) => *req.uri_mut() = uri, + Err(e) => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + original_req_uri, + "Failed to parse URL to Uri: {}", + e + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + } + } else { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + original_req_uri, + "Mismatch between payload URL and configured outbound URL, returning 500." + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + + // drop hop headers + for key in &[ + header::ACCEPT_ENCODING, + header::CONNECTION, + header::HOST, + header::PROXY_AUTHENTICATE, + header::PROXY_AUTHORIZATION, + header::TE, + header::TRANSFER_ENCODING, + header::TRAILER, + header::UPGRADE, + header::HeaderName::from_static("keep-alive"), + ] { + req.headers_mut().remove(key); + } + + req.headers_mut() + .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); + req.headers_mut() + .insert(header::ACCEPT, APPLICATION_JSON.parse()?); let https = HttpsConnector::new(); let client = hyper::Client::builder().build(https); @@ -537,7 +662,7 @@ async fn test_parse_http_get_payload() { let mut req = Request::new(Body::from(serialized_payload)); req.headers_mut().insert( HeaderName::from_static("accept"), - "application/json".parse().unwrap(), + APPLICATION_JSON.parse().unwrap(), ); let (mut req, payload): (Request, HttpGetPayload) = parse_payload(req).await.unwrap(); @@ -561,5 +686,5 @@ async fn test_parse_http_get_payload() { }; assert_eq!(payload, expected_payload); - assert_eq!(header_value, "application/json"); + assert_eq!(header_value, APPLICATION_JSON); } diff --git a/src/net/rpc.rs b/src/net/rpc.rs index 932098b..3256311 100644 --- a/src/net/rpc.rs +++ b/src/net/rpc.rs @@ -1,6 +1,6 @@ use bytes::Buf; use ctx::AppConfig; -use http::insert_jwt_to_http_header; +use http::{insert_jwt_to_http_header, APPLICATION_JSON}; use hyper::{body::aggregate, header, Body, Request}; use hyper_tls::HttpsConnector; use serde_json::from_reader; @@ -27,7 +27,7 @@ impl RpcClient { ) -> GenericResult { let mut req = Request::post(&self.url).body(Body::from(payload.to_string()))?; req.headers_mut() - .append(header::CONTENT_TYPE, "application/json".parse()?); + .append(header::CONTENT_TYPE, APPLICATION_JSON.parse()?); if is_authorized { insert_jwt_to_http_header(cfg, req.headers_mut()).await?; diff --git a/src/net/server.rs b/src/net/server.rs index d2a6bbd..3b607b0 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -8,7 +8,7 @@ use hyper::{Body, Request, Response, Server, StatusCode, Uri}; use crate::address_status::AddressStatusOperations; use crate::ctx::ProxyRoute; use crate::db::Db; -use crate::http::{http_handler, response_by_status, JsonRpcPayload, PayloadData}; +use crate::http::{http_handler, response_by_status, JsonRpcPayload, PayloadData, X_FORWARDED_FOR}; use crate::log_format; use crate::proof_of_funding::{verify_message_and_balance, ProofOfFundingError}; use crate::rate_limiter::RateLimitOperations; @@ -30,7 +30,7 @@ pub(crate) fn is_private_ip(ip: &IpAddr) -> bool { } fn get_real_address(req: &Request, remote_addr: &SocketAddr) -> GenericResult { - if let Some(ip) = req.headers().get("x-forwarded-for") { + if let Some(ip) = req.headers().get(X_FORWARDED_FOR) { let addr = IpAddr::from_str(ip.to_str()?)?; return Ok(SocketAddr::new(addr, remote_addr.port())); @@ -228,7 +228,7 @@ fn test_get_real_address() { assert_eq!("127.0.0.1", remote_addr.ip().to_string()); req.headers_mut().insert( - hyper::header::HeaderName::from_static("x-forwarded-for"), + hyper::header::HeaderName::from_static(X_FORWARDED_FOR), "0.0.0.0".parse().unwrap(), ); diff --git a/src/security/address_status.rs b/src/security/address_status.rs index be29afd..3347e79 100644 --- a/src/security/address_status.rs +++ b/src/security/address_status.rs @@ -2,6 +2,7 @@ use async_trait::async_trait; use bytes::Buf; use ctx::AppConfig; use db::Db; +use http::APPLICATION_JSON; use hyper::{header, Body, Request, Response, StatusCode}; use redis::FromRedisValue; use serde::{Deserialize, Serialize}; @@ -28,7 +29,7 @@ pub(crate) async fn post_address_status( Ok(Response::builder() .status(StatusCode::NO_CONTENT) - .header(header::CONTENT_TYPE, "application/json") + .header(header::CONTENT_TYPE, APPLICATION_JSON) .body(Body::from(Vec::new()))?) } @@ -47,7 +48,7 @@ pub(crate) async fn get_address_status_list(cfg: &AppConfig) -> GenericResult Date: Fri, 3 May 2024 14:28:47 +0700 Subject: [PATCH 07/39] docker-compose.yml for local tests and app json config --- assets/conf_docker_test.json | 23 +++++++++++++++++++++++ docker-compose.yml | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 assets/conf_docker_test.json create mode 100644 docker-compose.yml diff --git a/assets/conf_docker_test.json b/assets/conf_docker_test.json new file mode 100644 index 0000000..2a4304d --- /dev/null +++ b/assets/conf_docker_test.json @@ -0,0 +1,23 @@ +{ + "port": 6150, + "pubkey_path": "/usr/src/komodo-defi-proxy/assets/.pubkey_test", + "privkey_path": "/usr/src/komodo-defi-proxy/assets/.privkey_test", + "redis_connection_string": "redis://redis:6379", + "token_expiration_time": 300, + "proxy_routes": [ + { + "inbound_route": "/nft", + "outbound_route": "https://nft.proxy", + "proxy_type": "http_get", + "authorized": false, + "allowed_methods": [] + } + ], + "rate_limiter": { + "rp_1_min": 30, + "rp_5_min": 100, + "rp_15_min": 200, + "rp_30_min": 350, + "rp_60_min": 575 + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..572b553 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +version: '3.8' +services: + redis: + image: redis:latest + restart: always + ports: + - "6379:6379" + + proxy: + build: + context: ./ + dockerfile: Containerfile + ports: + - "6150:6150" + depends_on: + - redis + environment: + AUTH_APP_CONFIG_PATH: /usr/src/komodo-defi-proxy/assets/conf_docker_test.json + volumes: + - ./assets:/usr/src/komodo-defi-proxy/assets From 3136f4bb9f6ce7f9884b718864b3aa7ca3c161dc Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 3 May 2024 21:13:04 +0700 Subject: [PATCH 08/39] impl modify_request_uri func --- src/net/http.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++- src/net/server.rs | 2 +- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index 7f618fc..df1ae75 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -1,11 +1,13 @@ use std::net::SocketAddr; +use std::str::FromStr; use address_status::{get_address_status_list, post_address_status}; use ctx::{AppConfig, ProxyRoute, ProxyType}; use hyper::header::HeaderName; +use hyper::http::uri::PathAndQuery; use hyper::{ header::{self, HeaderValue}, - Body, HeaderMap, Method, Request, Response, StatusCode, + Body, HeaderMap, Method, Request, Response, StatusCode, Uri, }; use hyper_tls::HttpsConnector; use jwt::{get_cached_token_or_generate_one, JwtClaims}; @@ -392,6 +394,33 @@ async fn proxy_http_get( Ok(res) } +#[allow(dead_code)] +/// Modifies the URI of an HTTP request to replace the base URI with the one specified in `ProxyRoute` +/// while preserving the original request's path and query parameters. +async fn modify_request_uri( + req: &mut Request, + proxy_route: &ProxyRoute, +) -> GenericResult { + let original_req_uri = req.uri().clone(); + + let mut proxy_outbound_parts = proxy_route.outbound_route.parse::()?.into_parts(); + + let path_and_query = PathAndQuery::from_str( + original_req_uri + .path_and_query() + .map_or("/", |pq| pq.as_str()), + )?; + // Append the path and query from the original request URI to the proxy outbound URI. + proxy_outbound_parts.path_and_query = Some(path_and_query); + + // Reconstruct the full URI with the updated parts. + let new_uri = Uri::from_parts(proxy_outbound_parts)?; + + // Update the request URI. + *req.uri_mut() = new_uri; + Ok(original_req_uri) +} + pub(crate) async fn http_handler( cfg: &AppConfig, req: Request, @@ -688,3 +717,28 @@ async fn test_parse_http_get_payload() { assert_eq!(payload, expected_payload); assert_eq!(header_value, APPLICATION_JSON); } + +#[tokio::test] +async fn test_modify_request_uri() { + let orig_uri_str = "https://proxy.example:3535/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false"; + let mut req = Request::builder() + .uri(orig_uri_str) + .body(Body::empty()) + .unwrap(); + + let proxy_route = ProxyRoute { + inbound_route: String::new(), + outbound_route: "http://localhost:8000".to_string(), + proxy_type: ProxyType::HttpGet, + authorized: false, + allowed_methods: vec![], + }; + + let returned_orig_uri = modify_request_uri(&mut req, &proxy_route).await.unwrap(); + + assert_eq!( + req.uri(), + "http://localhost:8000/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false" + ); + assert_eq!(returned_orig_uri, orig_uri_str); +} diff --git a/src/net/server.rs b/src/net/server.rs index 3b607b0..4c1e2b3 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -212,7 +212,7 @@ pub(crate) async fn serve(cfg: &'static AppConfig) -> GenericResult<()> { let server = Server::bind(&addr).serve(handler); - log::info!("Komodo-DeFi-Poxy API serving on http://{}", addr); + log::info!("Komodo-DeFi-Proxy API serving on http://{}", addr); Ok(server.await?) } From 84a1f62bf6eeea116335335eed465811696a2b1d Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 5 May 2024 10:11:13 +0700 Subject: [PATCH 09/39] use modify_request_uri func in proxy_http_get --- src/net/http.rs | 57 ++++++++----------------------------------------- 1 file changed, 9 insertions(+), 48 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index df1ae75..300bab0 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -289,7 +289,7 @@ async fn proxy_http_get( remote_addr.ip(), payload.signed_message.address, req.uri(), - "Error inserting JWT into HTTP header: {}", + "Error inserting JWT into HTTP header: {}, returning 500.", e ) ); @@ -299,51 +299,15 @@ async fn proxy_http_get( let original_req_uri = req.uri().clone(); - // Parse the intended outbound URL from the ProxyRoute configuration - let proxy_outbound_uri = match proxy_route.outbound_route.parse::() { - Ok(r) => r, - Err(_) => { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - original_req_uri, - "Failed to parse outbound_route URL, returning 500." - ) - ); - return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); - } - }; - - // Check that the payload URL's Host and Port match the proxy route's outbound URL - if payload.url.host_str() == proxy_outbound_uri.host_str() - && payload.url.port_or_known_default() == proxy_outbound_uri.port_or_known_default() - { - match payload.url.as_str().parse() { - Ok(uri) => *req.uri_mut() = uri, - Err(e) => { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - original_req_uri, - "Failed to parse URL to Uri: {}", - e - ) - ); - return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); - } - } - } else { + if let Err(e) = modify_request_uri(&mut req, proxy_route).await { log::error!( "{}", log_format!( remote_addr.ip(), payload.signed_message.address, original_req_uri, - "Mismatch between payload URL and configured outbound URL, returning 500." + "Error modifying request Uri: {}, returning 500.", + e ) ); return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); @@ -394,19 +358,17 @@ async fn proxy_http_get( Ok(res) } -#[allow(dead_code)] /// Modifies the URI of an HTTP request to replace the base URI with the one specified in `ProxyRoute` /// while preserving the original request's path and query parameters. async fn modify_request_uri( req: &mut Request, proxy_route: &ProxyRoute, -) -> GenericResult { - let original_req_uri = req.uri().clone(); - +) -> GenericResult<()> { let mut proxy_outbound_parts = proxy_route.outbound_route.parse::()?.into_parts(); let path_and_query = PathAndQuery::from_str( - original_req_uri + req.uri() + .clone() .path_and_query() .map_or("/", |pq| pq.as_str()), )?; @@ -418,7 +380,7 @@ async fn modify_request_uri( // Update the request URI. *req.uri_mut() = new_uri; - Ok(original_req_uri) + Ok(()) } pub(crate) async fn http_handler( @@ -734,11 +696,10 @@ async fn test_modify_request_uri() { allowed_methods: vec![], }; - let returned_orig_uri = modify_request_uri(&mut req, &proxy_route).await.unwrap(); + modify_request_uri(&mut req, &proxy_route).await.unwrap(); assert_eq!( req.uri(), "http://localhost:8000/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false" ); - assert_eq!(returned_orig_uri, orig_uri_str); } From d980f642582db8aec8326c596e23db47d20467d5 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 5 May 2024 10:18:48 +0700 Subject: [PATCH 10/39] remove `url` dependency, as for modify_request_uri functionality Uri type is enough --- Cargo.lock | 2 -- Cargo.toml | 1 - src/net/http.rs | 8 ++------ 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d86739..f20b020 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,7 +717,6 @@ dependencies = [ "simple_logger", "tokio", "tokio-tungstenite", - "url", ] [[package]] @@ -1701,7 +1700,6 @@ dependencies = [ "idna", "matches", "percent-encoding", - "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c9ff2d2..92172dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,6 @@ tokio-tungstenite = { version = "0.20.0", features = ["native-tls"] } bitcrypto = { git = "https://github.com/KomodoPlatform/atomicDEX-API", branch = "dev" } ethkey = { git = "https://github.com/artemii235/parity-ethereum.git" } serialization = { git = "https://github.com/KomodoPlatform/atomicDEX-API", branch = "dev" } -url = { version = "2.2.2", features = ["serde"] } [target.x86_64-unknown-linux-gnu.dependencies] jemallocator = "0.5.0" diff --git a/src/net/http.rs b/src/net/http.rs index 300bab0..d643043 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -14,7 +14,6 @@ use jwt::{get_cached_token_or_generate_one, JwtClaims}; use serde::{Deserialize, Serialize}; use serde_json::json; use sign::SignedMessage; -use url::Url; use super::*; use crate::server::{is_private_ip, validation_middleware}; @@ -109,7 +108,6 @@ pub(crate) struct JsonRpcPayload { /// that the proxy will forward the GET request to, along with a `SignedMessage` for authentication and validation. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct HttpGetPayload { - pub(crate) url: Url, pub(crate) signed_message: SignedMessage, } @@ -562,7 +560,7 @@ fn test_get_proxy_route_by_inbound() { assert_eq!(proxy_route.outbound_route, "https://atomicdex.io"); - let url = Url::from_str("https://komodo.proxy:5535/nft-test").unwrap(); + let url = Uri::from_str("https://komodo.proxy:5535/nft-test").unwrap(); let path = url.path().to_string(); let proxy_route = cfg.get_proxy_route_by_inbound(path).unwrap(); assert_eq!(proxy_route.outbound_route, "https://nft.proxy"); @@ -638,9 +636,8 @@ async fn test_parse_json_rpc_payload() { #[tokio::test] async fn test_parse_http_get_payload() { - let url_string = "http://example.com"; let serialized_payload = json!({ - "url": url_string, + "url": "http://example.com", "signed_message": { "coin_ticker": "BTC", "address": "dummy-value", @@ -667,7 +664,6 @@ async fn test_parse_http_get_payload() { let header_value = req.headers().get("accept").unwrap(); let expected_payload = HttpGetPayload { - url: Url::parse(url_string).unwrap(), signed_message: SignedMessage { coin_ticker: String::from("BTC"), address: String::from("dummy-value"), From 69fe4b89469332f21b235e8d2bec0b729346bb0a Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 6 May 2024 17:25:23 +0700 Subject: [PATCH 11/39] return Url type in HttpGetPayload. we need it for Deserialize, Serialize, and for general impl path in req uri have to represent only inbound_route --- Cargo.lock | 2 ++ Cargo.toml | 1 + src/net/http.rs | 44 +++++++++++++++++++++++++++++--------------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f20b020..8d86739 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,6 +717,7 @@ dependencies = [ "simple_logger", "tokio", "tokio-tungstenite", + "url", ] [[package]] @@ -1700,6 +1701,7 @@ dependencies = [ "idna", "matches", "percent-encoding", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 92172dd..c9ff2d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ tokio-tungstenite = { version = "0.20.0", features = ["native-tls"] } bitcrypto = { git = "https://github.com/KomodoPlatform/atomicDEX-API", branch = "dev" } ethkey = { git = "https://github.com/artemii235/parity-ethereum.git" } serialization = { git = "https://github.com/KomodoPlatform/atomicDEX-API", branch = "dev" } +url = { version = "2.2.2", features = ["serde"] } [target.x86_64-unknown-linux-gnu.dependencies] jemallocator = "0.5.0" diff --git a/src/net/http.rs b/src/net/http.rs index d643043..86f6d07 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -14,6 +14,7 @@ use jwt::{get_cached_token_or_generate_one, JwtClaims}; use serde::{Deserialize, Serialize}; use serde_json::json; use sign::SignedMessage; +use url::Url; use super::*; use crate::server::{is_private_ip, validation_middleware}; @@ -95,7 +96,7 @@ where /// Represents a JSON RPC payload parsed from a proxy request. It combines standard JSON RPC method call /// fields with a `SignedMessage` for authentication and validation by the proxy. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub(crate) struct JsonRpcPayload { pub(crate) method: String, pub(crate) params: serde_json::value::Value, @@ -106,8 +107,9 @@ pub(crate) struct JsonRpcPayload { /// Represents a payload for HTTP GET request parsed from a proxy request. This struct contains the URL /// that the proxy will forward the GET request to, along with a `SignedMessage` for authentication and validation. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub(crate) struct HttpGetPayload { + uri: Url, pub(crate) signed_message: SignedMessage, } @@ -297,7 +299,7 @@ async fn proxy_http_get( let original_req_uri = req.uri().clone(); - if let Err(e) = modify_request_uri(&mut req, proxy_route).await { + if let Err(e) = modify_request_uri(&mut req, &payload, proxy_route).await { log::error!( "{}", log_format!( @@ -356,20 +358,19 @@ async fn proxy_http_get( Ok(res) } -/// Modifies the URI of an HTTP request to replace the base URI with the one specified in `ProxyRoute` -/// while preserving the original request's path and query parameters. +/// Modifies the URI of an HTTP request by replacing it to outbound URI specified in `ProxyRoute`, +/// while incorporating the path and query parameters from the payload's URI. async fn modify_request_uri( req: &mut Request, + payload: &HttpGetPayload, proxy_route: &ProxyRoute, ) -> GenericResult<()> { let mut proxy_outbound_parts = proxy_route.outbound_route.parse::()?.into_parts(); - let path_and_query = PathAndQuery::from_str( - req.uri() - .clone() - .path_and_query() - .map_or("/", |pq| pq.as_str()), - )?; + let payload_uri: Uri = payload.uri.as_str().parse()?; + + let path_and_query = + PathAndQuery::from_str(payload_uri.path_and_query().map_or("/", |pq| pq.as_str()))?; // Append the path and query from the original request URI to the proxy outbound URI. proxy_outbound_parts.path_and_query = Some(path_and_query); @@ -637,7 +638,7 @@ async fn test_parse_json_rpc_payload() { #[tokio::test] async fn test_parse_http_get_payload() { let serialized_payload = json!({ - "url": "http://example.com", + "uri": "https://example.com/test-path", "signed_message": { "coin_ticker": "BTC", "address": "dummy-value", @@ -664,6 +665,7 @@ async fn test_parse_http_get_payload() { let header_value = req.headers().get("accept").unwrap(); let expected_payload = HttpGetPayload { + uri: Url::from_str("https://example.com/test-path").unwrap(), signed_message: SignedMessage { coin_ticker: String::from("BTC"), address: String::from("dummy-value"), @@ -678,21 +680,33 @@ async fn test_parse_http_get_payload() { #[tokio::test] async fn test_modify_request_uri() { - let orig_uri_str = "https://proxy.example:3535/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false"; + let orig_uri_str = "https://proxy.example:3535/test-inbound"; let mut req = Request::builder() .uri(orig_uri_str) .body(Body::empty()) .unwrap(); + let payload = HttpGetPayload { + uri: Url::from_str("https://proxy.example:3535/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false").unwrap(), + signed_message: SignedMessage { + coin_ticker: String::from("BTC"), + address: String::from("dummy-value"), + timestamp_message: 1655320000, + signature: String::from("dummy-value"), + }, + }; + let proxy_route = ProxyRoute { - inbound_route: String::new(), + inbound_route: String::from_str("/test-inbound").unwrap(), outbound_route: "http://localhost:8000".to_string(), proxy_type: ProxyType::HttpGet, authorized: false, allowed_methods: vec![], }; - modify_request_uri(&mut req, &proxy_route).await.unwrap(); + modify_request_uri(&mut req, &payload, &proxy_route) + .await + .unwrap(); assert_eq!( req.uri(), From e022f406fa3fa369697a9fd9d6d4b09ee27ba043 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 6 May 2024 17:50:47 +0700 Subject: [PATCH 12/39] update note in modify_request_uri --- src/net/http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/http.rs b/src/net/http.rs index 86f6d07..bfb5c40 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -371,7 +371,7 @@ async fn modify_request_uri( let path_and_query = PathAndQuery::from_str(payload_uri.path_and_query().map_or("/", |pq| pq.as_str()))?; - // Append the path and query from the original request URI to the proxy outbound URI. + // Append the path and query from the payload URI to the proxy outbound URI. proxy_outbound_parts.path_and_query = Some(path_and_query); // Reconstruct the full URI with the updated parts. From 412739eae7c885ab3ebd6b591d3b3450945e1cbd Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 6 May 2024 21:18:35 +0700 Subject: [PATCH 13/39] update log name in macro_rules! log_format --- src/net/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/server.rs b/src/net/server.rs index 4c1e2b3..6388b62 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -18,7 +18,7 @@ use crate::{ctx::AppConfig, GenericError, GenericResult}; #[macro_export] macro_rules! log_format { ($ip: expr, $address: expr, $path: expr, $format: expr, $($args: tt)+) => {format!(concat!("[Ip: {} | Address: {} | Path: {}] ", $format), $ip, $address, $path, $($args)+)}; - ($ip: expr, $address: expr, $path: expr, $format: expr) => {format!(concat!("[Ip: {} | Pubkey: {} | Address: {}] ", $format), $ip, $address, $path)} + ($ip: expr, $address: expr, $path: expr, $format: expr) => {format!(concat!("[Ip: {} | Address: {} | Path: {}] ", $format), $ip, $address, $path)} } pub(crate) fn is_private_ip(ip: &IpAddr) -> bool { From bca233810c8f5399bfccfe186dfe7a49152be4fc Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 7 May 2024 09:45:02 +0700 Subject: [PATCH 14/39] provide error message in logs --- src/net/http.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index bfb5c40..4b27dad 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -213,15 +213,16 @@ async fn proxy_eth( let original_req_uri = req.uri().clone(); *req.uri_mut() = match proxy_route.outbound_route.parse() { Ok(uri) => uri, - Err(_) => { + Err(e) => { log::error!( "{}", log_format!( remote_addr.ip(), payload.signed_message.address, original_req_uri, - "Error type casting value of {} into Uri, returning 500.", - proxy_route.outbound_route + "Error type casting value of {} into Uri: {}, returning 500.", + proxy_route.outbound_route, + e ) ); return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); @@ -255,15 +256,16 @@ async fn proxy_eth( let target_uri = req.uri().clone(); let res = match client.request(req).await { Ok(t) => t, - Err(_) => { + Err(e) => { log::warn!( "{}", log_format!( remote_addr.ip(), payload.signed_message.address, original_req_uri, - "Couldn't reach {}, returning 503.", - target_uri + "Couldn't reach {}: {}. Returning 503.", + target_uri, + e ) ); return response_by_status(StatusCode::SERVICE_UNAVAILABLE); @@ -340,15 +342,16 @@ async fn proxy_http_get( let target_uri = req.uri().clone(); let res = match client.request(req).await { Ok(t) => t, - Err(_) => { + Err(e) => { log::warn!( "{}", log_format!( remote_addr.ip(), payload.signed_message.address, original_req_uri, - "Couldn't reach {}, returning 503.", - target_uri + "Couldn't reach {}: {}. Returning 503.", + target_uri, + e ) ); return response_by_status(StatusCode::SERVICE_UNAVAILABLE); From 80d92fbe1714c5052628b80d90c5a994640e3762 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 7 May 2024 11:27:34 +0700 Subject: [PATCH 15/39] remove header::CONTENT_LENGTH from http get request --- src/net/http.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/net/http.rs b/src/net/http.rs index 4b27dad..9844457 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -326,6 +326,7 @@ async fn proxy_http_get( header::TRANSFER_ENCODING, header::TRAILER, header::UPGRADE, + header::CONTENT_LENGTH, header::HeaderName::from_static("keep-alive"), ] { req.headers_mut().remove(key); From 0294f19b94cb0547d968948ba01a79494965de40 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 8 May 2024 14:27:58 +0700 Subject: [PATCH 16/39] simplify upsert_address_rate_in_pipe func, impl validation_middleware_http_get, have general validation_middleware func which handle all PayloadData types --- src/db.rs | 1 - src/net/http.rs | 1 - src/net/server.rs | 128 ++++++++++++++++++++++++++++++++--- src/net/websocket.rs | 4 +- src/security/rate_limiter.rs | 25 ++----- 5 files changed, 125 insertions(+), 34 deletions(-) diff --git a/src/db.rs b/src/db.rs index 6660ee8..9f8736f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -36,7 +36,6 @@ impl Db { } } - #[allow(dead_code)] pub(crate) async fn key_exists(&mut self, key: &str) -> GenericResult { Ok(redis::cmd("EXISTS") .arg(key) diff --git a/src/net/http.rs b/src/net/http.rs index 9844457..c6b00b3 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -149,7 +149,6 @@ async fn generate_payload_from_req( } } -// TODO handle eth and nft features async fn proxy( cfg: &AppConfig, req: Request, diff --git a/src/net/server.rs b/src/net/server.rs index 6388b62..feea671 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -5,13 +5,16 @@ use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server, StatusCode, Uri}; -use crate::address_status::AddressStatusOperations; +use crate::address_status::{AddressStatus, AddressStatusOperations}; use crate::ctx::ProxyRoute; use crate::db::Db; -use crate::http::{http_handler, response_by_status, JsonRpcPayload, PayloadData, X_FORWARDED_FOR}; +use crate::http::{ + http_handler, response_by_status, HttpGetPayload, JsonRpcPayload, PayloadData, X_FORWARDED_FOR, +}; use crate::log_format; use crate::proof_of_funding::{verify_message_and_balance, ProofOfFundingError}; use crate::rate_limiter::RateLimitOperations; +use crate::sign::SignOps; use crate::websocket::{should_upgrade_to_socket_conn, socket_handler}; use crate::{ctx::AppConfig, GenericError, GenericResult}; @@ -83,16 +86,15 @@ pub(crate) async fn validation_middleware( ) -> Result<(), StatusCode> { match payload { PayloadData::JsonRpc(payload) => { - validation_middleware_eth(cfg, payload, proxy_route, req_uri, remote_addr).await + validation_middleware_json_rpc(cfg, payload, proxy_route, req_uri, remote_addr).await } - PayloadData::HttpGet(_payload) => { - // TODO add validation_middleware for nft - Ok(()) + PayloadData::HttpGet(payload) => { + validation_middleware_http_get(cfg, payload, proxy_route, req_uri, remote_addr).await } } } -pub(crate) async fn validation_middleware_eth( +pub(crate) async fn validation_middleware_json_rpc( cfg: &AppConfig, payload: &JsonRpcPayload, proxy_route: &ProxyRoute, @@ -105,9 +107,9 @@ pub(crate) async fn validation_middleware_eth( .read_address_status(&payload.signed_message.address) .await { - crate::address_status::AddressStatus::Trusted => Ok(()), - crate::address_status::AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), - crate::address_status::AddressStatus::None => { + AddressStatus::Trusted => Ok(()), + AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), + AddressStatus::None => { let signed_message_status = verify_message_and_balance(cfg, payload, proxy_route).await; if let Err(ProofOfFundingError::InvalidSignedMessage) = signed_message_status { @@ -129,6 +131,7 @@ pub(crate) async fn validation_middleware_eth( payload.signed_message.coin_ticker, payload.signed_message.address ); + // TODO impl Optional rate limiter in ProxyRoute type and use it. if None, use cfg.rate_limiter as Default match db.rate_exceeded(&rate_limiter_key, &cfg.rate_limiter).await { Ok(false) => {} _ => { @@ -154,8 +157,8 @@ pub(crate) async fn validation_middleware_eth( payload.signed_message.address, req_uri, "Wallet {} has insufficient balance for coin {}, returning 406.", + payload.signed_message.address, payload.signed_message.coin_ticker, - payload.signed_message.address ) ); @@ -196,6 +199,109 @@ pub(crate) async fn validation_middleware_eth( } } +pub(crate) async fn validation_middleware_http_get( + cfg: &AppConfig, + payload: &HttpGetPayload, + _proxy_route: &ProxyRoute, + req_uri: &Uri, + remote_addr: &SocketAddr, +) -> Result<(), StatusCode> { + let mut db = Db::create_instance(cfg).await; + + match db + .read_address_status(&payload.signed_message.address) + .await + { + AddressStatus::Trusted => Ok(()), + AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), + AddressStatus::None => { + match payload.signed_message.verify_message() { + Ok(true) => {} + Ok(false) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Request has invalid signed message, returning 401" + ) + ); + + return Err(StatusCode::UNAUTHORIZED); + } + Err(e) => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "verify_message failed in coin {}: {}, returning 500.", + payload.signed_message.coin_ticker, + e + ) + ); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + + let rate_limiter_key = format!( + "{}:{}", + payload.signed_message.coin_ticker, payload.signed_message.address + ); + + // TODO impl Optional rate limiter in ProxyRoute type and use it. if None, use cfg.rate_limiter as Default + match db.rate_exceeded(&rate_limiter_key, &cfg.rate_limiter).await { + Ok(false) => {} + Ok(true) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Rate exceed for {}, returning 406.", + rate_limiter_key, + ) + ); + } + Err(e) => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Rate exceeded check failed in coin {}: {}, returning 500.", + payload.signed_message.coin_ticker, + e + ) + ); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + + if let Err(e) = db.rate_address(rate_limiter_key).await { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Rate incrementing failed in coin {}: {}, returning 500.", + payload.signed_message.coin_ticker, + e + ) + ); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(()) + } + } +} + /// Starts serving the proxy API on the configured port. This function sets up the HTTP server, /// binds it to the specified address, and listens for incoming requests. pub(crate) async fn serve(cfg: &'static AppConfig) -> GenericResult<()> { diff --git a/src/net/websocket.rs b/src/net/websocket.rs index cc670ec..5999e98 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -12,7 +12,7 @@ use crate::{ ctx::AppConfig, http::{response_by_status, JsonRpcPayload}, log_format, - server::validation_middleware_eth, + server::validation_middleware_json_rpc, GenericResult, }; @@ -174,7 +174,7 @@ pub(crate) async fn socket_handler( } // TODO add general validation_middleware support - match validation_middleware_eth( + match validation_middleware_json_rpc( &cfg, &payload, &proxy_route, diff --git a/src/security/rate_limiter.rs b/src/security/rate_limiter.rs index 653304d..3cc9161 100644 --- a/src/security/rate_limiter.rs +++ b/src/security/rate_limiter.rs @@ -43,25 +43,12 @@ impl RateLimitOperations for Db { address: &str, expire_time: usize, ) -> GenericResult<()> { - if !self.key_exists(db).await? { - pipe.hset(db, address, "1") - .cmd("EXPIRE") - .arg(db) - .arg(expire_time) - .arg("XX") - .query_async(&mut self.connection) - .await?; - } else { - pipe.cmd("HINCRBY") - .arg(db) - .arg(&[address, "1"]) - .cmd("EXPIRE") - .arg(db) - .arg(expire_time) - .arg("XX") - .query_async(&mut self.connection) - .await?; - } + // Atomic operation, which means it increments the value safely even when multiple clients are modifying the counter simultaneously. + pipe.atomic() + .hincr(db, address, 1) // Increment the hash value or create it if it doesn't exist + .expire(db, expire_time) // Set or reset the expiration time + .query_async(&mut self.connection) + .await?; Ok(()) } From cacce87c8eb6d2324c2645df1bde512316a714e7 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 8 May 2024 14:31:49 +0700 Subject: [PATCH 17/39] rename proxy_eth to proxy_json_rpc --- src/net/http.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index c6b00b3..7440c5a 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -159,7 +159,7 @@ async fn proxy( ) -> GenericResult> { match payload { PayloadData::JsonRpc(payload) => { - proxy_eth(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await + proxy_json_rpc(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await } PayloadData::HttpGet(payload) => { proxy_http_get(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await @@ -167,7 +167,7 @@ async fn proxy( } } -async fn proxy_eth( +async fn proxy_json_rpc( cfg: &AppConfig, mut req: Request, remote_addr: &SocketAddr, From 8a201da2ff32c4b323c80e4ede35f33bcec007fc Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 12 May 2024 16:53:51 +0700 Subject: [PATCH 18/39] add get_req: bool param into parse_payload function --- src/ctx.rs | 8 ++++---- src/net/http.rs | 27 +++++++++++++++------------ 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/ctx.rs b/src/ctx.rs index 7900d0d..6f9f55b 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -41,7 +41,7 @@ pub(crate) struct ProxyRoute { pub(crate) inbound_route: String, // The target URL to which requests are forwarded. pub(crate) outbound_route: String, - // The type of proxying to perform (e.g., JSON RPC, HTTP GET). + // The type of proxying to perform (e.g., JSON-RPC Call, HTTP GET). pub(crate) proxy_type: ProxyType, // Whether authorization is required for this route. #[serde(default)] @@ -51,13 +51,13 @@ pub(crate) struct ProxyRoute { pub(crate) allowed_methods: Vec, } -/// Enumerates different types of proxy operations supported, such as JSON RPC and HTTP GET. +/// Enumerates different types of proxy operations supported, such as JSON-RPC Call over HTTP POST and HTTP GET. /// This helps in applying specific handling logic based on the proxy type. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] pub(crate) enum ProxyType { - JsonRpc, - HttpGet, + JsonRpc, // JSON-RPC call using HTTP POST + HttpGet, // Standard HTTP GET request } /// Configuration for rate limiting to manage the number of requests allowed over specified time intervals. diff --git a/src/net/http.rs b/src/net/http.rs index 7440c5a..3c31468 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -69,14 +69,14 @@ pub(crate) async fn insert_jwt_to_http_header( Ok(()) } -/// Asynchronously parses an HTTP request's body into a specified type `T`, modifying the request -/// to have an empty body if the method is `GET`, and returning the original body otherwise. -/// Ensures that the body is not empty before attempting deserialization into the non-optional type `T`. -async fn parse_payload(req: Request) -> GenericResult<(Request, T)> +/// Asynchronously parses an HTTP request's body into a specified type `T`. If the request method is `GET`, +/// the function modifies the request to have an empty body. For other methods, it retains the original body. +/// The function ensures that the body is not empty before attempting deserialization into the non-optional type `T`. +async fn parse_payload(req: Request, get_req: bool) -> GenericResult<(Request, T)> where T: serde::de::DeserializeOwned, { - let (parts, body) = req.into_parts(); + let (mut parts, body) = req.into_parts(); let body_bytes = hyper::body::to_bytes(body).await?; if body_bytes.is_empty() { @@ -85,7 +85,8 @@ where let payload: T = serde_json::from_slice(&body_bytes)?; - let new_req = if parts.method == Method::GET { + let new_req = if get_req { + parts.method = Method::GET; Request::from_parts(parts, Body::empty()) } else { Request::from_parts(parts, Body::from(body_bytes)) @@ -94,7 +95,7 @@ where Ok((new_req, payload)) } -/// Represents a JSON RPC payload parsed from a proxy request. It combines standard JSON RPC method call +/// Represents a JSON-RPC Call payload parsed from a proxy request. It combines standard JSON RPC method call /// fields with a `SignedMessage` for authentication and validation by the proxy. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub(crate) struct JsonRpcPayload { @@ -139,11 +140,11 @@ async fn generate_payload_from_req( ) -> GenericResult<(Request, PayloadData)> { match proxy_type { ProxyType::JsonRpc => { - let (req, payload) = parse_payload::(req).await?; + let (req, payload) = parse_payload::(req, false).await?; Ok((req, PayloadData::JsonRpc(payload))) } ProxyType::HttpGet => { - let (req, payload) = parse_payload::(req).await?; + let (req, payload) = parse_payload::(req, true).await?; Ok((req, PayloadData::HttpGet(payload))) } } @@ -334,7 +335,7 @@ async fn proxy_http_get( req.headers_mut() .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); req.headers_mut() - .insert(header::ACCEPT, APPLICATION_JSON.parse()?); + .insert(header::CONTENT_TYPE, APPLICATION_JSON.parse()?); let https = HttpsConnector::new(); let client = hyper::Client::builder().build(https); @@ -611,7 +612,8 @@ async fn test_parse_json_rpc_payload() { "dummy-value".parse().unwrap(), ); - let (mut req, payload): (Request, JsonRpcPayload) = parse_payload(req).await.unwrap(); + let (mut req, payload): (Request, JsonRpcPayload) = + parse_payload(req, false).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( @@ -657,7 +659,8 @@ async fn test_parse_http_get_payload() { APPLICATION_JSON.parse().unwrap(), ); - let (mut req, payload): (Request, HttpGetPayload) = parse_payload(req).await.unwrap(); + let (mut req, payload): (Request, HttpGetPayload) = + parse_payload(req, true).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( From 8bd9d87c96d69a354b8bf8f6c722a5d35778ed19 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 13 May 2024 16:25:33 +0700 Subject: [PATCH 19/39] provide optional rate_limiter field in ProxyRoute, return NOT_ACCEPTABLE error if rate exceed for HTTP GET request --- assets/.config_test | 15 ++++++++--- assets/conf_docker_test.json | 9 ++++++- src/ctx.rs | 51 +++++++++++++++++++++++++----------- src/net/http.rs | 1 + src/net/server.rs | 22 ++++++++++------ 5 files changed, 71 insertions(+), 27 deletions(-) diff --git a/assets/.config_test b/assets/.config_test index 657fddb..eeb763a 100644 --- a/assets/.config_test +++ b/assets/.config_test @@ -10,21 +10,30 @@ "outbound_route": "https://komodoplatform.com", "proxy_type": "json_rpc", "authorized": false, - "allowed_methods": [] + "allowed_methods": [], + "rate_limiter": null }, { "inbound_route": "/test-2", "outbound_route": "https://atomicdex.io", "proxy_type": "json_rpc", "authorized": false, - "allowed_methods": [] + "allowed_methods": [], + "rate_limiter": null }, { "inbound_route": "/nft-test", "outbound_route": "https://nft.proxy", "proxy_type": "http_get", "authorized": false, - "allowed_methods": [] + "allowed_methods": [], + "rate_limiter": { + "rp_1_min": 60, + "rp_5_min": 60, + "rp_15_min": 60, + "rp_30_min": 60, + "rp_60_min": 60 + } } ], "rate_limiter": { diff --git a/assets/conf_docker_test.json b/assets/conf_docker_test.json index 2a4304d..59a2775 100644 --- a/assets/conf_docker_test.json +++ b/assets/conf_docker_test.json @@ -10,7 +10,14 @@ "outbound_route": "https://nft.proxy", "proxy_type": "http_get", "authorized": false, - "allowed_methods": [] + "allowed_methods": [], + "rate_limiter": { + "rp_1_min": 60, + "rp_5_min": 250, + "rp_15_min": 800, + "rp_30_min": 1500, + "rp_60_min": 3000 + } } ], "rate_limiter": { diff --git a/src/ctx.rs b/src/ctx.rs index 6f9f55b..61f6d13 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -17,19 +17,19 @@ pub(crate) fn get_app_config() -> &'static AppConfig { /// Configuration settings for the application, loaded typically from a JSON configuration file. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct AppConfig { - // Optional server port to listen on. If None in config file, then 5000 is default. + /// Optional server port to listen on. If None in config file, then 5000 is default. pub(crate) port: Option, - // Redis database connection string. + /// Redis database connection string. pub(crate) redis_connection_string: String, - // File path to the public key used for cryptographic operations. + /// File path to the public key used for cryptographic operations. pub(crate) pubkey_path: String, - // File path to the private key used for cryptographic operations. + /// File path to the private key used for cryptographic operations. pub(crate) privkey_path: String, - // Optional token expiration time in seconds. If None then 3600 is default. + /// Optional token expiration time in seconds. If None then 3600 is default. pub(crate) token_expiration_time: Option, - // Routing configurations for proxying requests. + /// Routing configurations for proxying requests. pub(crate) proxy_routes: Vec, - // Rate limiting settings for request handling. + /// Default rate limiting settings for request handling. pub(crate) rate_limiter: RateLimiter, } @@ -37,18 +37,21 @@ pub(crate) struct AppConfig { /// based on a specified proxy type and additional authorization and method filtering criteria. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct ProxyRoute { - // The incoming route pattern. + /// The incoming route pattern. pub(crate) inbound_route: String, - // The target URL to which requests are forwarded. + /// The target URL to which requests are forwarded. pub(crate) outbound_route: String, - // The type of proxying to perform (e.g., JSON-RPC Call, HTTP GET). + /// The type of proxying to perform (e.g., JSON-RPC Call, HTTP GET). pub(crate) proxy_type: ProxyType, - // Whether authorization is required for this route. + /// Whether authorization is required for this route. #[serde(default)] pub(crate) authorized: bool, // Specific HTTP methods allowed for this route. #[serde(default)] pub(crate) allowed_methods: Vec, + /// Optional custom rate limiter configuration for this route. If provided, + /// this configuration will be used instead of the default rate limiting settings. + pub(crate) rate_limiter: Option, } /// Enumerates different types of proxy operations supported, such as JSON-RPC Call over HTTP POST and HTTP GET. @@ -112,6 +115,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { proxy_type: ProxyType::JsonRpc, authorized: false, allowed_methods: Vec::default(), + rate_limiter: None, }, ProxyRoute { inbound_route: String::from("/test-2"), @@ -119,6 +123,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { proxy_type: ProxyType::JsonRpc, authorized: false, allowed_methods: Vec::default(), + rate_limiter: None, }, ProxyRoute { inbound_route: String::from("/nft-test"), @@ -126,9 +131,16 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { proxy_type: ProxyType::HttpGet, authorized: false, allowed_methods: Vec::default(), + rate_limiter: Some(RateLimiter { + rp_1_min: 60, + rp_5_min: 60, + rp_15_min: 60, + rp_30_min: 60, + rp_60_min: 60, + }), }, ]), - rate_limiter: ctx::RateLimiter { + rate_limiter: RateLimiter { rp_1_min: 555, rp_5_min: 555, rp_15_min: 555, @@ -152,21 +164,30 @@ fn test_app_config_serialzation_and_deserialization() { "outbound_route": "https://komodoplatform.com", "proxy_type":"json_rpc", "authorized": false, - "allowed_methods": [] + "allowed_methods": [], + "rate_limiter": null }, { "inbound_route": "/test-2", "outbound_route": "https://atomicdex.io", "proxy_type":"json_rpc", "authorized": false, - "allowed_methods": [] + "allowed_methods": [], + "rate_limiter": null }, { "inbound_route": "/nft-test", "outbound_route": "https://nft.proxy", "proxy_type":"http_get", "authorized": false, - "allowed_methods": [] + "allowed_methods": [], + "rate_limiter": { + "rp_1_min": 60, + "rp_5_min": 60, + "rp_15_min": 60, + "rp_30_min": 60, + "rp_60_min": 60 + } } ], "rate_limiter": { diff --git a/src/net/http.rs b/src/net/http.rs index 3c31468..76987e2 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -708,6 +708,7 @@ async fn test_modify_request_uri() { proxy_type: ProxyType::HttpGet, authorized: false, allowed_methods: vec![], + rate_limiter: None, }; modify_request_uri(&mut req, &payload, &proxy_route) diff --git a/src/net/server.rs b/src/net/server.rs index feea671..8ff550a 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -5,8 +5,9 @@ use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server, StatusCode, Uri}; +use super::{GenericError, GenericResult}; use crate::address_status::{AddressStatus, AddressStatusOperations}; -use crate::ctx::ProxyRoute; +use crate::ctx::{AppConfig, ProxyRoute}; use crate::db::Db; use crate::http::{ http_handler, response_by_status, HttpGetPayload, JsonRpcPayload, PayloadData, X_FORWARDED_FOR, @@ -16,7 +17,6 @@ use crate::proof_of_funding::{verify_message_and_balance, ProofOfFundingError}; use crate::rate_limiter::RateLimitOperations; use crate::sign::SignOps; use crate::websocket::{should_upgrade_to_socket_conn, socket_handler}; -use crate::{ctx::AppConfig, GenericError, GenericResult}; #[macro_export] macro_rules! log_format { @@ -76,7 +76,6 @@ async fn connection_handler( } } -// TODO handle eth and nft features pub(crate) async fn validation_middleware( cfg: &AppConfig, payload: &PayloadData, @@ -131,8 +130,11 @@ pub(crate) async fn validation_middleware_json_rpc( payload.signed_message.coin_ticker, payload.signed_message.address ); - // TODO impl Optional rate limiter in ProxyRoute type and use it. if None, use cfg.rate_limiter as Default - match db.rate_exceeded(&rate_limiter_key, &cfg.rate_limiter).await { + let rate_limiter = proxy_route + .rate_limiter + .as_ref() + .unwrap_or(&cfg.rate_limiter); + match db.rate_exceeded(&rate_limiter_key, rate_limiter).await { Ok(false) => {} _ => { log::warn!( @@ -202,7 +204,7 @@ pub(crate) async fn validation_middleware_json_rpc( pub(crate) async fn validation_middleware_http_get( cfg: &AppConfig, payload: &HttpGetPayload, - _proxy_route: &ProxyRoute, + proxy_route: &ProxyRoute, req_uri: &Uri, remote_addr: &SocketAddr, ) -> Result<(), StatusCode> { @@ -251,8 +253,11 @@ pub(crate) async fn validation_middleware_http_get( payload.signed_message.coin_ticker, payload.signed_message.address ); - // TODO impl Optional rate limiter in ProxyRoute type and use it. if None, use cfg.rate_limiter as Default - match db.rate_exceeded(&rate_limiter_key, &cfg.rate_limiter).await { + let rate_limiter = proxy_route + .rate_limiter + .as_ref() + .unwrap_or(&cfg.rate_limiter); + match db.rate_exceeded(&rate_limiter_key, rate_limiter).await { Ok(false) => {} Ok(true) => { log::warn!( @@ -265,6 +270,7 @@ pub(crate) async fn validation_middleware_http_get( rate_limiter_key, ) ); + return Err(StatusCode::NOT_ACCEPTABLE); } Err(e) => { log::error!( From 3456a42edad8da97f63b0466dcf81875a1531d98 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 14 May 2024 12:31:28 +0700 Subject: [PATCH 20/39] review: remove conf_docker_test.json and use .conf_test instead --- assets/.config_test | 16 ++++++++-------- assets/conf_docker_test.json | 30 ------------------------------ docker-compose.yml | 2 +- src/ctx.rs | 32 ++++++++++++++++---------------- 4 files changed, 25 insertions(+), 55 deletions(-) delete mode 100644 assets/conf_docker_test.json diff --git a/assets/.config_test b/assets/.config_test index eeb763a..1b392be 100644 --- a/assets/.config_test +++ b/assets/.config_test @@ -1,8 +1,8 @@ { - "port": 5000, - "redis_connection_string": "dummy-value", - "pubkey_path": "dummy-value", - "privkey_path": "dummy-value", + "port": 6150, + "redis_connection_string": "redis://redis:6379", + "pubkey_path": "/usr/src/komodo-defi-proxy/assets/.pubkey_test", + "privkey_path": "/usr/src/komodo-defi-proxy/assets/.privkey_test", "token_expiration_time": 300, "proxy_routes": [ { @@ -29,10 +29,10 @@ "allowed_methods": [], "rate_limiter": { "rp_1_min": 60, - "rp_5_min": 60, - "rp_15_min": 60, - "rp_30_min": 60, - "rp_60_min": 60 + "rp_5_min": 200, + "rp_15_min": 700, + "rp_30_min": 1000, + "rp_60_min": 2000 } } ], diff --git a/assets/conf_docker_test.json b/assets/conf_docker_test.json deleted file mode 100644 index 59a2775..0000000 --- a/assets/conf_docker_test.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "port": 6150, - "pubkey_path": "/usr/src/komodo-defi-proxy/assets/.pubkey_test", - "privkey_path": "/usr/src/komodo-defi-proxy/assets/.privkey_test", - "redis_connection_string": "redis://redis:6379", - "token_expiration_time": 300, - "proxy_routes": [ - { - "inbound_route": "/nft", - "outbound_route": "https://nft.proxy", - "proxy_type": "http_get", - "authorized": false, - "allowed_methods": [], - "rate_limiter": { - "rp_1_min": 60, - "rp_5_min": 250, - "rp_15_min": 800, - "rp_30_min": 1500, - "rp_60_min": 3000 - } - } - ], - "rate_limiter": { - "rp_1_min": 30, - "rp_5_min": 100, - "rp_15_min": 200, - "rp_30_min": 350, - "rp_60_min": 575 - } -} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 572b553..b8afe98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,6 @@ services: depends_on: - redis environment: - AUTH_APP_CONFIG_PATH: /usr/src/komodo-defi-proxy/assets/conf_docker_test.json + AUTH_APP_CONFIG_PATH: /usr/src/komodo-defi-proxy/assets/.config_test volumes: - ./assets:/usr/src/komodo-defi-proxy/assets diff --git a/src/ctx.rs b/src/ctx.rs index 61f6d13..9b1b738 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -103,10 +103,10 @@ impl AppConfig { #[cfg(test)] pub(crate) fn get_app_config_test_instance() -> AppConfig { AppConfig { - port: Some(5000), - redis_connection_string: String::from("dummy-value"), - pubkey_path: String::from("dummy-value"), - privkey_path: String::from("dummy-value"), + port: Some(6150), + redis_connection_string: String::from("redis://redis:6379"), + pubkey_path: String::from("/usr/src/komodo-defi-proxy/assets/.pubkey_test"), + privkey_path: String::from("/usr/src/komodo-defi-proxy/assets/.privkey_test"), token_expiration_time: Some(300), proxy_routes: Vec::from([ ProxyRoute { @@ -133,10 +133,10 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { allowed_methods: Vec::default(), rate_limiter: Some(RateLimiter { rp_1_min: 60, - rp_5_min: 60, - rp_15_min: 60, - rp_30_min: 60, - rp_60_min: 60, + rp_5_min: 200, + rp_15_min: 700, + rp_30_min: 1000, + rp_60_min: 2000, }), }, ]), @@ -153,10 +153,10 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { #[test] fn test_app_config_serialzation_and_deserialization() { let json_config = serde_json::json!({ - "port": 5000, - "redis_connection_string": "dummy-value", - "pubkey_path": "dummy-value", - "privkey_path": "dummy-value", + "port": 6150, + "redis_connection_string": "redis://redis:6379", + "pubkey_path": "/usr/src/komodo-defi-proxy/assets/.pubkey_test", + "privkey_path": "/usr/src/komodo-defi-proxy/assets/.privkey_test", "token_expiration_time": 300, "proxy_routes": [ { @@ -183,10 +183,10 @@ fn test_app_config_serialzation_and_deserialization() { "allowed_methods": [], "rate_limiter": { "rp_1_min": 60, - "rp_5_min": 60, - "rp_15_min": 60, - "rp_30_min": 60, - "rp_60_min": 60 + "rp_5_min": 200, + "rp_15_min": 700, + "rp_30_min": 1000, + "rp_60_min": 2000 } } ], From 1c600306d86c8fcb1f91760c165be5a6f3b081fd Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 14 May 2024 16:34:00 +0700 Subject: [PATCH 21/39] review: move quicknode and moralis logic into separate crates, pin redis:7.2.4-alpine3.19 version. fix: Execute the pipeline once after setting all commands and make upsert_address_rate_in_pipe not async, as it just mutates Pipeline. --- assets/.config_test | 6 +- docker-compose.yml | 2 +- src/ctx.rs | 27 +- src/main.rs | 5 +- src/net/http.rs | 510 +------------------------------ src/net/server.rs | 245 +-------------- src/net/websocket.rs | 8 +- src/proxy/mod.rs | 118 +++++++ src/proxy/moralis.rs | 328 ++++++++++++++++++++ src/proxy/quicknode.rs | 393 ++++++++++++++++++++++++ src/security/proof_of_funding.rs | 64 ---- src/security/rate_limiter.rs | 31 +- 12 files changed, 880 insertions(+), 857 deletions(-) create mode 100644 src/proxy/mod.rs create mode 100644 src/proxy/moralis.rs create mode 100644 src/proxy/quicknode.rs delete mode 100644 src/security/proof_of_funding.rs diff --git a/assets/.config_test b/assets/.config_test index 1b392be..20aff44 100644 --- a/assets/.config_test +++ b/assets/.config_test @@ -8,7 +8,7 @@ { "inbound_route": "/test", "outbound_route": "https://komodoplatform.com", - "proxy_type": "json_rpc", + "proxy_type": "quicknode", "authorized": false, "allowed_methods": [], "rate_limiter": null @@ -16,7 +16,7 @@ { "inbound_route": "/test-2", "outbound_route": "https://atomicdex.io", - "proxy_type": "json_rpc", + "proxy_type": "quicknode", "authorized": false, "allowed_methods": [], "rate_limiter": null @@ -24,7 +24,7 @@ { "inbound_route": "/nft-test", "outbound_route": "https://nft.proxy", - "proxy_type": "http_get", + "proxy_type": "moralis", "authorized": false, "allowed_methods": [], "rate_limiter": { diff --git a/docker-compose.yml b/docker-compose.yml index b8afe98..588b481 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: redis: - image: redis:latest + image: redis:7.2.4-alpine3.19 restart: always ports: - "6379:6379" diff --git a/src/ctx.rs b/src/ctx.rs index 9b1b738..8001294 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -1,9 +1,9 @@ -use std::env; - use once_cell::sync::OnceCell; +use proxy::ProxyType; use serde::{Deserialize, Serialize}; +use std::env; -use super::*; +pub(crate) use super::*; const DEFAULT_TOKEN_EXPIRATION_TIME: i64 = 3600; static CONFIG: OnceCell = OnceCell::new(); @@ -54,15 +54,6 @@ pub(crate) struct ProxyRoute { pub(crate) rate_limiter: Option, } -/// Enumerates different types of proxy operations supported, such as JSON-RPC Call over HTTP POST and HTTP GET. -/// This helps in applying specific handling logic based on the proxy type. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "snake_case")] -pub(crate) enum ProxyType { - JsonRpc, // JSON-RPC call using HTTP POST - HttpGet, // Standard HTTP GET request -} - /// Configuration for rate limiting to manage the number of requests allowed over specified time intervals. /// This prevents abuse and ensures fair usage of resources among all clients. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -112,7 +103,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { ProxyRoute { inbound_route: String::from("/test"), outbound_route: String::from("https://komodoplatform.com"), - proxy_type: ProxyType::JsonRpc, + proxy_type: ProxyType::Quicknode, authorized: false, allowed_methods: Vec::default(), rate_limiter: None, @@ -120,7 +111,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { ProxyRoute { inbound_route: String::from("/test-2"), outbound_route: String::from("https://atomicdex.io"), - proxy_type: ProxyType::JsonRpc, + proxy_type: ProxyType::Quicknode, authorized: false, allowed_methods: Vec::default(), rate_limiter: None, @@ -128,7 +119,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { ProxyRoute { inbound_route: String::from("/nft-test"), outbound_route: String::from("https://nft.proxy"), - proxy_type: ProxyType::HttpGet, + proxy_type: ProxyType::Moralis, authorized: false, allowed_methods: Vec::default(), rate_limiter: Some(RateLimiter { @@ -162,7 +153,7 @@ fn test_app_config_serialzation_and_deserialization() { { "inbound_route": "/test", "outbound_route": "https://komodoplatform.com", - "proxy_type":"json_rpc", + "proxy_type":"quicknode", "authorized": false, "allowed_methods": [], "rate_limiter": null @@ -170,7 +161,7 @@ fn test_app_config_serialzation_and_deserialization() { { "inbound_route": "/test-2", "outbound_route": "https://atomicdex.io", - "proxy_type":"json_rpc", + "proxy_type":"quicknode", "authorized": false, "allowed_methods": [], "rate_limiter": null @@ -178,7 +169,7 @@ fn test_app_config_serialzation_and_deserialization() { { "inbound_route": "/nft-test", "outbound_route": "https://nft.proxy", - "proxy_type":"http_get", + "proxy_type":"moralis", "authorized": false, "allowed_methods": [], "rate_limiter": { diff --git a/src/main.rs b/src/main.rs index 6d20493..49d9dec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,8 +10,7 @@ mod db; mod http; #[path = "security/jwt.rs"] mod jwt; -#[path = "security/proof_of_funding.rs"] -mod proof_of_funding; + #[path = "security/rate_limiter.rs"] mod rate_limiter; #[path = "net/rpc.rs"] @@ -23,6 +22,8 @@ mod sign; #[path = "net/websocket.rs"] mod websocket; +mod proxy; + #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))] #[global_allocator] static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; diff --git a/src/net/http.rs b/src/net/http.rs index 76987e2..7492330 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -1,23 +1,15 @@ -use std::net::SocketAddr; -use std::str::FromStr; - +use super::*; +use crate::proxy::{generate_payload_from_req, proxy, validation_middleware}; +use crate::server::is_private_ip; use address_status::{get_address_status_list, post_address_status}; -use ctx::{AppConfig, ProxyRoute, ProxyType}; -use hyper::header::HeaderName; -use hyper::http::uri::PathAndQuery; +use ctx::AppConfig; use hyper::{ header::{self, HeaderValue}, - Body, HeaderMap, Method, Request, Response, StatusCode, Uri, + Body, HeaderMap, Method, Request, Response, StatusCode, }; -use hyper_tls::HttpsConnector; use jwt::{get_cached_token_or_generate_one, JwtClaims}; -use serde::{Deserialize, Serialize}; use serde_json::json; -use sign::SignedMessage; -use url::Url; - -use super::*; -use crate::server::{is_private_ip, validation_middleware}; +use std::net::SocketAddr; /// Value pub(crate) const APPLICATION_JSON: &str = "application/json"; @@ -69,323 +61,6 @@ pub(crate) async fn insert_jwt_to_http_header( Ok(()) } -/// Asynchronously parses an HTTP request's body into a specified type `T`. If the request method is `GET`, -/// the function modifies the request to have an empty body. For other methods, it retains the original body. -/// The function ensures that the body is not empty before attempting deserialization into the non-optional type `T`. -async fn parse_payload(req: Request, get_req: bool) -> GenericResult<(Request, T)> -where - T: serde::de::DeserializeOwned, -{ - let (mut parts, body) = req.into_parts(); - let body_bytes = hyper::body::to_bytes(body).await?; - - if body_bytes.is_empty() { - return Err("Empty body cannot be deserialized into non-optional type T".into()); - } - - let payload: T = serde_json::from_slice(&body_bytes)?; - - let new_req = if get_req { - parts.method = Method::GET; - Request::from_parts(parts, Body::empty()) - } else { - Request::from_parts(parts, Body::from(body_bytes)) - }; - - Ok((new_req, payload)) -} - -/// Represents a JSON-RPC Call payload parsed from a proxy request. It combines standard JSON RPC method call -/// fields with a `SignedMessage` for authentication and validation by the proxy. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub(crate) struct JsonRpcPayload { - pub(crate) method: String, - pub(crate) params: serde_json::value::Value, - pub(crate) id: usize, - pub(crate) jsonrpc: String, - pub(crate) signed_message: SignedMessage, -} - -/// Represents a payload for HTTP GET request parsed from a proxy request. This struct contains the URL -/// that the proxy will forward the GET request to, along with a `SignedMessage` for authentication and validation. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub(crate) struct HttpGetPayload { - uri: Url, - pub(crate) signed_message: SignedMessage, -} - -/// Enumerates the types of payloads that can be processed by the proxy. -/// Each variant holds a specific payload type relevant to the proxy operation being performed. -#[derive(Clone, Debug, PartialEq)] -pub(crate) enum PayloadData { - JsonRpc(JsonRpcPayload), - HttpGet(HttpGetPayload), -} - -impl PayloadData { - /// Returns a reference to the `SignedMessage` contained within the payload. - fn signed_message(&self) -> &SignedMessage { - match self { - PayloadData::JsonRpc(json_rpc_payload) => &json_rpc_payload.signed_message, - PayloadData::HttpGet(http_get_payload) => &http_get_payload.signed_message, - } - } -} - -/// Asynchronously generates and parses payload data from an HTTP request based on the specified proxy type. -/// Returns a tuple containing the modified (if necessary) request and the parsed payload from req Body as `PayloadData`. -async fn generate_payload_from_req( - req: Request, - proxy_type: &ProxyType, -) -> GenericResult<(Request, PayloadData)> { - match proxy_type { - ProxyType::JsonRpc => { - let (req, payload) = parse_payload::(req, false).await?; - Ok((req, PayloadData::JsonRpc(payload))) - } - ProxyType::HttpGet => { - let (req, payload) = parse_payload::(req, true).await?; - Ok((req, PayloadData::HttpGet(payload))) - } - } -} - -async fn proxy( - cfg: &AppConfig, - req: Request, - remote_addr: &SocketAddr, - payload: PayloadData, - x_forwarded_for: HeaderValue, - proxy_route: &ProxyRoute, -) -> GenericResult> { - match payload { - PayloadData::JsonRpc(payload) => { - proxy_json_rpc(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await - } - PayloadData::HttpGet(payload) => { - proxy_http_get(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await - } - } -} - -async fn proxy_json_rpc( - cfg: &AppConfig, - mut req: Request, - remote_addr: &SocketAddr, - payload: JsonRpcPayload, - x_forwarded_for: HeaderValue, - proxy_route: &ProxyRoute, -) -> GenericResult> { - // check if requested method allowed - if !proxy_route.allowed_methods.contains(&payload.method) { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req.uri(), - "Method {} not allowed for, returning 403.", - payload.method - ) - ); - return response_by_status(StatusCode::FORBIDDEN); - } - - if proxy_route.authorized { - // modify outgoing request - if insert_jwt_to_http_header(cfg, req.headers_mut()) - .await - .is_err() - { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req.uri(), - "Error inserting JWT into http header, returning 500." - ) - ); - return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); - } - } - - let original_req_uri = req.uri().clone(); - *req.uri_mut() = match proxy_route.outbound_route.parse() { - Ok(uri) => uri, - Err(e) => { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - original_req_uri, - "Error type casting value of {} into Uri: {}, returning 500.", - proxy_route.outbound_route, - e - ) - ); - return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); - } - }; - - // drop hop headers - for key in &[ - header::ACCEPT_ENCODING, - header::CONNECTION, - header::HOST, - header::PROXY_AUTHENTICATE, - header::PROXY_AUTHORIZATION, - header::TE, - header::TRANSFER_ENCODING, - header::TRAILER, - header::UPGRADE, - header::HeaderName::from_static("keep-alive"), - ] { - req.headers_mut().remove(key); - } - - req.headers_mut() - .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); - req.headers_mut() - .insert(header::CONTENT_TYPE, APPLICATION_JSON.parse()?); - - let https = HttpsConnector::new(); - let client = hyper::Client::builder().build(https); - - let target_uri = req.uri().clone(); - let res = match client.request(req).await { - Ok(t) => t, - Err(e) => { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - original_req_uri, - "Couldn't reach {}: {}. Returning 503.", - target_uri, - e - ) - ); - return response_by_status(StatusCode::SERVICE_UNAVAILABLE); - } - }; - - Ok(res) -} - -async fn proxy_http_get( - cfg: &AppConfig, - mut req: Request, - remote_addr: &SocketAddr, - payload: HttpGetPayload, - x_forwarded_for: HeaderValue, - proxy_route: &ProxyRoute, -) -> GenericResult> { - if proxy_route.authorized { - if let Err(e) = insert_jwt_to_http_header(cfg, req.headers_mut()).await { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req.uri(), - "Error inserting JWT into HTTP header: {}, returning 500.", - e - ) - ); - return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); - } - } - - let original_req_uri = req.uri().clone(); - - if let Err(e) = modify_request_uri(&mut req, &payload, proxy_route).await { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - original_req_uri, - "Error modifying request Uri: {}, returning 500.", - e - ) - ); - return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); - } - - // drop hop headers - for key in &[ - header::ACCEPT_ENCODING, - header::CONNECTION, - header::HOST, - header::PROXY_AUTHENTICATE, - header::PROXY_AUTHORIZATION, - header::TE, - header::TRANSFER_ENCODING, - header::TRAILER, - header::UPGRADE, - header::CONTENT_LENGTH, - header::HeaderName::from_static("keep-alive"), - ] { - req.headers_mut().remove(key); - } - - req.headers_mut() - .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); - req.headers_mut() - .insert(header::CONTENT_TYPE, APPLICATION_JSON.parse()?); - - let https = HttpsConnector::new(); - let client = hyper::Client::builder().build(https); - - let target_uri = req.uri().clone(); - let res = match client.request(req).await { - Ok(t) => t, - Err(e) => { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - original_req_uri, - "Couldn't reach {}: {}. Returning 503.", - target_uri, - e - ) - ); - return response_by_status(StatusCode::SERVICE_UNAVAILABLE); - } - }; - - Ok(res) -} - -/// Modifies the URI of an HTTP request by replacing it to outbound URI specified in `ProxyRoute`, -/// while incorporating the path and query parameters from the payload's URI. -async fn modify_request_uri( - req: &mut Request, - payload: &HttpGetPayload, - proxy_route: &ProxyRoute, -) -> GenericResult<()> { - let mut proxy_outbound_parts = proxy_route.outbound_route.parse::()?.into_parts(); - - let payload_uri: Uri = payload.uri.as_str().parse()?; - - let path_and_query = - PathAndQuery::from_str(payload_uri.path_and_query().map_or("/", |pq| pq.as_str()))?; - // Append the path and query from the payload URI to the proxy outbound URI. - proxy_outbound_parts.path_and_query = Some(path_and_query); - - // Reconstruct the full URI with the updated parts. - let new_uri = Uri::from_parts(proxy_outbound_parts)?; - - // Update the request URI. - *req.uri_mut() = new_uri; - Ok(()) -} - pub(crate) async fn http_handler( cfg: &AppConfig, req: Request, @@ -506,46 +181,9 @@ pub(crate) async fn http_handler( .await } -#[test] -fn test_rpc_payload_serialzation_and_deserialization() { - let json_payload = json!({ - "method": "dummy-value", - "params": [], - "id": 1, - "jsonrpc": "2.0", - "signed_message": { - "coin_ticker": "ETH", - "address": "dummy-value", - "timestamp_message": 1655319963, - "signature": "dummy-value", - } - }); - - let actual_payload: JsonRpcPayload = serde_json::from_str(&json_payload.to_string()).unwrap(); - - let expected_payload = JsonRpcPayload { - method: String::from("dummy-value"), - params: json!([]), - id: 1, - jsonrpc: String::from("2.0"), - signed_message: SignedMessage { - coin_ticker: String::from("ETH"), - address: String::from("dummy-value"), - timestamp_message: 1655319963, - signature: String::from("dummy-value"), - }, - }; - - assert_eq!(actual_payload, expected_payload); - - // Backwards - let json = serde_json::to_value(expected_payload).unwrap(); - assert_eq!(json_payload, json); - assert_eq!(json_payload.to_string(), json.to_string()); -} - #[test] fn test_get_proxy_route_by_inbound() { + use hyper::Uri; use std::str::FromStr; let cfg = ctx::get_app_config_test_instance(); @@ -586,137 +224,3 @@ fn test_respond_by_status() { assert_eq!(res.status(), status_type); } } - -#[tokio::test] -async fn test_parse_json_rpc_payload() { - let serialized_payload = json!({ - "method": "dummy-value", - "params": [], - "id": 1, - "jsonrpc": "2.0", - "signed_message": { - "coin_ticker": "ETH", - "address": "dummy-value", - "timestamp_message": 1655319963, - "signature": "dummy-value", - } - }) - .to_string(); - - let mut req = Request::builder() - .method(Method::POST) - .body(Body::from(serialized_payload)) - .unwrap(); - req.headers_mut().insert( - HeaderName::from_static("dummy-header"), - "dummy-value".parse().unwrap(), - ); - - let (mut req, payload): (Request, JsonRpcPayload) = - parse_payload(req, false).await.unwrap(); - - let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); - assert!( - !body_bytes.is_empty(), - "Body should not be empty for non-GET methods" - ); - - let header_value = req.headers().get("dummy-header").unwrap(); - - let expected_payload = JsonRpcPayload { - method: String::from("dummy-value"), - params: json!([]), - id: 1, - jsonrpc: String::from("2.0"), - signed_message: SignedMessage { - coin_ticker: String::from("ETH"), - address: String::from("dummy-value"), - timestamp_message: 1655319963, - signature: String::from("dummy-value"), - }, - }; - - assert_eq!(payload, expected_payload); - assert_eq!(header_value, "dummy-value"); -} - -#[tokio::test] -async fn test_parse_http_get_payload() { - let serialized_payload = json!({ - "uri": "https://example.com/test-path", - "signed_message": { - "coin_ticker": "BTC", - "address": "dummy-value", - "timestamp_message": 1655320000, - "signature": "dummy-value", - } - }) - .to_string(); - - let mut req = Request::new(Body::from(serialized_payload)); - req.headers_mut().insert( - HeaderName::from_static("accept"), - APPLICATION_JSON.parse().unwrap(), - ); - - let (mut req, payload): (Request, HttpGetPayload) = - parse_payload(req, true).await.unwrap(); - - let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); - assert!( - body_bytes.is_empty(), - "Body should be empty for GET methods" - ); - - let header_value = req.headers().get("accept").unwrap(); - - let expected_payload = HttpGetPayload { - uri: Url::from_str("https://example.com/test-path").unwrap(), - signed_message: SignedMessage { - coin_ticker: String::from("BTC"), - address: String::from("dummy-value"), - timestamp_message: 1655320000, - signature: String::from("dummy-value"), - }, - }; - - assert_eq!(payload, expected_payload); - assert_eq!(header_value, APPLICATION_JSON); -} - -#[tokio::test] -async fn test_modify_request_uri() { - let orig_uri_str = "https://proxy.example:3535/test-inbound"; - let mut req = Request::builder() - .uri(orig_uri_str) - .body(Body::empty()) - .unwrap(); - - let payload = HttpGetPayload { - uri: Url::from_str("https://proxy.example:3535/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false").unwrap(), - signed_message: SignedMessage { - coin_ticker: String::from("BTC"), - address: String::from("dummy-value"), - timestamp_message: 1655320000, - signature: String::from("dummy-value"), - }, - }; - - let proxy_route = ProxyRoute { - inbound_route: String::from_str("/test-inbound").unwrap(), - outbound_route: "http://localhost:8000".to_string(), - proxy_type: ProxyType::HttpGet, - authorized: false, - allowed_methods: vec![], - rate_limiter: None, - }; - - modify_request_uri(&mut req, &payload, &proxy_route) - .await - .unwrap(); - - assert_eq!( - req.uri(), - "http://localhost:8000/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false" - ); -} diff --git a/src/net/server.rs b/src/net/server.rs index 8ff550a..231f1e9 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -3,19 +3,12 @@ use std::str::FromStr; use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Request, Response, Server, StatusCode, Uri}; +use hyper::{Body, Request, Response, Server, StatusCode}; use super::{GenericError, GenericResult}; -use crate::address_status::{AddressStatus, AddressStatusOperations}; -use crate::ctx::{AppConfig, ProxyRoute}; -use crate::db::Db; -use crate::http::{ - http_handler, response_by_status, HttpGetPayload, JsonRpcPayload, PayloadData, X_FORWARDED_FOR, -}; +use crate::ctx::AppConfig; +use crate::http::{http_handler, response_by_status, X_FORWARDED_FOR}; use crate::log_format; -use crate::proof_of_funding::{verify_message_and_balance, ProofOfFundingError}; -use crate::rate_limiter::RateLimitOperations; -use crate::sign::SignOps; use crate::websocket::{should_upgrade_to_socket_conn, socket_handler}; #[macro_export] @@ -76,238 +69,6 @@ async fn connection_handler( } } -pub(crate) async fn validation_middleware( - cfg: &AppConfig, - payload: &PayloadData, - proxy_route: &ProxyRoute, - req_uri: &Uri, - remote_addr: &SocketAddr, -) -> Result<(), StatusCode> { - match payload { - PayloadData::JsonRpc(payload) => { - validation_middleware_json_rpc(cfg, payload, proxy_route, req_uri, remote_addr).await - } - PayloadData::HttpGet(payload) => { - validation_middleware_http_get(cfg, payload, proxy_route, req_uri, remote_addr).await - } - } -} - -pub(crate) async fn validation_middleware_json_rpc( - cfg: &AppConfig, - payload: &JsonRpcPayload, - proxy_route: &ProxyRoute, - req_uri: &Uri, - remote_addr: &SocketAddr, -) -> Result<(), StatusCode> { - let mut db = Db::create_instance(cfg).await; - - match db - .read_address_status(&payload.signed_message.address) - .await - { - AddressStatus::Trusted => Ok(()), - AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), - AddressStatus::None => { - let signed_message_status = verify_message_and_balance(cfg, payload, proxy_route).await; - - if let Err(ProofOfFundingError::InvalidSignedMessage) = signed_message_status { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "Request has invalid signed message, returning 401" - ) - ); - - return Err(StatusCode::UNAUTHORIZED); - }; - - let rate_limiter_key = format!( - "{}:{}", - payload.signed_message.coin_ticker, payload.signed_message.address - ); - - let rate_limiter = proxy_route - .rate_limiter - .as_ref() - .unwrap_or(&cfg.rate_limiter); - match db.rate_exceeded(&rate_limiter_key, rate_limiter).await { - Ok(false) => {} - _ => { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "Rate exceed for {}, checking balance for {} address.", - rate_limiter_key, - payload.signed_message.address - ) - ); - - match verify_message_and_balance(cfg, payload, proxy_route).await { - Ok(_) => {} - Err(ProofOfFundingError::InsufficientBalance) => { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "Wallet {} has insufficient balance for coin {}, returning 406.", - payload.signed_message.address, - payload.signed_message.coin_ticker, - ) - ); - - return Err(StatusCode::NOT_ACCEPTABLE); - } - e => { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "verify_message_and_balance failed in coin {}: {:?}", - payload.signed_message.coin_ticker, - e - ) - ); - return Err(StatusCode::INTERNAL_SERVER_ERROR); - } - } - } - }; - - if db.rate_address(rate_limiter_key).await.is_err() { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "Rate incrementing failed." - ) - ); - }; - - Ok(()) - } - } -} - -pub(crate) async fn validation_middleware_http_get( - cfg: &AppConfig, - payload: &HttpGetPayload, - proxy_route: &ProxyRoute, - req_uri: &Uri, - remote_addr: &SocketAddr, -) -> Result<(), StatusCode> { - let mut db = Db::create_instance(cfg).await; - - match db - .read_address_status(&payload.signed_message.address) - .await - { - AddressStatus::Trusted => Ok(()), - AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), - AddressStatus::None => { - match payload.signed_message.verify_message() { - Ok(true) => {} - Ok(false) => { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "Request has invalid signed message, returning 401" - ) - ); - - return Err(StatusCode::UNAUTHORIZED); - } - Err(e) => { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "verify_message failed in coin {}: {}, returning 500.", - payload.signed_message.coin_ticker, - e - ) - ); - return Err(StatusCode::INTERNAL_SERVER_ERROR); - } - } - - let rate_limiter_key = format!( - "{}:{}", - payload.signed_message.coin_ticker, payload.signed_message.address - ); - - let rate_limiter = proxy_route - .rate_limiter - .as_ref() - .unwrap_or(&cfg.rate_limiter); - match db.rate_exceeded(&rate_limiter_key, rate_limiter).await { - Ok(false) => {} - Ok(true) => { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "Rate exceed for {}, returning 406.", - rate_limiter_key, - ) - ); - return Err(StatusCode::NOT_ACCEPTABLE); - } - Err(e) => { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "Rate exceeded check failed in coin {}: {}, returning 500.", - payload.signed_message.coin_ticker, - e - ) - ); - return Err(StatusCode::INTERNAL_SERVER_ERROR); - } - } - - if let Err(e) = db.rate_address(rate_limiter_key).await { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - payload.signed_message.address, - req_uri, - "Rate incrementing failed in coin {}: {}, returning 500.", - payload.signed_message.coin_ticker, - e - ) - ); - return Err(StatusCode::INTERNAL_SERVER_ERROR); - }; - - Ok(()) - } - } -} - /// Starts serving the proxy API on the configured port. This function sets up the HTTP server, /// binds it to the specified address, and listens for incoming requests. pub(crate) async fn serve(cfg: &'static AppConfig) -> GenericResult<()> { diff --git a/src/net/websocket.rs b/src/net/websocket.rs index 5999e98..5c74a28 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -10,9 +10,9 @@ use tokio_tungstenite::{ use crate::{ ctx::AppConfig, - http::{response_by_status, JsonRpcPayload}, + http::response_by_status, log_format, - server::validation_middleware_json_rpc, + proxy::{validation_middleware_quicknode, QuicknodePayload}, GenericResult, }; @@ -137,7 +137,7 @@ pub(crate) async fn socket_handler( match msg { Some(Ok(msg)) => { if let Message::Text(msg) = msg { - let payload: JsonRpcPayload = match serde_json::from_str(&msg) { + let payload: QuicknodePayload = match serde_json::from_str(&msg) { Ok(t) => t, Err(e) => { if let Err(e) = inbound_socket.send(format!("Invalid payload. {e}").into()).await { @@ -174,7 +174,7 @@ pub(crate) async fn socket_handler( } // TODO add general validation_middleware support - match validation_middleware_json_rpc( + match validation_middleware_quicknode( &cfg, &payload, &proxy_route, diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs new file mode 100644 index 0000000..1a6ec4b --- /dev/null +++ b/src/proxy/mod.rs @@ -0,0 +1,118 @@ +use crate::ctx::{AppConfig, GenericResult, ProxyRoute}; +use crate::sign::SignedMessage; +use hyper::header::HeaderValue; +use hyper::{Body, Method, Request, Response, StatusCode, Uri}; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +mod moralis; +use moralis::{proxy_moralis, validation_middleware_moralis, MoralisPayload}; +mod quicknode; +pub(crate) use quicknode::{proxy_quicknode, validation_middleware_quicknode, QuicknodePayload}; + +/// Enumerates different proxy types supported by the application, focusing on separating feature logic. +/// This allows for differentiated handling based on what the proxy should do with the request, +/// directing each to the appropriate service or API based on its designated proxy type. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ProxyType { + Quicknode, + Moralis, +} + +/// Represents the types of payloads that can be processed by the proxy, with each variant tailored to a specific proxy type. +/// This helps in managing the logic for routing and processing requests appropriately within the proxy layer. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum PayloadData { + Quicknode(QuicknodePayload), + Moralis(MoralisPayload), +} + +impl PayloadData { + /// Returns a reference to the `SignedMessage` contained within the payload. + pub(crate) fn signed_message(&self) -> &SignedMessage { + match self { + PayloadData::Quicknode(payload) => &payload.signed_message, + PayloadData::Moralis(payload) => &payload.signed_message, + } + } +} + +/// Asynchronously generates and organizes payload data from an HTTP request based on the specified proxy type. +/// This function ensures that requests are properly formatted to the correct service, +/// returning a tuple with the modified request and the structured payload. +pub(crate) async fn generate_payload_from_req( + req: Request, + proxy_type: &ProxyType, +) -> GenericResult<(Request, PayloadData)> { + match proxy_type { + ProxyType::Quicknode => { + let (req, payload) = parse_payload::(req, false).await?; + Ok((req, PayloadData::Quicknode(payload))) + } + ProxyType::Moralis => { + let (req, payload) = parse_payload::(req, true).await?; + Ok((req, PayloadData::Moralis(payload))) + } + } +} + +pub(crate) async fn proxy( + cfg: &AppConfig, + req: Request, + remote_addr: &SocketAddr, + payload: PayloadData, + x_forwarded_for: HeaderValue, + proxy_route: &ProxyRoute, +) -> GenericResult> { + match payload { + PayloadData::Quicknode(payload) => { + proxy_quicknode(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await + } + PayloadData::Moralis(payload) => { + proxy_moralis(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await + } + } +} + +pub(crate) async fn validation_middleware( + cfg: &AppConfig, + payload: &PayloadData, + proxy_route: &ProxyRoute, + req_uri: &Uri, + remote_addr: &SocketAddr, +) -> Result<(), StatusCode> { + match payload { + PayloadData::Quicknode(payload) => { + validation_middleware_quicknode(cfg, payload, proxy_route, req_uri, remote_addr).await + } + PayloadData::Moralis(payload) => { + validation_middleware_moralis(cfg, payload, proxy_route, req_uri, remote_addr).await + } + } +} + +/// Asynchronously parses an HTTP request's body into a specified type `T`. If the request method is `GET`, +/// the function modifies the request to have an empty body. For other methods, it retains the original body. +/// The function ensures that the body is not empty before attempting deserialization into the non-optional type `T`. +async fn parse_payload(req: Request, get_req: bool) -> GenericResult<(Request, T)> +where + T: serde::de::DeserializeOwned, +{ + let (mut parts, body) = req.into_parts(); + let body_bytes = hyper::body::to_bytes(body).await?; + + if body_bytes.is_empty() { + return Err("Empty body cannot be deserialized into non-optional type T".into()); + } + + let payload: T = serde_json::from_slice(&body_bytes)?; + + let new_req = if get_req { + parts.method = Method::GET; + Request::from_parts(parts, Body::empty()) + } else { + Request::from_parts(parts, Body::from(body_bytes)) + }; + + Ok((new_req, payload)) +} diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs new file mode 100644 index 0000000..c93ecef --- /dev/null +++ b/src/proxy/moralis.rs @@ -0,0 +1,328 @@ +use crate::address_status::{AddressStatus, AddressStatusOperations}; +use crate::ctx::{AppConfig, ProxyRoute}; +use crate::db::Db; +use crate::http::{ + insert_jwt_to_http_header, response_by_status, APPLICATION_JSON, X_FORWARDED_FOR, +}; +use crate::rate_limiter::RateLimitOperations; +use crate::sign::{SignOps, SignedMessage}; +use crate::{log_format, GenericResult}; +use hyper::header::{HeaderName, HeaderValue}; +use hyper::http::uri::PathAndQuery; +use hyper::{header, Body, Request, Response, StatusCode, Uri}; +use hyper_tls::HttpsConnector; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +use std::str::FromStr; +use url::Url; + +/// Represents a payload for HTTP GET requests, specifically parsed for the Moralis API within the proxy. +/// This struct contains the destination URL that the proxy will forward the GET request to, ensuring correct service routing. +/// It also includes a `SignedMessage` for authentication and validation, confirming the legitimacy of the request and enhancing security. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub(crate) struct MoralisPayload { + uri: Url, + pub(crate) signed_message: SignedMessage, +} + +pub(crate) async fn proxy_moralis( + cfg: &AppConfig, + mut req: Request, + remote_addr: &SocketAddr, + payload: MoralisPayload, + x_forwarded_for: HeaderValue, + proxy_route: &ProxyRoute, +) -> GenericResult> { + if proxy_route.authorized { + if let Err(e) = insert_jwt_to_http_header(cfg, req.headers_mut()).await { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req.uri(), + "Error inserting JWT into HTTP header: {}, returning 500.", + e + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + } + + let original_req_uri = req.uri().clone(); + + if let Err(e) = modify_request_uri(&mut req, &payload, proxy_route).await { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + original_req_uri, + "Error modifying request Uri: {}, returning 500.", + e + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + + // drop hop headers + for key in &[ + header::ACCEPT_ENCODING, + header::CONNECTION, + header::HOST, + header::PROXY_AUTHENTICATE, + header::PROXY_AUTHORIZATION, + header::TE, + header::TRANSFER_ENCODING, + header::TRAILER, + header::UPGRADE, + header::CONTENT_LENGTH, + header::HeaderName::from_static("keep-alive"), + ] { + req.headers_mut().remove(key); + } + + req.headers_mut() + .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); + req.headers_mut() + .insert(header::CONTENT_TYPE, APPLICATION_JSON.parse()?); + + let https = HttpsConnector::new(); + let client = hyper::Client::builder().build(https); + + let target_uri = req.uri().clone(); + let res = match client.request(req).await { + Ok(t) => t, + Err(e) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + original_req_uri, + "Couldn't reach {}: {}. Returning 503.", + target_uri, + e + ) + ); + return response_by_status(StatusCode::SERVICE_UNAVAILABLE); + } + }; + + Ok(res) +} + +/// Modifies the URI of an HTTP request by replacing it to outbound URI specified in `ProxyRoute`, +/// while incorporating the path and query parameters from the payload's URI. +async fn modify_request_uri( + req: &mut Request, + payload: &MoralisPayload, + proxy_route: &ProxyRoute, +) -> GenericResult<()> { + let mut proxy_outbound_parts = proxy_route.outbound_route.parse::()?.into_parts(); + + let payload_uri: Uri = payload.uri.as_str().parse()?; + + let path_and_query = + PathAndQuery::from_str(payload_uri.path_and_query().map_or("/", |pq| pq.as_str()))?; + // Append the path and query from the payload URI to the proxy outbound URI. + proxy_outbound_parts.path_and_query = Some(path_and_query); + + // Reconstruct the full URI with the updated parts. + let new_uri = Uri::from_parts(proxy_outbound_parts)?; + + // Update the request URI. + *req.uri_mut() = new_uri; + Ok(()) +} + +pub(crate) async fn validation_middleware_moralis( + cfg: &AppConfig, + payload: &MoralisPayload, + proxy_route: &ProxyRoute, + req_uri: &Uri, + remote_addr: &SocketAddr, +) -> Result<(), StatusCode> { + let mut db = Db::create_instance(cfg).await; + + match db + .read_address_status(&payload.signed_message.address) + .await + { + AddressStatus::Trusted => Ok(()), + AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), + AddressStatus::None => { + match payload.signed_message.verify_message() { + Ok(true) => {} + Ok(false) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Request has invalid signed message, returning 401" + ) + ); + + return Err(StatusCode::UNAUTHORIZED); + } + Err(e) => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "verify_message failed in coin {}: {}, returning 500.", + payload.signed_message.coin_ticker, + e + ) + ); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + + let rate_limiter_key = format!( + "{}:{}", + payload.signed_message.coin_ticker, payload.signed_message.address + ); + + let rate_limiter = proxy_route + .rate_limiter + .as_ref() + .unwrap_or(&cfg.rate_limiter); + match db.rate_exceeded(&rate_limiter_key, rate_limiter).await { + Ok(false) => {} + Ok(true) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Rate exceed for {}, returning 406.", + rate_limiter_key, + ) + ); + return Err(StatusCode::NOT_ACCEPTABLE); + } + Err(e) => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Rate exceeded check failed in coin {}: {}, returning 500.", + payload.signed_message.coin_ticker, + e + ) + ); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + + if let Err(e) = db.rate_address(rate_limiter_key).await { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Rate incrementing failed in coin {}: {}, returning 500.", + payload.signed_message.coin_ticker, + e + ) + ); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(()) + } + } +} + +#[tokio::test] +async fn test_parse_moralis_payload() { + use super::parse_payload; + + let serialized_payload = serde_json::json!({ + "uri": "https://example.com/test-path", + "signed_message": { + "coin_ticker": "BTC", + "address": "dummy-value", + "timestamp_message": 1655320000, + "signature": "dummy-value", + } + }) + .to_string(); + + let mut req = Request::new(Body::from(serialized_payload)); + req.headers_mut().insert( + HeaderName::from_static("accept"), + APPLICATION_JSON.parse().unwrap(), + ); + + let (mut req, payload) = parse_payload::(req, true).await.unwrap(); + + let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); + assert!( + body_bytes.is_empty(), + "Body should be empty for GET methods" + ); + + let header_value = req.headers().get("accept").unwrap(); + + let expected_payload = MoralisPayload { + uri: Url::from_str("https://example.com/test-path").unwrap(), + signed_message: SignedMessage { + coin_ticker: String::from("BTC"), + address: String::from("dummy-value"), + timestamp_message: 1655320000, + signature: String::from("dummy-value"), + }, + }; + + assert_eq!(payload, expected_payload); + assert_eq!(header_value, APPLICATION_JSON); +} + +#[tokio::test] +async fn test_modify_request_uri() { + use super::ProxyType; + + let orig_uri_str = "https://proxy.example:3535/test-inbound"; + let mut req = Request::builder() + .uri(orig_uri_str) + .body(Body::empty()) + .unwrap(); + + let payload = MoralisPayload { + uri: Url::from_str("https://proxy.example:3535/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false").unwrap(), + signed_message: SignedMessage { + coin_ticker: String::from("BTC"), + address: String::from("dummy-value"), + timestamp_message: 1655320000, + signature: String::from("dummy-value"), + }, + }; + + let proxy_route = ProxyRoute { + inbound_route: String::from_str("/test-inbound").unwrap(), + outbound_route: "http://localhost:8000".to_string(), + proxy_type: ProxyType::Moralis, + authorized: false, + allowed_methods: vec![], + rate_limiter: None, + }; + + modify_request_uri(&mut req, &payload, &proxy_route) + .await + .unwrap(); + + assert_eq!( + req.uri(), + "http://localhost:8000/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false" + ); +} diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs new file mode 100644 index 0000000..f5ad50c --- /dev/null +++ b/src/proxy/quicknode.rs @@ -0,0 +1,393 @@ +use crate::address_status::{AddressStatus, AddressStatusOperations}; +use crate::ctx::{AppConfig, ProxyRoute}; +use crate::db::Db; +use crate::http::{ + insert_jwt_to_http_header, response_by_status, APPLICATION_JSON, X_FORWARDED_FOR, +}; +use crate::rate_limiter::RateLimitOperations; +use crate::rpc::Json; +use crate::sign::{SignOps, SignedMessage}; +use crate::{log_format, rpc, GenericResult}; +use hyper::header::{HeaderName, HeaderValue}; +use hyper::{header, Body, Request, Response, StatusCode, Uri}; +use hyper_tls::HttpsConnector; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::net::SocketAddr; + +/// Represents a payload for JSON-RPC calls, tailored for the Quicknode API within the proxy. +/// This struct combines standard JSON RPC method call fields (method, params, id, jsonrpc) with a `SignedMessage` +/// for authentication and validation, facilitating secure and validated interactions with the Quicknode service. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub(crate) struct QuicknodePayload { + pub(crate) method: String, + pub(crate) params: serde_json::value::Value, + pub(crate) id: usize, + pub(crate) jsonrpc: String, + pub(crate) signed_message: SignedMessage, +} + +#[derive(Debug)] +enum ProofOfFundingError { + InvalidSignedMessage, + InsufficientBalance, + ErrorFromRpcCall, + #[allow(dead_code)] + RpcCallFailed(String), +} + +pub(crate) async fn proxy_quicknode( + cfg: &AppConfig, + mut req: Request, + remote_addr: &SocketAddr, + payload: QuicknodePayload, + x_forwarded_for: HeaderValue, + proxy_route: &ProxyRoute, +) -> GenericResult> { + // check if requested method allowed + if !proxy_route.allowed_methods.contains(&payload.method) { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req.uri(), + "Method {} not allowed for, returning 403.", + payload.method + ) + ); + return response_by_status(StatusCode::FORBIDDEN); + } + + if proxy_route.authorized { + // modify outgoing request + if insert_jwt_to_http_header(cfg, req.headers_mut()) + .await + .is_err() + { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req.uri(), + "Error inserting JWT into http header, returning 500." + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + } + + let original_req_uri = req.uri().clone(); + *req.uri_mut() = match proxy_route.outbound_route.parse() { + Ok(uri) => uri, + Err(e) => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + original_req_uri, + "Error type casting value of {} into Uri: {}, returning 500.", + proxy_route.outbound_route, + e + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + }; + + // drop hop headers + for key in &[ + header::ACCEPT_ENCODING, + header::CONNECTION, + header::HOST, + header::PROXY_AUTHENTICATE, + header::PROXY_AUTHORIZATION, + header::TE, + header::TRANSFER_ENCODING, + header::TRAILER, + header::UPGRADE, + header::HeaderName::from_static("keep-alive"), + ] { + req.headers_mut().remove(key); + } + + req.headers_mut() + .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); + req.headers_mut() + .insert(header::CONTENT_TYPE, APPLICATION_JSON.parse()?); + + let https = HttpsConnector::new(); + let client = hyper::Client::builder().build(https); + + let target_uri = req.uri().clone(); + let res = match client.request(req).await { + Ok(t) => t, + Err(e) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + original_req_uri, + "Couldn't reach {}: {}. Returning 503.", + target_uri, + e + ) + ); + return response_by_status(StatusCode::SERVICE_UNAVAILABLE); + } + }; + + Ok(res) +} + +pub(crate) async fn validation_middleware_quicknode( + cfg: &AppConfig, + payload: &QuicknodePayload, + proxy_route: &ProxyRoute, + req_uri: &Uri, + remote_addr: &SocketAddr, +) -> Result<(), StatusCode> { + let mut db = Db::create_instance(cfg).await; + + match db + .read_address_status(&payload.signed_message.address) + .await + { + AddressStatus::Trusted => Ok(()), + AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), + AddressStatus::None => { + let signed_message_status = verify_message_and_balance(cfg, payload, proxy_route).await; + + if let Err(ProofOfFundingError::InvalidSignedMessage) = signed_message_status { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Request has invalid signed message, returning 401" + ) + ); + + return Err(StatusCode::UNAUTHORIZED); + }; + + let rate_limiter_key = format!( + "{}:{}", + payload.signed_message.coin_ticker, payload.signed_message.address + ); + + let rate_limiter = proxy_route + .rate_limiter + .as_ref() + .unwrap_or(&cfg.rate_limiter); + match db.rate_exceeded(&rate_limiter_key, rate_limiter).await { + Ok(false) => {} + _ => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Rate exceed for {}, checking balance for {} address.", + rate_limiter_key, + payload.signed_message.address + ) + ); + + match verify_message_and_balance(cfg, payload, proxy_route).await { + Ok(_) => {} + Err(ProofOfFundingError::InsufficientBalance) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Wallet {} has insufficient balance for coin {}, returning 406.", + payload.signed_message.address, + payload.signed_message.coin_ticker, + ) + ); + + return Err(StatusCode::NOT_ACCEPTABLE); + } + e => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "verify_message_and_balance failed in coin {}: {:?}", + payload.signed_message.coin_ticker, + e + ) + ); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + } + }; + + if db.rate_address(rate_limiter_key).await.is_err() { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + payload.signed_message.address, + req_uri, + "Rate incrementing failed." + ) + ); + }; + + Ok(()) + } + } +} + +async fn verify_message_and_balance( + cfg: &AppConfig, + payload: &QuicknodePayload, + proxy_route: &ProxyRoute, +) -> Result<(), ProofOfFundingError> { + if let Ok(true) = payload.signed_message.verify_message() { + let mut db = Db::create_instance(cfg).await; + + // We don't want to send balance requests everytime when user sends requests. + if let Ok(true) = db.key_exists(&payload.signed_message.address).await { + return Ok(()); + } + + let rpc_payload = json!({ + "id": 1, + "jsonrpc": "2.0", + "method": "eth_getBalance", + "params": [payload.signed_message.address, "latest"] + }); + + let rpc_client = + // TODO: Use the current transport instead of forcing to use http (even if it's rare, this might not work on certain nodes) + rpc::RpcClient::new(proxy_route.outbound_route.replace("ws", "http").clone()); + + match rpc_client + .send(cfg, rpc_payload, proxy_route.authorized) + .await + { + Ok(res) if res["result"] != Json::Null && res["result"] != "0x0" => { + // cache this address for 60 seconds + let _ = db + .insert_cache(&payload.signed_message.address, "", 60) + .await; + + return Ok(()); + } + Ok(res) if res["error"] != Json::Null => { + return Err(ProofOfFundingError::ErrorFromRpcCall); + } + Ok(_) => return Err(ProofOfFundingError::InsufficientBalance), + Err(e) => return Err(ProofOfFundingError::RpcCallFailed(e.to_string())), + }; + } + + Err(ProofOfFundingError::InvalidSignedMessage) +} + +#[test] +fn test_quicknode_payload_serialzation_and_deserialization() { + let json_payload = json!({ + "method": "dummy-value", + "params": [], + "id": 1, + "jsonrpc": "2.0", + "signed_message": { + "coin_ticker": "ETH", + "address": "dummy-value", + "timestamp_message": 1655319963, + "signature": "dummy-value", + } + }); + + let actual_payload: QuicknodePayload = serde_json::from_str(&json_payload.to_string()).unwrap(); + + let expected_payload = QuicknodePayload { + method: String::from("dummy-value"), + params: json!([]), + id: 1, + jsonrpc: String::from("2.0"), + signed_message: SignedMessage { + coin_ticker: String::from("ETH"), + address: String::from("dummy-value"), + timestamp_message: 1655319963, + signature: String::from("dummy-value"), + }, + }; + + assert_eq!(actual_payload, expected_payload); + + // Backwards + let json = serde_json::to_value(expected_payload).unwrap(); + assert_eq!(json_payload, json); + assert_eq!(json_payload.to_string(), json.to_string()); +} + +#[tokio::test] +async fn test_parse_quicknode_payload() { + use super::parse_payload; + use hyper::Method; + + let serialized_payload = json!({ + "method": "dummy-value", + "params": [], + "id": 1, + "jsonrpc": "2.0", + "signed_message": { + "coin_ticker": "ETH", + "address": "dummy-value", + "timestamp_message": 1655319963, + "signature": "dummy-value", + } + }) + .to_string(); + + let mut req = Request::builder() + .method(Method::POST) + .body(Body::from(serialized_payload)) + .unwrap(); + req.headers_mut().insert( + HeaderName::from_static("dummy-header"), + "dummy-value".parse().unwrap(), + ); + + let (mut req, payload): (Request, QuicknodePayload) = + parse_payload::(req, false).await.unwrap(); + + let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); + assert!( + !body_bytes.is_empty(), + "Body should not be empty for non-GET methods" + ); + + let header_value = req.headers().get("dummy-header").unwrap(); + + let expected_payload = QuicknodePayload { + method: String::from("dummy-value"), + params: json!([]), + id: 1, + jsonrpc: String::from("2.0"), + signed_message: SignedMessage { + coin_ticker: String::from("ETH"), + address: String::from("dummy-value"), + timestamp_message: 1655319963, + signature: String::from("dummy-value"), + }, + }; + + assert_eq!(payload, expected_payload); + assert_eq!(header_value, "dummy-value"); +} diff --git a/src/security/proof_of_funding.rs b/src/security/proof_of_funding.rs deleted file mode 100644 index 890de94..0000000 --- a/src/security/proof_of_funding.rs +++ /dev/null @@ -1,64 +0,0 @@ -use ctx::{AppConfig, ProxyRoute}; -use db::Db; -use http::JsonRpcPayload; -use rpc::Json; -use serde_json::json; -use sign::SignOps; - -use super::*; - -#[derive(Debug)] -pub(crate) enum ProofOfFundingError { - InvalidSignedMessage, - InsufficientBalance, - ErrorFromRpcCall, - #[allow(dead_code)] - RpcCallFailed(String), -} - -pub(crate) async fn verify_message_and_balance( - cfg: &AppConfig, - payload: &JsonRpcPayload, - proxy_route: &ProxyRoute, -) -> Result<(), ProofOfFundingError> { - if let Ok(true) = payload.signed_message.verify_message() { - let mut db = Db::create_instance(cfg).await; - - // We don't want to send balance requests everytime when user sends requests. - if let Ok(true) = db.key_exists(&payload.signed_message.address).await { - return Ok(()); - } - - let rpc_payload = json!({ - "id": 1, - "jsonrpc": "2.0", - "method": "eth_getBalance", - "params": [payload.signed_message.address, "latest"] - }); - - let rpc_client = - // TODO: Use the current transport instead of forcing to use http (even if it's rare, this might not work on certain nodes) - rpc::RpcClient::new(proxy_route.outbound_route.replace("ws", "http").clone()); - - match rpc_client - .send(cfg, rpc_payload, proxy_route.authorized) - .await - { - Ok(res) if res["result"] != Json::Null && res["result"] != "0x0" => { - // cache this address for 60 seconds - let _ = db - .insert_cache(&payload.signed_message.address, "", 60) - .await; - - return Ok(()); - } - Ok(res) if res["error"] != Json::Null => { - return Err(ProofOfFundingError::ErrorFromRpcCall); - } - Ok(_) => return Err(ProofOfFundingError::InsufficientBalance), - Err(e) => return Err(ProofOfFundingError::RpcCallFailed(e.to_string())), - }; - } - - Err(ProofOfFundingError::InvalidSignedMessage) -} diff --git a/src/security/rate_limiter.rs b/src/security/rate_limiter.rs index 3cc9161..96df068 100644 --- a/src/security/rate_limiter.rs +++ b/src/security/rate_limiter.rs @@ -13,13 +13,13 @@ pub(crate) const DB_RP_60_MIN: &str = "rp:60_min"; #[async_trait] pub(crate) trait RateLimitOperations { - async fn upsert_address_rate_in_pipe( + fn upsert_address_rate_in_pipe( &mut self, pipe: &mut Pipeline, db: &str, address: &str, expire_time: usize, - ) -> GenericResult<()>; + ); async fn rate_address(&mut self, address: String) -> GenericResult<()>; async fn did_exceed_in_single_time_frame( &mut self, @@ -36,38 +36,29 @@ pub(crate) trait RateLimitOperations { #[async_trait] impl RateLimitOperations for Db { - async fn upsert_address_rate_in_pipe( + fn upsert_address_rate_in_pipe( &mut self, pipe: &mut Pipeline, db: &str, address: &str, expire_time: usize, - ) -> GenericResult<()> { + ) { // Atomic operation, which means it increments the value safely even when multiple clients are modifying the counter simultaneously. pipe.atomic() .hincr(db, address, 1) // Increment the hash value or create it if it doesn't exist - .expire(db, expire_time) // Set or reset the expiration time - .query_async(&mut self.connection) - .await?; - - Ok(()) + .expire(db, expire_time); // Set or reset the expiration time } /// semi-lazy IP rate implementation for 5 different time frames. async fn rate_address(&mut self, address: String) -> GenericResult<()> { let mut pipe = redis::pipe(); - self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_1_MIN, &address, 60) - .await?; - self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_5_MIN, &address, 300) - .await?; - self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_15_MIN, &address, 900) - .await?; - self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_30_MIN, &address, 1800) - .await?; - self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_60_MIN, &address, 3600) - .await?; - + self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_1_MIN, &address, 60); + self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_5_MIN, &address, 300); + self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_15_MIN, &address, 900); + self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_30_MIN, &address, 1800); + self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_60_MIN, &address, 3600); + // Execute the pipeline once after setting all commands pipe.query_async(&mut self.connection).await?; Ok(()) From d9922eeb44c694c160e2084d8ce0499bd86fde85 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 15 May 2024 11:52:42 +0700 Subject: [PATCH 22/39] remove #[allow(dead_code)] in ProofOfFundingError --- src/proxy/quicknode.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs index f5ad50c..f0273ba 100644 --- a/src/proxy/quicknode.rs +++ b/src/proxy/quicknode.rs @@ -32,7 +32,6 @@ enum ProofOfFundingError { InvalidSignedMessage, InsufficientBalance, ErrorFromRpcCall, - #[allow(dead_code)] RpcCallFailed(String), } From 12f1944b6733837b32808d287ab99855e51a03c0 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 16 May 2024 13:40:13 +0700 Subject: [PATCH 23/39] review: update doc comments and fields --- README.md | 2 +- assets/.config_test | 6 +++--- src/ctx.rs | 32 +++++++++++++++++--------------- src/net/server.rs | 4 ++-- src/net/websocket.rs | 2 +- src/proxy/moralis.rs | 2 +- src/proxy/quicknode.rs | 2 +- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c1403f4..73ef63d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Create the configuration file for app runtime. "inbound_route": "/dev", "outbound_route": "http://localhost:8000", "authorized": false, - "allowed_methods": [ + "allowed_rpc_methods": [ "eth_blockNumber", "eth_gasPrice" ] diff --git a/assets/.config_test b/assets/.config_test index 20aff44..b746995 100644 --- a/assets/.config_test +++ b/assets/.config_test @@ -10,7 +10,7 @@ "outbound_route": "https://komodoplatform.com", "proxy_type": "quicknode", "authorized": false, - "allowed_methods": [], + "allowed_rpc_methods": [], "rate_limiter": null }, { @@ -18,7 +18,7 @@ "outbound_route": "https://atomicdex.io", "proxy_type": "quicknode", "authorized": false, - "allowed_methods": [], + "allowed_rpc_methods": [], "rate_limiter": null }, { @@ -26,7 +26,7 @@ "outbound_route": "https://nft.proxy", "proxy_type": "moralis", "authorized": false, - "allowed_methods": [], + "allowed_rpc_methods": [], "rate_limiter": { "rp_1_min": 60, "rp_5_min": 200, diff --git a/src/ctx.rs b/src/ctx.rs index 8001294..12483f7 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -6,6 +6,7 @@ use std::env; pub(crate) use super::*; const DEFAULT_TOKEN_EXPIRATION_TIME: i64 = 3600; +pub(crate) const DEFAULT_PORT: u16 = 5000; static CONFIG: OnceCell = OnceCell::new(); pub(crate) fn get_app_config() -> &'static AppConfig { @@ -17,19 +18,20 @@ pub(crate) fn get_app_config() -> &'static AppConfig { /// Configuration settings for the application, loaded typically from a JSON configuration file. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) struct AppConfig { - /// Optional server port to listen on. If None in config file, then 5000 is default. + /// Optional server port to listen on. If None in config file, then [DEFAULT_PORT] will be used. pub(crate) port: Option, /// Redis database connection string. pub(crate) redis_connection_string: String, - /// File path to the public key used for cryptographic operations. + /// File path to the public key used for user verification and authentication. pub(crate) pubkey_path: String, - /// File path to the private key used for cryptographic operations. + /// File path to the private key used for user verification and authentication. pub(crate) privkey_path: String, - /// Optional token expiration time in seconds. If None then 3600 is default. + /// Optional token expiration time in seconds. + /// If None then the [DEFAULT_TOKEN_EXPIRATION_TIME] will be used. pub(crate) token_expiration_time: Option, - /// Routing configurations for proxying requests. + /// List of proxy routes. pub(crate) proxy_routes: Vec, - /// Default rate limiting settings for request handling. + /// The default rate limiting rules for maintaining the frequency of incoming traffic for per client. pub(crate) rate_limiter: RateLimiter, } @@ -41,14 +43,14 @@ pub(crate) struct ProxyRoute { pub(crate) inbound_route: String, /// The target URL to which requests are forwarded. pub(crate) outbound_route: String, - /// The type of proxying to perform (e.g., JSON-RPC Call, HTTP GET). + /// The type of proxying to perform, directing requests to the appropriate service or API. pub(crate) proxy_type: ProxyType, /// Whether authorization is required for this route. #[serde(default)] pub(crate) authorized: bool, - // Specific HTTP methods allowed for this route. + /// Specific RPC methods allowed for this route. #[serde(default)] - pub(crate) allowed_methods: Vec, + pub(crate) allowed_rpc_methods: Vec, /// Optional custom rate limiter configuration for this route. If provided, /// this configuration will be used instead of the default rate limiting settings. pub(crate) rate_limiter: Option, @@ -105,7 +107,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { outbound_route: String::from("https://komodoplatform.com"), proxy_type: ProxyType::Quicknode, authorized: false, - allowed_methods: Vec::default(), + allowed_rpc_methods: Vec::default(), rate_limiter: None, }, ProxyRoute { @@ -113,7 +115,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { outbound_route: String::from("https://atomicdex.io"), proxy_type: ProxyType::Quicknode, authorized: false, - allowed_methods: Vec::default(), + allowed_rpc_methods: Vec::default(), rate_limiter: None, }, ProxyRoute { @@ -121,7 +123,7 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { outbound_route: String::from("https://nft.proxy"), proxy_type: ProxyType::Moralis, authorized: false, - allowed_methods: Vec::default(), + allowed_rpc_methods: Vec::default(), rate_limiter: Some(RateLimiter { rp_1_min: 60, rp_5_min: 200, @@ -155,7 +157,7 @@ fn test_app_config_serialzation_and_deserialization() { "outbound_route": "https://komodoplatform.com", "proxy_type":"quicknode", "authorized": false, - "allowed_methods": [], + "allowed_rpc_methods": [], "rate_limiter": null }, { @@ -163,7 +165,7 @@ fn test_app_config_serialzation_and_deserialization() { "outbound_route": "https://atomicdex.io", "proxy_type":"quicknode", "authorized": false, - "allowed_methods": [], + "allowed_rpc_methods": [], "rate_limiter": null }, { @@ -171,7 +173,7 @@ fn test_app_config_serialzation_and_deserialization() { "outbound_route": "https://nft.proxy", "proxy_type":"moralis", "authorized": false, - "allowed_methods": [], + "allowed_rpc_methods": [], "rate_limiter": { "rp_1_min": 60, "rp_5_min": 200, diff --git a/src/net/server.rs b/src/net/server.rs index 231f1e9..790a3ff 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -6,7 +6,7 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server, StatusCode}; use super::{GenericError, GenericResult}; -use crate::ctx::AppConfig; +use crate::ctx::{AppConfig, DEFAULT_PORT}; use crate::http::{http_handler, response_by_status, X_FORWARDED_FOR}; use crate::log_format; use crate::websocket::{should_upgrade_to_socket_conn, socket_handler}; @@ -72,7 +72,7 @@ async fn connection_handler( /// Starts serving the proxy API on the configured port. This function sets up the HTTP server, /// binds it to the specified address, and listens for incoming requests. pub(crate) async fn serve(cfg: &'static AppConfig) -> GenericResult<()> { - let addr = format!("0.0.0.0:{}", cfg.port.unwrap_or(5000)).parse()?; + let addr = format!("0.0.0.0:{}", cfg.port.unwrap_or(DEFAULT_PORT)).parse()?; let handler = make_service_fn(move |c_stream: &AddrStream| { let remote_addr = c_stream.remote_addr(); diff --git a/src/net/websocket.rs b/src/net/websocket.rs index 5c74a28..75132da 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -157,7 +157,7 @@ pub(crate) async fn socket_handler( }; - if !proxy_route.allowed_methods.contains(&payload.method) { + if !proxy_route.allowed_rpc_methods.contains(&payload.method) { if let Err(e) = inbound_socket.send("Method not allowed.".into()).await { log::error!( "{}", diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index c93ecef..c1bbb06 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -313,7 +313,7 @@ async fn test_modify_request_uri() { outbound_route: "http://localhost:8000".to_string(), proxy_type: ProxyType::Moralis, authorized: false, - allowed_methods: vec![], + allowed_rpc_methods: vec![], rate_limiter: None, }; diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs index f0273ba..bd2ca78 100644 --- a/src/proxy/quicknode.rs +++ b/src/proxy/quicknode.rs @@ -44,7 +44,7 @@ pub(crate) async fn proxy_quicknode( proxy_route: &ProxyRoute, ) -> GenericResult> { // check if requested method allowed - if !proxy_route.allowed_methods.contains(&payload.method) { + if !proxy_route.allowed_rpc_methods.contains(&payload.method) { log::warn!( "{}", log_format!( From 95f909d814021778e1fa5f753d42de914c4d45cc Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 16 May 2024 13:51:21 +0700 Subject: [PATCH 24/39] readme: provide Docker Compose commands and update the configuration file description --- README.md | 76 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 73ef63d..24655f4 100644 --- a/README.md +++ b/README.md @@ -12,29 +12,31 @@ Create the configuration file for app runtime. ```json { - "port": 6150, - "pubkey_path": "/path_to_publick_key.pem", - "privkey_path": "/path_to_private_key.pem", - "redis_connection_string": "redis://localhost", - "token_expiration_time": 300, - "proxy_routes": [ - { - "inbound_route": "/dev", - "outbound_route": "http://localhost:8000", - "authorized": false, - "allowed_rpc_methods": [ - "eth_blockNumber", - "eth_gasPrice" - ] - } - ], - "rate_limiter": { - "rp_1_min": 30, - "rp_5_min": 100, - "rp_15_min": 200, - "rp_30_min": 350, - "rp_60_min": 575 - } + "port": 6150, + "pubkey_path": "/path_to_publick_key.pem", + "privkey_path": "/path_to_private_key.pem", + "redis_connection_string": "redis://localhost", + "token_expiration_time": 300, + "proxy_routes": [ + { + "inbound_route": "/dev", + "outbound_route": "http://localhost:8000", + "proxy_type": "quicknode", + "authorized": false, + "allowed_rpc_methods": [ + "eth_blockNumber", + "eth_gasPrice" + ], + "rate_limiter": null + } + ], + "rate_limiter": { + "rp_1_min": 30, + "rp_5_min": 100, + "rp_15_min": 200, + "rp_30_min": 350, + "rp_60_min": 575 + } } ``` @@ -97,3 +99,31 @@ curl -v --url "'$mm2_address'" -s --data '{ "id": 0 }' ``` + +### How to run KomodoDefi-Proxy Service with Docker Compose + +If you want to test features locally, you can run Docker containers using Docker Compose commands. + +1. **Update Configuration**: + In the `.config_test` file, update the `proxy_routes` field by adding `ProxyRoutes` with the necessary parameters. + +2. **Build the Docker Image**: + ```sh + docker compose build + ``` + +3. **Run Containers in Detached Mode**: + ```sh + docker compose up -d + ``` + +4. **Follow the Logs**: + Open a new terminal window or tab and execute this command to follow the logs of all services defined in the Docker Compose file. The `-f` (or `--follow`) option ensures that new log entries are continuously displayed as they are produced, while the `-t` (or `--timestamps`) option adds timestamps to each log entry. + ```sh + docker compose logs -f -t + ``` + +5. **Stop the Containers**: + ```sh + docker compose down + ``` \ No newline at end of file From cd04389644ea4144218aad049e7283321809ce36 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 16 May 2024 14:11:42 +0700 Subject: [PATCH 25/39] remove empty lines between some modules in main.rs --- src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 49d9dec..f0c8f52 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,6 @@ mod db; mod http; #[path = "security/jwt.rs"] mod jwt; - #[path = "security/rate_limiter.rs"] mod rate_limiter; #[path = "net/rpc.rs"] @@ -21,7 +20,6 @@ mod server; mod sign; #[path = "net/websocket.rs"] mod websocket; - mod proxy; #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))] From 7e87c8fbe566f8376ae8ea88b682b193c0e01860 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 16 May 2024 14:22:42 +0700 Subject: [PATCH 26/39] add #[allow(dead_code)] annotation --- src/main.rs | 2 +- src/proxy/quicknode.rs | 1 + src/security/address_status.rs | 2 ++ src/security/sign.rs | 2 ++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index f0c8f52..2db1909 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ mod db; mod http; #[path = "security/jwt.rs"] mod jwt; +mod proxy; #[path = "security/rate_limiter.rs"] mod rate_limiter; #[path = "net/rpc.rs"] @@ -20,7 +21,6 @@ mod server; mod sign; #[path = "net/websocket.rs"] mod websocket; -mod proxy; #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))] #[global_allocator] diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs index bd2ca78..676a18b 100644 --- a/src/proxy/quicknode.rs +++ b/src/proxy/quicknode.rs @@ -32,6 +32,7 @@ enum ProofOfFundingError { InvalidSignedMessage, InsufficientBalance, ErrorFromRpcCall, + #[allow(dead_code)] RpcCallFailed(String), } diff --git a/src/security/address_status.rs b/src/security/address_status.rs index 3347e79..05b0d50 100644 --- a/src/security/address_status.rs +++ b/src/security/address_status.rs @@ -82,6 +82,7 @@ impl FromRedisValue for AddressStatus { #[async_trait] pub(crate) trait AddressStatusOperations { + #[allow(dead_code)] async fn insert_address_status( &mut self, address: String, @@ -97,6 +98,7 @@ pub(crate) trait AddressStatusOperations { #[async_trait] impl AddressStatusOperations for Db { + #[allow(dead_code)] async fn insert_address_status( &mut self, address: String, diff --git a/src/security/sign.rs b/src/security/sign.rs index ce9c34a..33a0945 100644 --- a/src/security/sign.rs +++ b/src/security/sign.rs @@ -16,6 +16,7 @@ pub(crate) trait SignOps { fn is_valid_checksum_addr(&self) -> bool; fn valid_addr_from_str(&self) -> Result; fn addr_from_str(&self) -> Result; + #[allow(dead_code)] fn sign_message(&mut self, secret: &Secret) -> GenericResult<()>; fn verify_message(&self) -> GenericResult; } @@ -104,6 +105,7 @@ impl SignOps for SignedMessage { Address::from_str(&self.address[2..]).map_err(|e| e.to_string()) } + #[allow(dead_code)] fn sign_message(&mut self, secret: &Secret) -> GenericResult<()> { let signature = sign(secret, &H256::from(self.sign_message_hash()))?; self.signature = format!("0x{}", signature); From e8f296db9068610699423629fde554faa2d3a632 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 16 May 2024 20:41:28 +0700 Subject: [PATCH 27/39] impl parse_header_payload func --- src/net/http.rs | 5 +++-- src/proxy/mod.rs | 39 ++++++++++++++++++++++++--------------- src/proxy/moralis.rs | 21 +++++++++++++-------- src/proxy/quicknode.rs | 4 ++-- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index 7492330..3736d8a 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -112,14 +112,15 @@ pub(crate) async fn http_handler( let (req, payload) = match generate_payload_from_req(req, &proxy_route.proxy_type).await { Ok(t) => t, - Err(_) => { + Err(e) => { log::warn!( "{}", log_format!( remote_addr.ip(), String::from("-"), req_uri, - "Received invalid http payload, returning 401." + "Received invalid http payload: {}, returning 401.", + e ) ); return response_by_status(StatusCode::UNAUTHORIZED); diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index 1a6ec4b..dd46b89 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -1,7 +1,8 @@ use crate::ctx::{AppConfig, GenericResult, ProxyRoute}; use crate::sign::SignedMessage; use hyper::header::HeaderValue; -use hyper::{Body, Method, Request, Response, StatusCode, Uri}; +use hyper::{Body, Request, Response, StatusCode, Uri}; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; mod moralis; @@ -9,6 +10,8 @@ use moralis::{proxy_moralis, validation_middleware_moralis, MoralisPayload}; mod quicknode; pub(crate) use quicknode::{proxy_quicknode, validation_middleware_quicknode, QuicknodePayload}; +const X_AUTH_PAYLOAD: &str = "X-Auth-Payload"; + /// Enumerates different proxy types supported by the application, focusing on separating feature logic. /// This allows for differentiated handling based on what the proxy should do with the request, /// directing each to the appropriate service or API based on its designated proxy type. @@ -46,11 +49,11 @@ pub(crate) async fn generate_payload_from_req( ) -> GenericResult<(Request, PayloadData)> { match proxy_type { ProxyType::Quicknode => { - let (req, payload) = parse_payload::(req, false).await?; + let (req, payload) = parse_body_payload::(req).await?; Ok((req, PayloadData::Quicknode(payload))) } ProxyType::Moralis => { - let (req, payload) = parse_payload::(req, true).await?; + let (req, payload) = parse_header_payload::(req).await?; Ok((req, PayloadData::Moralis(payload))) } } @@ -94,25 +97,31 @@ pub(crate) async fn validation_middleware( /// Asynchronously parses an HTTP request's body into a specified type `T`. If the request method is `GET`, /// the function modifies the request to have an empty body. For other methods, it retains the original body. /// The function ensures that the body is not empty before attempting deserialization into the non-optional type `T`. -async fn parse_payload(req: Request, get_req: bool) -> GenericResult<(Request, T)> +async fn parse_body_payload(req: Request) -> GenericResult<(Request, T)> where - T: serde::de::DeserializeOwned, + T: DeserializeOwned, { - let (mut parts, body) = req.into_parts(); + let (parts, body) = req.into_parts(); let body_bytes = hyper::body::to_bytes(body).await?; - if body_bytes.is_empty() { return Err("Empty body cannot be deserialized into non-optional type T".into()); } - let payload: T = serde_json::from_slice(&body_bytes)?; + let new_req = Request::from_parts(parts, Body::from(body_bytes)); + Ok((new_req, payload)) +} - let new_req = if get_req { - parts.method = Method::GET; - Request::from_parts(parts, Body::empty()) - } else { - Request::from_parts(parts, Body::from(body_bytes)) - }; - +async fn parse_header_payload(req: Request) -> GenericResult<(Request, T)> +where + T: DeserializeOwned, +{ + let (parts, body) = req.into_parts(); + let header_value = parts + .headers + .get(X_AUTH_PAYLOAD) + .ok_or("Missing X-Auth-Payload header")? + .to_str()?; + let payload: T = serde_json::from_str(header_value)?; + let new_req = Request::from_parts(parts, body); Ok((new_req, payload)) } diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index c1bbb06..6babe57 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -245,7 +245,8 @@ pub(crate) async fn validation_middleware_moralis( #[tokio::test] async fn test_parse_moralis_payload() { - use super::parse_payload; + use super::{parse_header_payload, X_AUTH_PAYLOAD}; + use hyper::Method; let serialized_payload = serde_json::json!({ "uri": "https://example.com/test-path", @@ -258,13 +259,17 @@ async fn test_parse_moralis_payload() { }) .to_string(); - let mut req = Request::new(Body::from(serialized_payload)); - req.headers_mut().insert( - HeaderName::from_static("accept"), - APPLICATION_JSON.parse().unwrap(), - ); + let req = Request::builder() + .method(Method::GET) + .header(header::ACCEPT, HeaderValue::from_static(APPLICATION_JSON)) + .header( + X_AUTH_PAYLOAD, + HeaderValue::from_str(&serialized_payload).unwrap(), + ) + .body(Body::empty()) + .unwrap(); - let (mut req, payload) = parse_payload::(req, true).await.unwrap(); + let (mut req, payload) = parse_header_payload::(req).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( @@ -272,7 +277,7 @@ async fn test_parse_moralis_payload() { "Body should be empty for GET methods" ); - let header_value = req.headers().get("accept").unwrap(); + let header_value = req.headers().get(header::ACCEPT).unwrap(); let expected_payload = MoralisPayload { uri: Url::from_str("https://example.com/test-path").unwrap(), diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs index 676a18b..892d9b6 100644 --- a/src/proxy/quicknode.rs +++ b/src/proxy/quicknode.rs @@ -338,7 +338,7 @@ fn test_quicknode_payload_serialzation_and_deserialization() { #[tokio::test] async fn test_parse_quicknode_payload() { - use super::parse_payload; + use super::parse_body_payload; use hyper::Method; let serialized_payload = json!({ @@ -365,7 +365,7 @@ async fn test_parse_quicknode_payload() { ); let (mut req, payload): (Request, QuicknodePayload) = - parse_payload::(req, false).await.unwrap(); + parse_body_payload::(req).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( From b28a24461e0d4c1dc47bdfd7c208d76053ff0631 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 17 May 2024 11:33:44 +0700 Subject: [PATCH 28/39] impl remove_unnecessary_headers func to avoid some code duplication --- src/proxy/mod.rs | 33 ++++++++++++++++++++++++++++++++- src/proxy/moralis.rs | 31 ++++++++++++++----------------- src/proxy/quicknode.rs | 19 ++++--------------- 3 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index dd46b89..fa5ea11 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -1,6 +1,7 @@ use crate::ctx::{AppConfig, GenericResult, ProxyRoute}; use crate::sign::SignedMessage; -use hyper::header::HeaderValue; +use hyper::header; +use hyper::header::{HeaderName, HeaderValue}; use hyper::{Body, Request, Response, StatusCode, Uri}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -11,6 +12,7 @@ mod quicknode; pub(crate) use quicknode::{proxy_quicknode, validation_middleware_quicknode, QuicknodePayload}; const X_AUTH_PAYLOAD: &str = "X-Auth-Payload"; +const KEEP_ALIVE: &str = "keep-alive"; /// Enumerates different proxy types supported by the application, focusing on separating feature logic. /// This allows for differentiated handling based on what the proxy should do with the request, @@ -125,3 +127,32 @@ where let new_req = Request::from_parts(parts, body); Ok((new_req, payload)) } + +fn remove_unnecessary_headers( + req: &mut Request, + additional_headers_to_remove: &[HeaderName], +) -> GenericResult<()> { + // List of common hop headers to be removed + let mut headers_to_remove = vec![ + header::ACCEPT_ENCODING, + header::CONNECTION, + header::HOST, + header::PROXY_AUTHENTICATE, + header::PROXY_AUTHORIZATION, + header::TE, + header::TRANSFER_ENCODING, + header::TRAILER, + header::UPGRADE, + HeaderName::from_static(KEEP_ALIVE), + ]; + + // Extend with additional headers to remove + headers_to_remove.extend_from_slice(additional_headers_to_remove); + + // Remove headers + for key in &headers_to_remove { + req.headers_mut().remove(key); + } + + Ok(()) +} diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index 6babe57..026e837 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -4,6 +4,7 @@ use crate::db::Db; use crate::http::{ insert_jwt_to_http_header, response_by_status, APPLICATION_JSON, X_FORWARDED_FOR, }; +use crate::proxy::{remove_unnecessary_headers, X_AUTH_PAYLOAD}; use crate::rate_limiter::RateLimitOperations; use crate::sign::{SignOps, SignedMessage}; use crate::{log_format, GenericResult}; @@ -65,27 +66,16 @@ pub(crate) async fn proxy_moralis( return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); } - // drop hop headers - for key in &[ - header::ACCEPT_ENCODING, - header::CONNECTION, - header::HOST, - header::PROXY_AUTHENTICATE, - header::PROXY_AUTHORIZATION, - header::TE, - header::TRANSFER_ENCODING, - header::TRAILER, - header::UPGRADE, + let additional_headers = &[ header::CONTENT_LENGTH, - header::HeaderName::from_static("keep-alive"), - ] { - req.headers_mut().remove(key); - } + HeaderName::from_bytes(X_AUTH_PAYLOAD.as_bytes())?, + ]; + remove_unnecessary_headers(&mut req, additional_headers)?; req.headers_mut() .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); req.headers_mut() - .insert(header::CONTENT_TYPE, APPLICATION_JSON.parse()?); + .insert(header::ACCEPT, APPLICATION_JSON.parse()?); let https = HttpsConnector::new(); let client = hyper::Client::builder().build(https); @@ -245,7 +235,8 @@ pub(crate) async fn validation_middleware_moralis( #[tokio::test] async fn test_parse_moralis_payload() { - use super::{parse_header_payload, X_AUTH_PAYLOAD}; + use super::parse_header_payload; + use hyper::header::HeaderName; use hyper::Method; let serialized_payload = serde_json::json!({ @@ -291,6 +282,12 @@ async fn test_parse_moralis_payload() { assert_eq!(payload, expected_payload); assert_eq!(header_value, APPLICATION_JSON); + + let additional_headers = &[ + header::CONTENT_LENGTH, + HeaderName::from_bytes(X_AUTH_PAYLOAD.as_bytes()).unwrap(), + ]; + remove_unnecessary_headers(&mut req, additional_headers).unwrap(); } #[tokio::test] diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs index 892d9b6..832eb98 100644 --- a/src/proxy/quicknode.rs +++ b/src/proxy/quicknode.rs @@ -4,6 +4,7 @@ use crate::db::Db; use crate::http::{ insert_jwt_to_http_header, response_by_status, APPLICATION_JSON, X_FORWARDED_FOR, }; +use crate::proxy::remove_unnecessary_headers; use crate::rate_limiter::RateLimitOperations; use crate::rpc::Json; use crate::sign::{SignOps, SignedMessage}; @@ -97,21 +98,7 @@ pub(crate) async fn proxy_quicknode( } }; - // drop hop headers - for key in &[ - header::ACCEPT_ENCODING, - header::CONNECTION, - header::HOST, - header::PROXY_AUTHENTICATE, - header::PROXY_AUTHORIZATION, - header::TE, - header::TRANSFER_ENCODING, - header::TRAILER, - header::UPGRADE, - header::HeaderName::from_static("keep-alive"), - ] { - req.headers_mut().remove(key); - } + remove_unnecessary_headers(&mut req, &[])?; req.headers_mut() .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); @@ -390,4 +377,6 @@ async fn test_parse_quicknode_payload() { assert_eq!(payload, expected_payload); assert_eq!(header_value, "dummy-value"); + + remove_unnecessary_headers(&mut req, &[]).unwrap(); } From 800de2ce437a7a4c029e5a8f0ecf1285dd93dcbb Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 20 May 2024 15:44:53 +0700 Subject: [PATCH 29/39] review: rename remove_unnecessary_headers to remove_hop_by_hop_headers --- src/proxy/mod.rs | 2 +- src/proxy/moralis.rs | 6 +++--- src/proxy/quicknode.rs | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index fa5ea11..6facc02 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -128,7 +128,7 @@ where Ok((new_req, payload)) } -fn remove_unnecessary_headers( +fn remove_hop_by_hop_headers( req: &mut Request, additional_headers_to_remove: &[HeaderName], ) -> GenericResult<()> { diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index 026e837..56904e1 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -4,7 +4,7 @@ use crate::db::Db; use crate::http::{ insert_jwt_to_http_header, response_by_status, APPLICATION_JSON, X_FORWARDED_FOR, }; -use crate::proxy::{remove_unnecessary_headers, X_AUTH_PAYLOAD}; +use crate::proxy::{remove_hop_by_hop_headers, X_AUTH_PAYLOAD}; use crate::rate_limiter::RateLimitOperations; use crate::sign::{SignOps, SignedMessage}; use crate::{log_format, GenericResult}; @@ -70,7 +70,7 @@ pub(crate) async fn proxy_moralis( header::CONTENT_LENGTH, HeaderName::from_bytes(X_AUTH_PAYLOAD.as_bytes())?, ]; - remove_unnecessary_headers(&mut req, additional_headers)?; + remove_hop_by_hop_headers(&mut req, additional_headers)?; req.headers_mut() .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); @@ -287,7 +287,7 @@ async fn test_parse_moralis_payload() { header::CONTENT_LENGTH, HeaderName::from_bytes(X_AUTH_PAYLOAD.as_bytes()).unwrap(), ]; - remove_unnecessary_headers(&mut req, additional_headers).unwrap(); + remove_hop_by_hop_headers(&mut req, additional_headers).unwrap(); } #[tokio::test] diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs index 832eb98..d81fc29 100644 --- a/src/proxy/quicknode.rs +++ b/src/proxy/quicknode.rs @@ -4,7 +4,7 @@ use crate::db::Db; use crate::http::{ insert_jwt_to_http_header, response_by_status, APPLICATION_JSON, X_FORWARDED_FOR, }; -use crate::proxy::remove_unnecessary_headers; +use crate::proxy::remove_hop_by_hop_headers; use crate::rate_limiter::RateLimitOperations; use crate::rpc::Json; use crate::sign::{SignOps, SignedMessage}; @@ -98,7 +98,7 @@ pub(crate) async fn proxy_quicknode( } }; - remove_unnecessary_headers(&mut req, &[])?; + remove_hop_by_hop_headers(&mut req, &[])?; req.headers_mut() .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); @@ -378,5 +378,5 @@ async fn test_parse_quicknode_payload() { assert_eq!(payload, expected_payload); assert_eq!(header_value, "dummy-value"); - remove_unnecessary_headers(&mut req, &[]).unwrap(); + remove_hop_by_hop_headers(&mut req, &[]).unwrap(); } From 0508f33f630b5f8de970920353d4a2cf9a38b13e Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 20 May 2024 16:03:23 +0700 Subject: [PATCH 30/39] review: update docker section in readme --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 24655f4..bc91f08 100644 --- a/README.md +++ b/README.md @@ -107,23 +107,19 @@ If you want to test features locally, you can run Docker containers using Docker 1. **Update Configuration**: In the `.config_test` file, update the `proxy_routes` field by adding `ProxyRoutes` with the necessary parameters. -2. **Build the Docker Image**: - ```sh - docker compose build - ``` - -3. **Run Containers in Detached Mode**: +2. **Run Containers in Detached Mode**: + To start the containers, run the following command. This will build the images if they are not already built or if changes are detected in the Dockerfile or the build context. ```sh docker compose up -d ``` -4. **Follow the Logs**: +3. **Follow the Logs**: Open a new terminal window or tab and execute this command to follow the logs of all services defined in the Docker Compose file. The `-f` (or `--follow`) option ensures that new log entries are continuously displayed as they are produced, while the `-t` (or `--timestamps`) option adds timestamps to each log entry. ```sh docker compose logs -f -t ``` -5. **Stop the Containers**: +4. **Stop the Containers**: ```sh docker compose down ``` \ No newline at end of file From c639542afc548a1ce9423dbc54584f07149b7fa0 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 20 May 2024 16:46:24 +0700 Subject: [PATCH 31/39] review: update Middleware description in readme --- README.md | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index bc91f08..f30fa1d 100644 --- a/README.md +++ b/README.md @@ -56,17 +56,34 @@ Expose configuration file's path as an environment variable in `AUTH_APP_CONFIG_ 3) If the incoming request comes from the same network, step 4 will be by-passed. -4) Request will be handled in the middleware with: - - Status Checker: Checks if the wallet address status is blocked, allowed, or trusted and does the following: - - Blocked: Return `403 Forbidden` immediately - - Allowed: process continues with the rate limiter - - Trusted: bypass rate limiter and proof of funding - - Rate Limiter: First, verify the signed message, and if not valid, return 401 Unauthorized immediately. If valid, then calculate the request count with time interval specified in the application configuration. If the wallet address sent too many request than the expected amount, process continues with the proof of funding. If not, by-passes the proof of funding. - - Proof of Funding: Return `406 Not Acceptable` if wallet has 0 balance. Otherwise, we assume that request is valid and process continues as usual. +4) Request Handling in the Middleware: -5) Find target route by requested endpoint + **For Quicknode:** + - **Status Checker**: + - **Blocked**: Return `403 Forbidden` immediately. + - **Allowed**: Process continues with the rate limiter. + - **Trusted**: Bypass rate limiter and proof of funding. -6) Check if requested rpc call is allowed in application configuration + - **Rate Limiter**: + - First, verify the signed message. If not valid, return `401 Unauthorized` immediately. + - If valid, calculate the request count with the time interval specified in the application configuration. If the wallet address has sent too many requests than the expected amount, process continues with the proof of funding. If not, bypass the proof of funding. + + - **Proof of Funding**: + - Return `406 Not Acceptable` if the wallet has a 0 balance. Otherwise, assume the request is valid and process it as usual. + + **For Moralis:** + - **Status Checker**: + - **Blocked**: Return `403 Forbidden` immediately. + - **Allowed**: Process continues with the rate limiter. + - **Trusted**: Bypass the rate limiter. + + - **Rate Limiter**: + - First, verify the signed message. If not valid, return `401 Unauthorized` immediately. + - If valid, calculate the request count with the time interval specified in the application configuration. If the wallet address has sent too many requests, return an error `406 Not Acceptable` indicating that the wallet address must wait for some time before making more requests. + +5) Find target route by requested endpoint. + +6) Check if requested rpc call is allowed in application configuration. 7) Generate JWT token with RSA algorithm using pub-priv keys specified in the application configuration, and insert the token to the request header. From e607cbbea250755e7750dac7048abffdde1345ad Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 23 May 2024 13:52:23 +0700 Subject: [PATCH 32/39] update GET req handling for Moralis --- src/ctx.rs | 13 ++-- src/net/http.rs | 18 +++--- src/net/websocket.rs | 2 +- src/proxy/mod.rs | 9 +-- src/proxy/moralis.rs | 138 +++++++++++++++++++------------------------ 5 files changed, 83 insertions(+), 97 deletions(-) diff --git a/src/ctx.rs b/src/ctx.rs index 12483f7..6d5192c 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -80,16 +80,15 @@ impl AppConfig { .unwrap_or(DEFAULT_TOKEN_EXPIRATION_TIME) } - pub(crate) fn get_proxy_route_by_inbound(&self, inbound: String) -> Option<&ProxyRoute> { + pub(crate) fn get_proxy_route_by_inbound(&self, inbound: &str) -> Option<&ProxyRoute> { let route_index = self.proxy_routes.iter().position(|r| { - r.inbound_route == inbound || r.inbound_route.to_owned() + "/" == inbound + r.inbound_route == inbound + || r.inbound_route == "/".to_owned() + inbound + || r.inbound_route.to_owned() + "/" == inbound + || "/".to_owned() + &*r.inbound_route == "/".to_owned() + inbound }); - if let Some(index) = route_index { - return Some(&self.proxy_routes[index]); - } - - None + route_index.map(|index| &self.proxy_routes[index]) } } diff --git a/src/net/http.rs b/src/net/http.rs index 3736d8a..a107083 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -93,8 +93,14 @@ pub(crate) async fn http_handler( return handle_preflight(); } + let inbound_route = match req.method() { + // should take the second element as path string starts with a delimiter + &Method::GET => req.uri().path().split('/').nth(1).unwrap_or("").to_string(), + _ => req.uri().path().to_string(), + }; + // create proxy_route before payload, as we need proxy_type from it for payload generation - let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().path().to_string()) { + let proxy_route = match cfg.get_proxy_route_by_inbound(&inbound_route) { Some(proxy_route) => proxy_route, None => { log::warn!( @@ -192,21 +198,17 @@ fn test_get_proxy_route_by_inbound() { // If we leave this code line `let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().to_string()) {` // inbound_route cant be "/test", as it's not uri. I suppose inbound actually should be a Path. // Two options: in `req.uri().to_string()` path() is missing or "/test" in test is wrong and the whole url should be. - let proxy_route = cfg - .get_proxy_route_by_inbound(String::from("/test")) - .unwrap(); + let proxy_route = cfg.get_proxy_route_by_inbound("/test").unwrap(); assert_eq!(proxy_route.outbound_route, "https://komodoplatform.com"); - let proxy_route = cfg - .get_proxy_route_by_inbound(String::from("/test-2")) - .unwrap(); + let proxy_route = cfg.get_proxy_route_by_inbound("/test-2").unwrap(); assert_eq!(proxy_route.outbound_route, "https://atomicdex.io"); let url = Uri::from_str("https://komodo.proxy:5535/nft-test").unwrap(); let path = url.path().to_string(); - let proxy_route = cfg.get_proxy_route_by_inbound(path).unwrap(); + let proxy_route = cfg.get_proxy_route_by_inbound(&path).unwrap(); assert_eq!(proxy_route.outbound_route, "https://nft.proxy"); } diff --git a/src/net/websocket.rs b/src/net/websocket.rs index 75132da..0b02782 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -27,7 +27,7 @@ pub(crate) async fn socket_handler( remote_addr: SocketAddr, ) -> GenericResult> { let inbound_route = req.uri().to_string(); - let proxy_route = match cfg.get_proxy_route_by_inbound(inbound_route) { + let proxy_route = match cfg.get_proxy_route_by_inbound(&inbound_route) { Some(proxy_route) => proxy_route.clone(), None => { log::warn!( diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index 6facc02..993204b 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -7,7 +7,7 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; mod moralis; -use moralis::{proxy_moralis, validation_middleware_moralis, MoralisPayload}; +use moralis::{proxy_moralis, validation_middleware_moralis}; mod quicknode; pub(crate) use quicknode::{proxy_quicknode, validation_middleware_quicknode, QuicknodePayload}; @@ -29,7 +29,8 @@ pub(crate) enum ProxyType { #[derive(Clone, Debug, PartialEq)] pub(crate) enum PayloadData { Quicknode(QuicknodePayload), - Moralis(MoralisPayload), + /// Moralis feature requires only Signed Message in X-Auth-Payload header + Moralis(SignedMessage), } impl PayloadData { @@ -37,7 +38,7 @@ impl PayloadData { pub(crate) fn signed_message(&self) -> &SignedMessage { match self { PayloadData::Quicknode(payload) => &payload.signed_message, - PayloadData::Moralis(payload) => &payload.signed_message, + PayloadData::Moralis(payload) => payload, } } } @@ -55,7 +56,7 @@ pub(crate) async fn generate_payload_from_req( Ok((req, PayloadData::Quicknode(payload))) } ProxyType::Moralis => { - let (req, payload) = parse_header_payload::(req).await?; + let (req, payload) = parse_header_payload::(req).await?; Ok((req, PayloadData::Moralis(payload))) } } diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index 56904e1..2205f36 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -12,25 +12,14 @@ use hyper::header::{HeaderName, HeaderValue}; use hyper::http::uri::PathAndQuery; use hyper::{header, Body, Request, Response, StatusCode, Uri}; use hyper_tls::HttpsConnector; -use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::str::FromStr; -use url::Url; - -/// Represents a payload for HTTP GET requests, specifically parsed for the Moralis API within the proxy. -/// This struct contains the destination URL that the proxy will forward the GET request to, ensuring correct service routing. -/// It also includes a `SignedMessage` for authentication and validation, confirming the legitimacy of the request and enhancing security. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub(crate) struct MoralisPayload { - uri: Url, - pub(crate) signed_message: SignedMessage, -} pub(crate) async fn proxy_moralis( cfg: &AppConfig, mut req: Request, remote_addr: &SocketAddr, - payload: MoralisPayload, + signed_message: SignedMessage, x_forwarded_for: HeaderValue, proxy_route: &ProxyRoute, ) -> GenericResult> { @@ -40,7 +29,7 @@ pub(crate) async fn proxy_moralis( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req.uri(), "Error inserting JWT into HTTP header: {}, returning 500.", e @@ -52,12 +41,12 @@ pub(crate) async fn proxy_moralis( let original_req_uri = req.uri().clone(); - if let Err(e) = modify_request_uri(&mut req, &payload, proxy_route).await { + if let Err(e) = modify_request_uri(&mut req, proxy_route).await { log::error!( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, original_req_uri, "Error modifying request Uri: {}, returning 500.", e @@ -88,7 +77,7 @@ pub(crate) async fn proxy_moralis( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, original_req_uri, "Couldn't reach {}: {}. Returning 503.", target_uri, @@ -102,54 +91,70 @@ pub(crate) async fn proxy_moralis( Ok(res) } -/// Modifies the URI of an HTTP request by replacing it to outbound URI specified in `ProxyRoute`, -/// while incorporating the path and query parameters from the payload's URI. +/// Modifies the URI of an HTTP request by replacing its base URI with the outbound URI specified in `ProxyRoute`, +/// while incorporating the path and query parameters from the original request URI. Additionally, this function +/// removes the first path segment from the original URI. async fn modify_request_uri( req: &mut Request, - payload: &MoralisPayload, proxy_route: &ProxyRoute, ) -> GenericResult<()> { - let mut proxy_outbound_parts = proxy_route.outbound_route.parse::()?.into_parts(); + let proxy_base_uri = proxy_route.outbound_route.parse::()?; + + let req_uri = req.uri().clone(); + + // Remove the first path segment + let mut path_segments: Vec<&str> = req_uri + .path() + .split('/') + .filter(|s| !s.is_empty()) + .collect(); + if !path_segments.is_empty() { + path_segments.remove(0); + } + let new_path = format!("/{}", path_segments.join("/")); - let payload_uri: Uri = payload.uri.as_str().parse()?; + // Construct the new path and query + let path_and_query_str = match req_uri.query() { + Some(query) => format!("{}?{}", new_path, query), + None => new_path, + }; - let path_and_query = - PathAndQuery::from_str(payload_uri.path_and_query().map_or("/", |pq| pq.as_str()))?; - // Append the path and query from the payload URI to the proxy outbound URI. + let path_and_query = PathAndQuery::from_str(&path_and_query_str)?; + + // Update the proxy URI with the new path and query + let mut proxy_outbound_parts = proxy_base_uri.into_parts(); proxy_outbound_parts.path_and_query = Some(path_and_query); - // Reconstruct the full URI with the updated parts. + // Reconstruct the full URI with the updated parts let new_uri = Uri::from_parts(proxy_outbound_parts)?; - // Update the request URI. + // Update the request URI *req.uri_mut() = new_uri; + Ok(()) } pub(crate) async fn validation_middleware_moralis( cfg: &AppConfig, - payload: &MoralisPayload, + signed_message: &SignedMessage, proxy_route: &ProxyRoute, req_uri: &Uri, remote_addr: &SocketAddr, ) -> Result<(), StatusCode> { let mut db = Db::create_instance(cfg).await; - match db - .read_address_status(&payload.signed_message.address) - .await - { + match db.read_address_status(&signed_message.address).await { AddressStatus::Trusted => Ok(()), AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), AddressStatus::None => { - match payload.signed_message.verify_message() { + match signed_message.verify_message() { Ok(true) => {} Ok(false) => { log::warn!( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "Request has invalid signed message, returning 401" ) @@ -162,10 +167,10 @@ pub(crate) async fn validation_middleware_moralis( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "verify_message failed in coin {}: {}, returning 500.", - payload.signed_message.coin_ticker, + signed_message.coin_ticker, e ) ); @@ -173,10 +178,8 @@ pub(crate) async fn validation_middleware_moralis( } } - let rate_limiter_key = format!( - "{}:{}", - payload.signed_message.coin_ticker, payload.signed_message.address - ); + let rate_limiter_key = + format!("{}:{}", signed_message.coin_ticker, signed_message.address); let rate_limiter = proxy_route .rate_limiter @@ -189,7 +192,7 @@ pub(crate) async fn validation_middleware_moralis( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "Rate exceed for {}, returning 406.", rate_limiter_key, @@ -202,10 +205,10 @@ pub(crate) async fn validation_middleware_moralis( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "Rate exceeded check failed in coin {}: {}, returning 500.", - payload.signed_message.coin_ticker, + signed_message.coin_ticker, e ) ); @@ -218,10 +221,10 @@ pub(crate) async fn validation_middleware_moralis( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "Rate incrementing failed in coin {}: {}, returning 500.", - payload.signed_message.coin_ticker, + signed_message.coin_ticker, e ) ); @@ -240,13 +243,10 @@ async fn test_parse_moralis_payload() { use hyper::Method; let serialized_payload = serde_json::json!({ - "uri": "https://example.com/test-path", - "signed_message": { - "coin_ticker": "BTC", - "address": "dummy-value", - "timestamp_message": 1655320000, - "signature": "dummy-value", - } + "coin_ticker": "BTC", + "address": "dummy-value", + "timestamp_message": 1655320000, + "signature": "dummy-value", }) .to_string(); @@ -260,7 +260,7 @@ async fn test_parse_moralis_payload() { .body(Body::empty()) .unwrap(); - let (mut req, payload) = parse_header_payload::(req).await.unwrap(); + let (mut req, payload) = parse_header_payload::(req).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( @@ -270,14 +270,11 @@ async fn test_parse_moralis_payload() { let header_value = req.headers().get(header::ACCEPT).unwrap(); - let expected_payload = MoralisPayload { - uri: Url::from_str("https://example.com/test-path").unwrap(), - signed_message: SignedMessage { - coin_ticker: String::from("BTC"), - address: String::from("dummy-value"), - timestamp_message: 1655320000, - signature: String::from("dummy-value"), - }, + let expected_payload = SignedMessage { + coin_ticker: String::from("BTC"), + address: String::from("dummy-value"), + timestamp_message: 1655320000, + signature: String::from("dummy-value"), }; assert_eq!(payload, expected_payload); @@ -294,24 +291,13 @@ async fn test_parse_moralis_payload() { async fn test_modify_request_uri() { use super::ProxyType; - let orig_uri_str = "https://proxy.example:3535/test-inbound"; let mut req = Request::builder() - .uri(orig_uri_str) + .uri("https://komodoproxy.com/nft-proxy/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC") .body(Body::empty()) .unwrap(); - let payload = MoralisPayload { - uri: Url::from_str("https://proxy.example:3535/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false").unwrap(), - signed_message: SignedMessage { - coin_ticker: String::from("BTC"), - address: String::from("dummy-value"), - timestamp_message: 1655320000, - signature: String::from("dummy-value"), - }, - }; - let proxy_route = ProxyRoute { - inbound_route: String::from_str("/test-inbound").unwrap(), + inbound_route: String::from_str("/nft-proxy").unwrap(), outbound_route: "http://localhost:8000".to_string(), proxy_type: ProxyType::Moralis, authorized: false, @@ -319,12 +305,10 @@ async fn test_modify_request_uri() { rate_limiter: None, }; - modify_request_uri(&mut req, &payload, &proxy_route) - .await - .unwrap(); + modify_request_uri(&mut req, &proxy_route).await.unwrap(); assert_eq!( req.uri(), - "http://localhost:8000/api/v2/item/0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB/1?chain=eth&format=decimal&normalizeMetadata=true&media_items=false" + "http://localhost:8000/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC" ); } From e98cf0883a4d5bdb00ccbc3987f6c454387a5bef Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 24 May 2024 14:01:28 +0700 Subject: [PATCH 33/39] use req.uri().path() to find ProxyRote in websocket. Note: in the current stage only quicknode feature supports both websocket and http. --- src/net/websocket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/websocket.rs b/src/net/websocket.rs index 0b02782..0b2aea5 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -26,7 +26,7 @@ pub(crate) async fn socket_handler( mut req: Request, remote_addr: SocketAddr, ) -> GenericResult> { - let inbound_route = req.uri().to_string(); + let inbound_route = req.uri().path().to_string(); let proxy_route = match cfg.get_proxy_route_by_inbound(&inbound_route) { Some(proxy_route) => proxy_route.clone(), None => { From d86b97f2b906b0abf848740a7580b31243528911 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 5 Jun 2024 09:09:20 +0700 Subject: [PATCH 34/39] refactor quicknode: move SignedMessage to Header --- src/net/websocket.rs | 10 +-- src/proxy/mod.rs | 93 +++++++++++++++++++-------- src/proxy/moralis.rs | 12 ++-- src/proxy/quicknode.rs | 140 ++++++++++++++++++++++------------------- 4 files changed, 152 insertions(+), 103 deletions(-) diff --git a/src/net/websocket.rs b/src/net/websocket.rs index 0b2aea5..2b22150 100644 --- a/src/net/websocket.rs +++ b/src/net/websocket.rs @@ -12,7 +12,7 @@ use crate::{ ctx::AppConfig, http::response_by_status, log_format, - proxy::{validation_middleware_quicknode, QuicknodePayload}, + proxy::{validation_middleware_quicknode, QuicknodeSocketPayload}, GenericResult, }; @@ -137,7 +137,7 @@ pub(crate) async fn socket_handler( match msg { Some(Ok(msg)) => { if let Message::Text(msg) = msg { - let payload: QuicknodePayload = match serde_json::from_str(&msg) { + let socket_payload: QuicknodeSocketPayload = match serde_json::from_str(&msg) { Ok(t) => t, Err(e) => { if let Err(e) = inbound_socket.send(format!("Invalid payload. {e}").into()).await { @@ -155,7 +155,7 @@ pub(crate) async fn socket_handler( continue; }, }; - + let (payload, signed_message) = socket_payload.into_parts(); if !proxy_route.allowed_rpc_methods.contains(&payload.method) { if let Err(e) = inbound_socket.send("Method not allowed.".into()).await { @@ -173,10 +173,10 @@ pub(crate) async fn socket_handler( continue; } - // TODO add general validation_middleware support + // TODO add general validation_middleware support (if have new features which support websocket) match validation_middleware_quicknode( &cfg, - &payload, + &signed_message, &proxy_route, req.uri(), &remote_addr, diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index 993204b..1f3d4cb 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -9,7 +9,9 @@ use std::net::SocketAddr; mod moralis; use moralis::{proxy_moralis, validation_middleware_moralis}; mod quicknode; -pub(crate) use quicknode::{proxy_quicknode, validation_middleware_quicknode, QuicknodePayload}; +pub(crate) use quicknode::{ + proxy_quicknode, validation_middleware_quicknode, QuicknodePayload, QuicknodeSocketPayload, +}; const X_AUTH_PAYLOAD: &str = "X-Auth-Payload"; const KEEP_ALIVE: &str = "keep-alive"; @@ -28,7 +30,10 @@ pub(crate) enum ProxyType { /// This helps in managing the logic for routing and processing requests appropriately within the proxy layer. #[derive(Clone, Debug, PartialEq)] pub(crate) enum PayloadData { - Quicknode(QuicknodePayload), + Quicknode { + payload: QuicknodePayload, + signed_message: SignedMessage, + }, /// Moralis feature requires only Signed Message in X-Auth-Payload header Moralis(SignedMessage), } @@ -37,7 +42,7 @@ impl PayloadData { /// Returns a reference to the `SignedMessage` contained within the payload. pub(crate) fn signed_message(&self) -> &SignedMessage { match self { - PayloadData::Quicknode(payload) => &payload.signed_message, + PayloadData::Quicknode { signed_message, .. } => signed_message, PayloadData::Moralis(payload) => payload, } } @@ -45,19 +50,24 @@ impl PayloadData { /// Asynchronously generates and organizes payload data from an HTTP request based on the specified proxy type. /// This function ensures that requests are properly formatted to the correct service, -/// returning a tuple with the modified request and the structured payload. +/// returning a tuple with the request and the structured payload. pub(crate) async fn generate_payload_from_req( req: Request, proxy_type: &ProxyType, ) -> GenericResult<(Request, PayloadData)> { match proxy_type { ProxyType::Quicknode => { - let (req, payload) = parse_body_payload::(req).await?; - Ok((req, PayloadData::Quicknode(payload))) + let (req, payload, signed_message) = + parse_body_payload::(req).await?; + let payload_data = PayloadData::Quicknode { + payload, + signed_message, + }; + Ok((req, payload_data)) } ProxyType::Moralis => { - let (req, payload) = parse_header_payload::(req).await?; - Ok((req, PayloadData::Moralis(payload))) + let (req, signed_message) = parse_header_payload(req).await?; + Ok((req, PayloadData::Moralis(signed_message))) } } } @@ -71,11 +81,31 @@ pub(crate) async fn proxy( proxy_route: &ProxyRoute, ) -> GenericResult> { match payload { - PayloadData::Quicknode(payload) => { - proxy_quicknode(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await + PayloadData::Quicknode { + payload, + signed_message, + } => { + proxy_quicknode( + cfg, + req, + remote_addr, + payload, + signed_message, + x_forwarded_for, + proxy_route, + ) + .await } - PayloadData::Moralis(payload) => { - proxy_moralis(cfg, req, remote_addr, payload, x_forwarded_for, proxy_route).await + PayloadData::Moralis(signed_message) => { + proxy_moralis( + cfg, + req, + remote_addr, + signed_message, + x_forwarded_for, + proxy_route, + ) + .await } } } @@ -88,43 +118,53 @@ pub(crate) async fn validation_middleware( remote_addr: &SocketAddr, ) -> Result<(), StatusCode> { match payload { - PayloadData::Quicknode(payload) => { - validation_middleware_quicknode(cfg, payload, proxy_route, req_uri, remote_addr).await + PayloadData::Quicknode { signed_message, .. } => { + validation_middleware_quicknode(cfg, signed_message, proxy_route, req_uri, remote_addr) + .await } - PayloadData::Moralis(payload) => { - validation_middleware_moralis(cfg, payload, proxy_route, req_uri, remote_addr).await + PayloadData::Moralis(signed_message) => { + validation_middleware_moralis(cfg, signed_message, proxy_route, req_uri, remote_addr) + .await } } } -/// Asynchronously parses an HTTP request's body into a specified type `T`. If the request method is `GET`, -/// the function modifies the request to have an empty body. For other methods, it retains the original body. -/// The function ensures that the body is not empty before attempting deserialization into the non-optional type `T`. -async fn parse_body_payload(req: Request) -> GenericResult<(Request, T)> +/// Parses the request body and the `X-Auth-Payload` header into a payload and signed message. +/// +/// This function extracts the `X-Auth-Payload` header from the request, parses it into a `SignedMessage`, +/// and then reads and deserializes the request body into a specified type `T`. +/// If the body is empty or the header is missing, an error is returned. +async fn parse_body_payload( + req: Request, +) -> GenericResult<(Request, T, SignedMessage)> where T: DeserializeOwned, { let (parts, body) = req.into_parts(); + let header_value = parts + .headers + .get(X_AUTH_PAYLOAD) + .ok_or("Missing X-Auth-Payload header")? + .to_str()?; + let signed_message: SignedMessage = serde_json::from_str(header_value)?; let body_bytes = hyper::body::to_bytes(body).await?; if body_bytes.is_empty() { return Err("Empty body cannot be deserialized into non-optional type T".into()); } let payload: T = serde_json::from_slice(&body_bytes)?; let new_req = Request::from_parts(parts, Body::from(body_bytes)); - Ok((new_req, payload)) + Ok((new_req, payload, signed_message)) } -async fn parse_header_payload(req: Request) -> GenericResult<(Request, T)> -where - T: DeserializeOwned, -{ +/// Parses [SignedMessage] value from X-Auth-Payload header +async fn parse_header_payload(req: Request) -> GenericResult<(Request, SignedMessage)> { let (parts, body) = req.into_parts(); let header_value = parts .headers .get(X_AUTH_PAYLOAD) .ok_or("Missing X-Auth-Payload header")? .to_str()?; - let payload: T = serde_json::from_str(header_value)?; + let payload: SignedMessage = serde_json::from_str(header_value)?; let new_req = Request::from_parts(parts, body); Ok((new_req, payload)) } @@ -145,6 +185,7 @@ fn remove_hop_by_hop_headers( header::TRAILER, header::UPGRADE, HeaderName::from_static(KEEP_ALIVE), + HeaderName::from_bytes(X_AUTH_PAYLOAD.as_bytes())?, ]; // Extend with additional headers to remove diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index 2205f36..269fbd7 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -4,7 +4,7 @@ use crate::db::Db; use crate::http::{ insert_jwt_to_http_header, response_by_status, APPLICATION_JSON, X_FORWARDED_FOR, }; -use crate::proxy::{remove_hop_by_hop_headers, X_AUTH_PAYLOAD}; +use crate::proxy::remove_hop_by_hop_headers; use crate::rate_limiter::RateLimitOperations; use crate::sign::{SignOps, SignedMessage}; use crate::{log_format, GenericResult}; @@ -55,11 +55,7 @@ pub(crate) async fn proxy_moralis( return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); } - let additional_headers = &[ - header::CONTENT_LENGTH, - HeaderName::from_bytes(X_AUTH_PAYLOAD.as_bytes())?, - ]; - remove_hop_by_hop_headers(&mut req, additional_headers)?; + remove_hop_by_hop_headers(&mut req, &[header::CONTENT_LENGTH])?; req.headers_mut() .insert(HeaderName::from_static(X_FORWARDED_FOR), x_forwarded_for); @@ -238,7 +234,7 @@ pub(crate) async fn validation_middleware_moralis( #[tokio::test] async fn test_parse_moralis_payload() { - use super::parse_header_payload; + use super::{parse_header_payload, X_AUTH_PAYLOAD}; use hyper::header::HeaderName; use hyper::Method; @@ -260,7 +256,7 @@ async fn test_parse_moralis_payload() { .body(Body::empty()) .unwrap(); - let (mut req, payload) = parse_header_payload::(req).await.unwrap(); + let (mut req, payload) = parse_header_payload(req).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs index d81fc29..2dda0f3 100644 --- a/src/proxy/quicknode.rs +++ b/src/proxy/quicknode.rs @@ -17,17 +17,39 @@ use serde_json::json; use std::net::SocketAddr; /// Represents a payload for JSON-RPC calls, tailored for the Quicknode API within the proxy. -/// This struct combines standard JSON RPC method call fields (method, params, id, jsonrpc) with a `SignedMessage` -/// for authentication and validation, facilitating secure and validated interactions with the Quicknode service. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub(crate) struct QuicknodePayload { pub(crate) method: String, pub(crate) params: serde_json::value::Value, pub(crate) id: usize, pub(crate) jsonrpc: String, +} + +/// Used for websocket connection. +/// It combines standard JSON RPC method call fields (method, params, id, jsonrpc) with a `SignedMessage` +/// for authentication and validation, facilitating secure and validated interactions with the Quicknode service. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub(crate) struct QuicknodeSocketPayload { + pub(crate) method: String, + pub(crate) params: serde_json::value::Value, + pub(crate) id: usize, + pub(crate) jsonrpc: String, pub(crate) signed_message: SignedMessage, } +impl QuicknodeSocketPayload { + pub(crate) fn into_parts(self) -> (QuicknodePayload, SignedMessage) { + let payload = QuicknodePayload { + method: self.method, + params: self.params, + id: self.id, + jsonrpc: self.jsonrpc, + }; + let signed_message = self.signed_message; + (payload, signed_message) + } +} + #[derive(Debug)] enum ProofOfFundingError { InvalidSignedMessage, @@ -42,6 +64,7 @@ pub(crate) async fn proxy_quicknode( mut req: Request, remote_addr: &SocketAddr, payload: QuicknodePayload, + signed_message: SignedMessage, x_forwarded_for: HeaderValue, proxy_route: &ProxyRoute, ) -> GenericResult> { @@ -51,7 +74,7 @@ pub(crate) async fn proxy_quicknode( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req.uri(), "Method {} not allowed for, returning 403.", payload.method @@ -70,7 +93,7 @@ pub(crate) async fn proxy_quicknode( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req.uri(), "Error inserting JWT into http header, returning 500." ) @@ -87,7 +110,7 @@ pub(crate) async fn proxy_quicknode( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, original_req_uri, "Error type casting value of {} into Uri: {}, returning 500.", proxy_route.outbound_route, @@ -116,7 +139,7 @@ pub(crate) async fn proxy_quicknode( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, original_req_uri, "Couldn't reach {}: {}. Returning 503.", target_uri, @@ -132,28 +155,26 @@ pub(crate) async fn proxy_quicknode( pub(crate) async fn validation_middleware_quicknode( cfg: &AppConfig, - payload: &QuicknodePayload, + signed_message: &SignedMessage, proxy_route: &ProxyRoute, req_uri: &Uri, remote_addr: &SocketAddr, ) -> Result<(), StatusCode> { let mut db = Db::create_instance(cfg).await; - match db - .read_address_status(&payload.signed_message.address) - .await - { + match db.read_address_status(&signed_message.address).await { AddressStatus::Trusted => Ok(()), AddressStatus::Blocked => Err(StatusCode::FORBIDDEN), AddressStatus::None => { - let signed_message_status = verify_message_and_balance(cfg, payload, proxy_route).await; + let signed_message_status = + verify_message_and_balance(cfg, signed_message, proxy_route).await; if let Err(ProofOfFundingError::InvalidSignedMessage) = signed_message_status { log::warn!( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "Request has invalid signed message, returning 401" ) @@ -162,10 +183,8 @@ pub(crate) async fn validation_middleware_quicknode( return Err(StatusCode::UNAUTHORIZED); }; - let rate_limiter_key = format!( - "{}:{}", - payload.signed_message.coin_ticker, payload.signed_message.address - ); + let rate_limiter_key = + format!("{}:{}", signed_message.coin_ticker, signed_message.address); let rate_limiter = proxy_route .rate_limiter @@ -178,26 +197,26 @@ pub(crate) async fn validation_middleware_quicknode( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "Rate exceed for {}, checking balance for {} address.", rate_limiter_key, - payload.signed_message.address + signed_message.address ) ); - match verify_message_and_balance(cfg, payload, proxy_route).await { + match verify_message_and_balance(cfg, signed_message, proxy_route).await { Ok(_) => {} Err(ProofOfFundingError::InsufficientBalance) => { log::warn!( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "Wallet {} has insufficient balance for coin {}, returning 406.", - payload.signed_message.address, - payload.signed_message.coin_ticker, + signed_message.address, + signed_message.coin_ticker, ) ); @@ -208,10 +227,10 @@ pub(crate) async fn validation_middleware_quicknode( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "verify_message_and_balance failed in coin {}: {:?}", - payload.signed_message.coin_ticker, + signed_message.coin_ticker, e ) ); @@ -226,7 +245,7 @@ pub(crate) async fn validation_middleware_quicknode( "{}", log_format!( remote_addr.ip(), - payload.signed_message.address, + signed_message.address, req_uri, "Rate incrementing failed." ) @@ -240,14 +259,14 @@ pub(crate) async fn validation_middleware_quicknode( async fn verify_message_and_balance( cfg: &AppConfig, - payload: &QuicknodePayload, + signed_message: &SignedMessage, proxy_route: &ProxyRoute, ) -> Result<(), ProofOfFundingError> { - if let Ok(true) = payload.signed_message.verify_message() { + if let Ok(true) = signed_message.verify_message() { let mut db = Db::create_instance(cfg).await; // We don't want to send balance requests everytime when user sends requests. - if let Ok(true) = db.key_exists(&payload.signed_message.address).await { + if let Ok(true) = db.key_exists(&signed_message.address).await { return Ok(()); } @@ -255,7 +274,7 @@ async fn verify_message_and_balance( "id": 1, "jsonrpc": "2.0", "method": "eth_getBalance", - "params": [payload.signed_message.address, "latest"] + "params": [signed_message.address, "latest"] }); let rpc_client = @@ -268,9 +287,7 @@ async fn verify_message_and_balance( { Ok(res) if res["result"] != Json::Null && res["result"] != "0x0" => { // cache this address for 60 seconds - let _ = db - .insert_cache(&payload.signed_message.address, "", 60) - .await; + let _ = db.insert_cache(&signed_message.address, "", 60).await; return Ok(()); } @@ -291,13 +308,7 @@ fn test_quicknode_payload_serialzation_and_deserialization() { "method": "dummy-value", "params": [], "id": 1, - "jsonrpc": "2.0", - "signed_message": { - "coin_ticker": "ETH", - "address": "dummy-value", - "timestamp_message": 1655319963, - "signature": "dummy-value", - } + "jsonrpc": "2.0" }); let actual_payload: QuicknodePayload = serde_json::from_str(&json_payload.to_string()).unwrap(); @@ -307,12 +318,6 @@ fn test_quicknode_payload_serialzation_and_deserialization() { params: json!([]), id: 1, jsonrpc: String::from("2.0"), - signed_message: SignedMessage { - coin_ticker: String::from("ETH"), - address: String::from("dummy-value"), - timestamp_message: 1655319963, - signature: String::from("dummy-value"), - }, }; assert_eq!(actual_payload, expected_payload); @@ -325,25 +330,31 @@ fn test_quicknode_payload_serialzation_and_deserialization() { #[tokio::test] async fn test_parse_quicknode_payload() { - use super::parse_body_payload; + use super::{parse_body_payload, X_AUTH_PAYLOAD}; use hyper::Method; let serialized_payload = json!({ "method": "dummy-value", "params": [], "id": 1, - "jsonrpc": "2.0", - "signed_message": { - "coin_ticker": "ETH", - "address": "dummy-value", - "timestamp_message": 1655319963, - "signature": "dummy-value", - } + "jsonrpc": "2.0" + }) + .to_string(); + + let serialized_auth_value = json!({ + "coin_ticker": "ETH", + "address": "dummy-value", + "timestamp_message": 1655319963, + "signature": "dummy-value", }) .to_string(); let mut req = Request::builder() .method(Method::POST) + .header( + X_AUTH_PAYLOAD, + HeaderValue::from_str(&serialized_auth_value).unwrap(), + ) .body(Body::from(serialized_payload)) .unwrap(); req.headers_mut().insert( @@ -351,7 +362,7 @@ async fn test_parse_quicknode_payload() { "dummy-value".parse().unwrap(), ); - let (mut req, payload): (Request, QuicknodePayload) = + let (mut req, payload, signed_message) = parse_body_payload::(req).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); @@ -360,23 +371,24 @@ async fn test_parse_quicknode_payload() { "Body should not be empty for non-GET methods" ); - let header_value = req.headers().get("dummy-header").unwrap(); + let dummy_header_value = req.headers().get("dummy-header").unwrap(); + assert_eq!(dummy_header_value, "dummy-value"); let expected_payload = QuicknodePayload { method: String::from("dummy-value"), params: json!([]), id: 1, jsonrpc: String::from("2.0"), - signed_message: SignedMessage { - coin_ticker: String::from("ETH"), - address: String::from("dummy-value"), - timestamp_message: 1655319963, - signature: String::from("dummy-value"), - }, }; - assert_eq!(payload, expected_payload); - assert_eq!(header_value, "dummy-value"); + + let expected_auth_value = SignedMessage { + coin_ticker: String::from("ETH"), + address: String::from("dummy-value"), + timestamp_message: 1655319963, + signature: String::from("dummy-value"), + }; + assert_eq!(signed_message, expected_auth_value); remove_hop_by_hop_headers(&mut req, &[]).unwrap(); } From 226db21f41c0461ecdb45499daca1467d7fc199c Mon Sep 17 00:00:00 2001 From: laruh Date: Sat, 22 Jun 2024 14:01:48 +0700 Subject: [PATCH 35/39] review: remove note about inbound_route in test --- src/net/http.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index a107083..a7e0534 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -195,9 +195,6 @@ fn test_get_proxy_route_by_inbound() { let cfg = ctx::get_app_config_test_instance(); - // If we leave this code line `let proxy_route = match cfg.get_proxy_route_by_inbound(req.uri().to_string()) {` - // inbound_route cant be "/test", as it's not uri. I suppose inbound actually should be a Path. - // Two options: in `req.uri().to_string()` path() is missing or "/test" in test is wrong and the whole url should be. let proxy_route = cfg.get_proxy_route_by_inbound("/test").unwrap(); assert_eq!(proxy_route.outbound_route, "https://komodoplatform.com"); From e81cfb91ad505da4b36d23e55220d26850d904b8 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 27 Jun 2024 13:33:19 +0700 Subject: [PATCH 36/39] review: update docs, rename parse functions, modify_request_uri non async --- src/net/http.rs | 4 ++-- src/proxy/mod.rs | 11 ++++++----- src/proxy/moralis.rs | 13 +++++-------- src/proxy/quicknode.rs | 7 ++++--- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index a7e0534..437667d 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -11,9 +11,9 @@ use jwt::{get_cached_token_or_generate_one, JwtClaims}; use serde_json::json; use std::net::SocketAddr; -/// Value +/// Header value for `hyper::header::CONTENT_TYPE` pub(crate) const APPLICATION_JSON: &str = "application/json"; -/// Header +/// Represents `X-Forwarded-For` Header key pub(crate) const X_FORWARDED_FOR: &str = "x-forwarded-for"; async fn get_healthcheck() -> GenericResult> { let json = json!({ diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index 1f3d4cb..8471b13 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -30,11 +30,12 @@ pub(crate) enum ProxyType { /// This helps in managing the logic for routing and processing requests appropriately within the proxy layer. #[derive(Clone, Debug, PartialEq)] pub(crate) enum PayloadData { + /// Quicknode feature requires body payload and Signed Message in X-Auth-Payload header Quicknode { payload: QuicknodePayload, signed_message: SignedMessage, }, - /// Moralis feature requires only Signed Message in X-Auth-Payload header + /// Moralis feature requires only Signed Message in X-Auth-Payload header and doesn't have body Moralis(SignedMessage), } @@ -58,7 +59,7 @@ pub(crate) async fn generate_payload_from_req( match proxy_type { ProxyType::Quicknode => { let (req, payload, signed_message) = - parse_body_payload::(req).await?; + parse_body_and_auth_header::(req).await?; let payload_data = PayloadData::Quicknode { payload, signed_message, @@ -66,7 +67,7 @@ pub(crate) async fn generate_payload_from_req( Ok((req, payload_data)) } ProxyType::Moralis => { - let (req, signed_message) = parse_header_payload(req).await?; + let (req, signed_message) = parse_auth_header(req).await?; Ok((req, PayloadData::Moralis(signed_message))) } } @@ -134,7 +135,7 @@ pub(crate) async fn validation_middleware( /// This function extracts the `X-Auth-Payload` header from the request, parses it into a `SignedMessage`, /// and then reads and deserializes the request body into a specified type `T`. /// If the body is empty or the header is missing, an error is returned. -async fn parse_body_payload( +async fn parse_body_and_auth_header( req: Request, ) -> GenericResult<(Request, T, SignedMessage)> where @@ -157,7 +158,7 @@ where } /// Parses [SignedMessage] value from X-Auth-Payload header -async fn parse_header_payload(req: Request) -> GenericResult<(Request, SignedMessage)> { +async fn parse_auth_header(req: Request) -> GenericResult<(Request, SignedMessage)> { let (parts, body) = req.into_parts(); let header_value = parts .headers diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index 269fbd7..99c4a80 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -41,7 +41,7 @@ pub(crate) async fn proxy_moralis( let original_req_uri = req.uri().clone(); - if let Err(e) = modify_request_uri(&mut req, proxy_route).await { + if let Err(e) = modify_request_uri(&mut req, proxy_route) { log::error!( "{}", log_format!( @@ -90,10 +90,7 @@ pub(crate) async fn proxy_moralis( /// Modifies the URI of an HTTP request by replacing its base URI with the outbound URI specified in `ProxyRoute`, /// while incorporating the path and query parameters from the original request URI. Additionally, this function /// removes the first path segment from the original URI. -async fn modify_request_uri( - req: &mut Request, - proxy_route: &ProxyRoute, -) -> GenericResult<()> { +fn modify_request_uri(req: &mut Request, proxy_route: &ProxyRoute) -> GenericResult<()> { let proxy_base_uri = proxy_route.outbound_route.parse::()?; let req_uri = req.uri().clone(); @@ -234,7 +231,7 @@ pub(crate) async fn validation_middleware_moralis( #[tokio::test] async fn test_parse_moralis_payload() { - use super::{parse_header_payload, X_AUTH_PAYLOAD}; + use super::{parse_auth_header, X_AUTH_PAYLOAD}; use hyper::header::HeaderName; use hyper::Method; @@ -256,7 +253,7 @@ async fn test_parse_moralis_payload() { .body(Body::empty()) .unwrap(); - let (mut req, payload) = parse_header_payload(req).await.unwrap(); + let (mut req, payload) = parse_auth_header(req).await.unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( @@ -301,7 +298,7 @@ async fn test_modify_request_uri() { rate_limiter: None, }; - modify_request_uri(&mut req, &proxy_route).await.unwrap(); + modify_request_uri(&mut req, &proxy_route).unwrap(); assert_eq!( req.uri(), diff --git a/src/proxy/quicknode.rs b/src/proxy/quicknode.rs index 2dda0f3..8945118 100644 --- a/src/proxy/quicknode.rs +++ b/src/proxy/quicknode.rs @@ -330,7 +330,7 @@ fn test_quicknode_payload_serialzation_and_deserialization() { #[tokio::test] async fn test_parse_quicknode_payload() { - use super::{parse_body_payload, X_AUTH_PAYLOAD}; + use super::{parse_body_and_auth_header, X_AUTH_PAYLOAD}; use hyper::Method; let serialized_payload = json!({ @@ -362,8 +362,9 @@ async fn test_parse_quicknode_payload() { "dummy-value".parse().unwrap(), ); - let (mut req, payload, signed_message) = - parse_body_payload::(req).await.unwrap(); + let (mut req, payload, signed_message) = parse_body_and_auth_header::(req) + .await + .unwrap(); let body_bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); assert!( From 765f58449837f57952311d86fcd82b79000191a5 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 27 Jun 2024 18:25:58 +0700 Subject: [PATCH 37/39] state unit types explicitly to adapt to new never type fallback --- src/db.rs | 2 +- src/security/address_status.rs | 4 ++-- src/security/jwt.rs | 2 +- src/security/rate_limiter.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/db.rs b/src/db.rs index 9f8736f..1146b6e 100644 --- a/src/db.rs +++ b/src/db.rs @@ -53,7 +53,7 @@ impl Db { .arg(key) .arg(seconds) .arg(value) - .query_async(&mut self.connection) + .query_async::<_, ()>(&mut self.connection) .await?; Ok(()) diff --git a/src/security/address_status.rs b/src/security/address_status.rs index 05b0d50..b9dd8c7 100644 --- a/src/security/address_status.rs +++ b/src/security/address_status.rs @@ -107,7 +107,7 @@ impl AddressStatusOperations for Db { Ok(redis::cmd("HSET") .arg(DB_STATUS_LIST) .arg(&[address, format!("{}", status as i8)]) - .query_async(&mut self.connection) + .query_async::<_, ()>(&mut self.connection) .await?) } @@ -121,7 +121,7 @@ impl AddressStatusOperations for Db { .map(|v| (v.address.clone(), v.status)) .collect(); pipe.hset_multiple(DB_STATUS_LIST, &formatted); - pipe.query_async(&mut self.connection).await?; + pipe.query_async::<_, ()>(&mut self.connection).await?; Ok(()) } diff --git a/src/security/jwt.rs b/src/security/jwt.rs index 6978267..c2622a2 100644 --- a/src/security/jwt.rs +++ b/src/security/jwt.rs @@ -102,7 +102,7 @@ pub(crate) async fn generate_jwt_and_cache_it( .arg("EX") .arg(cfg.token_expiration_time() - 60) // expire 60 seconds before token's expiration .arg("NX") - .query_async(&mut conn) + .query_async::<_, ()>(&mut conn) .await?; Ok(token) diff --git a/src/security/rate_limiter.rs b/src/security/rate_limiter.rs index 96df068..ba96f4c 100644 --- a/src/security/rate_limiter.rs +++ b/src/security/rate_limiter.rs @@ -59,7 +59,7 @@ impl RateLimitOperations for Db { self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_30_MIN, &address, 1800); self.upsert_address_rate_in_pipe(&mut pipe, DB_RP_60_MIN, &address, 3600); // Execute the pipeline once after setting all commands - pipe.query_async(&mut self.connection).await?; + pipe.query_async::<_, ()>(&mut self.connection).await?; Ok(()) } From 64224336f8de840ec6a3297974f0846b2859d065 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 28 Jun 2024 14:40:12 +0700 Subject: [PATCH 38/39] impl `get_proxy_route_extracting_uri_inbound` func and `test_get_proxy_route_by_uri_inbound` test, update modify_request_uri logic --- assets/.config_test | 28 +++++++++++ src/ctx.rs | 116 +++++++++++++++++++++++++++++++++++++++++++ src/net/http.rs | 105 +++++++++++++++++++++++++++++++-------- src/proxy/moralis.rs | 52 ++++--------------- src/security/jwt.rs | 1 + 5 files changed, 239 insertions(+), 63 deletions(-) diff --git a/assets/.config_test b/assets/.config_test index b746995..c8b140f 100644 --- a/assets/.config_test +++ b/assets/.config_test @@ -34,6 +34,34 @@ "rp_30_min": 1000, "rp_60_min": 2000 } + }, + { + "inbound_route": "/nft-test/special", + "outbound_route": "https://nft.special", + "proxy_type": "moralis", + "authorized": false, + "allowed_rpc_methods": [], + "rate_limiter": { + "rp_1_min": 60, + "rp_5_min": 200, + "rp_15_min": 700, + "rp_30_min": 1000, + "rp_60_min": 2000 + } + }, + { + "inbound_route": "/", + "outbound_route": "https://adex.io", + "proxy_type": "moralis", + "authorized": false, + "allowed_rpc_methods": [], + "rate_limiter": { + "rp_1_min": 60, + "rp_5_min": 200, + "rp_15_min": 700, + "rp_30_min": 1000, + "rp_60_min": 2000 + } } ], "rate_limiter": { diff --git a/src/ctx.rs b/src/ctx.rs index 6d5192c..6b90834 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -1,3 +1,4 @@ +use hyper::Uri; use once_cell::sync::OnceCell; use proxy::ProxyType; use serde::{Deserialize, Serialize}; @@ -90,6 +91,65 @@ impl AppConfig { route_index.map(|index| &self.proxy_routes[index]) } + + /// Finds the best matching proxy route based on the provided URI's path and updates the URI by + /// removing the matched path segments while preserving the query parameters. + pub(crate) fn get_proxy_route_extracting_uri_inbound( + &self, + uri: &mut Uri, + ) -> GenericResult> { + let path_segments: Vec<&str> = uri.path().split('/').filter(|s| !s.is_empty()).collect(); + + let mut best_match: Option<(&ProxyRoute, usize)> = None; + + for r in &self.proxy_routes { + let route_segments: Vec<&str> = r + .inbound_route + .split('/') + .filter(|s| !s.is_empty()) + .collect(); + // Count the number of segments that match between the route and the path + let matched_segments = route_segments + .iter() + .zip(&path_segments) + .take_while(|(route_seg, path_seg)| route_seg == path_seg) + .count(); + // Update best_match if this route fully matches (all its segments are matched) + // and best_match is None or has more matched segments than the current best match + if matched_segments == route_segments.len() // Ensure all segments of the route are matched + && (best_match.is_none() || matched_segments > best_match.unwrap().1) + { + best_match = Some((r, matched_segments)); + } + } + + if let Some((route, matched_segments)) = best_match { + // Construct the remaining path by skipping matched segments and accumulating the rest + let remaining_path: String = path_segments.iter().skip(matched_segments).fold( + String::new(), + |mut acc, segment| { + acc.push('/'); + acc.push_str(segment); + acc + }, + ); + + // Construct the new path and query + let new_path_and_query = match uri.query() { + Some(query) => format!("{}?{}", remaining_path, query), + None => remaining_path, + }; + + let mut parts = uri.clone().into_parts(); + parts.path_and_query = Some(new_path_and_query.parse()?); + let new_uri = Uri::from_parts(parts)?; + *uri = new_uri; + + return Ok(Some(route)); + } + + Ok(None) + } } #[cfg(test)] @@ -131,6 +191,34 @@ pub(crate) fn get_app_config_test_instance() -> AppConfig { rp_60_min: 2000, }), }, + ProxyRoute { + inbound_route: String::from("/nft-test/special"), + outbound_route: String::from("https://nft.special"), + proxy_type: ProxyType::Moralis, + authorized: false, + allowed_rpc_methods: Vec::default(), + rate_limiter: Some(RateLimiter { + rp_1_min: 60, + rp_5_min: 200, + rp_15_min: 700, + rp_30_min: 1000, + rp_60_min: 2000, + }), + }, + ProxyRoute { + inbound_route: String::from("/"), + outbound_route: String::from("https://adex.io"), + proxy_type: ProxyType::Moralis, + authorized: false, + allowed_rpc_methods: Vec::default(), + rate_limiter: Some(RateLimiter { + rp_1_min: 60, + rp_5_min: 200, + rp_15_min: 700, + rp_30_min: 1000, + rp_60_min: 2000, + }), + }, ]), rate_limiter: RateLimiter { rp_1_min: 555, @@ -180,6 +268,34 @@ fn test_app_config_serialzation_and_deserialization() { "rp_30_min": 1000, "rp_60_min": 2000 } + }, + { + "inbound_route": "/nft-test/special", + "outbound_route": "https://nft.special", + "proxy_type":"moralis", + "authorized": false, + "allowed_rpc_methods": [], + "rate_limiter": { + "rp_1_min": 60, + "rp_5_min": 200, + "rp_15_min": 700, + "rp_30_min": 1000, + "rp_60_min": 2000 + } + }, + { + "inbound_route": "/", + "outbound_route": "https://adex.io", + "proxy_type":"moralis", + "authorized": false, + "allowed_rpc_methods": [], + "rate_limiter": { + "rp_1_min": 60, + "rp_5_min": 200, + "rp_15_min": 700, + "rp_30_min": 1000, + "rp_60_min": 2000 + } } ], "rate_limiter": { diff --git a/src/net/http.rs b/src/net/http.rs index 437667d..b37d40b 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -63,7 +63,7 @@ pub(crate) async fn insert_jwt_to_http_header( pub(crate) async fn http_handler( cfg: &AppConfig, - req: Request, + mut req: Request, remote_addr: SocketAddr, ) -> GenericResult> { let req_uri = req.uri().clone(); @@ -93,27 +93,50 @@ pub(crate) async fn http_handler( return handle_preflight(); } - let inbound_route = match req.method() { - // should take the second element as path string starts with a delimiter - &Method::GET => req.uri().path().split('/').nth(1).unwrap_or("").to_string(), - _ => req.uri().path().to_string(), - }; - - // create proxy_route before payload, as we need proxy_type from it for payload generation - let proxy_route = match cfg.get_proxy_route_by_inbound(&inbound_route) { - Some(proxy_route) => proxy_route, - None => { - log::warn!( - "{}", - log_format!( - remote_addr.ip(), - String::from("-"), - req.uri(), - "Proxy route not found, returning 404." - ) - ); - return response_by_status(StatusCode::NOT_FOUND); - } + let proxy_route = match req.method() { + &Method::GET => match cfg.get_proxy_route_extracting_uri_inbound(req.uri_mut()) { + Ok(Some(route)) => route, + Ok(None) => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + String::from("-"), + req_uri, + "Proxy route not found for GET request, returning 404." + ) + ); + return response_by_status(StatusCode::NOT_FOUND); + } + Err(e) => { + log::error!( + "{}", + log_format!( + remote_addr.ip(), + String::from("-"), + req_uri, + "Error finding proxy route for GET request: {}, returning 500.", + e + ) + ); + return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); + } + }, + _ => match cfg.get_proxy_route_by_inbound(req.uri().path()) { + Some(proxy_route) => proxy_route, + None => { + log::warn!( + "{}", + log_format!( + remote_addr.ip(), + String::from("-"), + req_uri, + "Proxy route not found for non-GET request, returning 404." + ) + ); + return response_by_status(StatusCode::NOT_FOUND); + } + }, }; let (req, payload) = match generate_payload_from_req(req, &proxy_route.proxy_type).await { @@ -209,6 +232,44 @@ fn test_get_proxy_route_by_inbound() { assert_eq!(proxy_route.outbound_route, "https://nft.proxy"); } +#[test] +fn test_get_proxy_route_by_uri_inbound() { + use hyper::Uri; + use std::str::FromStr; + + let cfg = ctx::get_app_config_test_instance(); + + // test "/nft-test" inbound case + let mut url = Uri::from_str("https://komodo.proxy:5535/nft-test/nft/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); + let proxy_route = cfg + .get_proxy_route_extracting_uri_inbound(&mut url) + .unwrap() + .unwrap(); + assert_eq!(proxy_route.outbound_route, "https://nft.proxy"); + let expected = Uri::from_str("https://komodo.proxy:5535/nft/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); + assert_eq!(expected, url); + + // test "/nft-test/special" inbound case + let mut url = Uri::from_str("https://komodo.proxy:3333/nft-test/special/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); + let proxy_route = cfg + .get_proxy_route_extracting_uri_inbound(&mut url) + .unwrap() + .unwrap(); + assert_eq!(proxy_route.outbound_route, "https://nft.special"); + let expected = Uri::from_str("https://komodo.proxy:3333/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); + assert_eq!(expected, url); + + // test "/" inbound case + let mut url = Uri::from_str("https://komodo.proxy:0333/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); + let proxy_route = cfg + .get_proxy_route_extracting_uri_inbound(&mut url) + .unwrap() + .unwrap(); + assert_eq!(proxy_route.outbound_route, "https://adex.io"); + let expected = Uri::from_str("https://komodo.proxy:0333/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); + assert_eq!(expected, url); +} + #[test] fn test_respond_by_status() { let all_supported_status_codes = [ diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index 99c4a80..f491b9d 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -9,11 +9,9 @@ use crate::rate_limiter::RateLimitOperations; use crate::sign::{SignOps, SignedMessage}; use crate::{log_format, GenericResult}; use hyper::header::{HeaderName, HeaderValue}; -use hyper::http::uri::PathAndQuery; use hyper::{header, Body, Request, Response, StatusCode, Uri}; use hyper_tls::HttpsConnector; use std::net::SocketAddr; -use std::str::FromStr; pub(crate) async fn proxy_moralis( cfg: &AppConfig, @@ -41,14 +39,14 @@ pub(crate) async fn proxy_moralis( let original_req_uri = req.uri().clone(); - if let Err(e) = modify_request_uri(&mut req, proxy_route) { + if let Err(e) = modify_request_base_uri(&mut req, proxy_route) { log::error!( "{}", log_format!( remote_addr.ip(), signed_message.address, original_req_uri, - "Error modifying request Uri: {}, returning 500.", + "Error modifying request base Uri: {}, returning 500.", e ) ); @@ -87,43 +85,14 @@ pub(crate) async fn proxy_moralis( Ok(res) } -/// Modifies the URI of an HTTP request by replacing its base URI with the outbound URI specified in `ProxyRoute`, -/// while incorporating the path and query parameters from the original request URI. Additionally, this function -/// removes the first path segment from the original URI. -fn modify_request_uri(req: &mut Request, proxy_route: &ProxyRoute) -> GenericResult<()> { +/// Modifies the request URI to use the proxy base URI from `outbound_route`. +fn modify_request_base_uri(req: &mut Request, proxy_route: &ProxyRoute) -> GenericResult<()> { let proxy_base_uri = proxy_route.outbound_route.parse::()?; - - let req_uri = req.uri().clone(); - - // Remove the first path segment - let mut path_segments: Vec<&str> = req_uri - .path() - .split('/') - .filter(|s| !s.is_empty()) - .collect(); - if !path_segments.is_empty() { - path_segments.remove(0); - } - let new_path = format!("/{}", path_segments.join("/")); - - // Construct the new path and query - let path_and_query_str = match req_uri.query() { - Some(query) => format!("{}?{}", new_path, query), - None => new_path, - }; - - let path_and_query = PathAndQuery::from_str(&path_and_query_str)?; - - // Update the proxy URI with the new path and query - let mut proxy_outbound_parts = proxy_base_uri.into_parts(); - proxy_outbound_parts.path_and_query = Some(path_and_query); - - // Reconstruct the full URI with the updated parts - let new_uri = Uri::from_parts(proxy_outbound_parts)?; - - // Update the request URI + let original_uri = req.uri(); + let mut base_uri_parts = proxy_base_uri.into_parts(); + base_uri_parts.path_and_query = original_uri.path_and_query().cloned(); + let new_uri = Uri::from_parts(base_uri_parts)?; *req.uri_mut() = new_uri; - Ok(()) } @@ -283,9 +252,10 @@ async fn test_parse_moralis_payload() { #[tokio::test] async fn test_modify_request_uri() { use super::ProxyType; + use std::str::FromStr; let mut req = Request::builder() - .uri("https://komodoproxy.com/nft-proxy/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC") + .uri("https://komodoproxy.com/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC") .body(Body::empty()) .unwrap(); @@ -298,7 +268,7 @@ async fn test_modify_request_uri() { rate_limiter: None, }; - modify_request_uri(&mut req, &proxy_route).unwrap(); + modify_request_base_uri(&mut req, &proxy_route).unwrap(); assert_eq!( req.uri(), diff --git a/src/security/jwt.rs b/src/security/jwt.rs index c2622a2..3d67909 100644 --- a/src/security/jwt.rs +++ b/src/security/jwt.rs @@ -38,6 +38,7 @@ impl JwtClaims { } static AUTH_DECODING_KEY: OnceCell = OnceCell::new(); + #[allow(dead_code)] pub(crate) fn get_decoding_key(cfg: &AppConfig) -> &'static DecodingKey { let buffer_closure = || -> Vec { read_file_buffer(&cfg.pubkey_path) }; From e3d81fa408562697588947cfba18b2db14a62e86 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 2 Jul 2024 19:54:05 +0700 Subject: [PATCH 39/39] review: avoid multiple iterations in get proxy with Uri functionality, move delete inbound from Uri to modify_request_uri --- src/ctx.rs | 64 +++++------------------------------------- src/net/http.rs | 40 ++++----------------------- src/proxy/moralis.rs | 66 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 68 insertions(+), 102 deletions(-) diff --git a/src/ctx.rs b/src/ctx.rs index 6b90834..7bece5f 100644 --- a/src/ctx.rs +++ b/src/ctx.rs @@ -92,63 +92,13 @@ impl AppConfig { route_index.map(|index| &self.proxy_routes[index]) } - /// Finds the best matching proxy route based on the provided URI's path and updates the URI by - /// removing the matched path segments while preserving the query parameters. - pub(crate) fn get_proxy_route_extracting_uri_inbound( - &self, - uri: &mut Uri, - ) -> GenericResult> { - let path_segments: Vec<&str> = uri.path().split('/').filter(|s| !s.is_empty()).collect(); - - let mut best_match: Option<(&ProxyRoute, usize)> = None; - - for r in &self.proxy_routes { - let route_segments: Vec<&str> = r - .inbound_route - .split('/') - .filter(|s| !s.is_empty()) - .collect(); - // Count the number of segments that match between the route and the path - let matched_segments = route_segments - .iter() - .zip(&path_segments) - .take_while(|(route_seg, path_seg)| route_seg == path_seg) - .count(); - // Update best_match if this route fully matches (all its segments are matched) - // and best_match is None or has more matched segments than the current best match - if matched_segments == route_segments.len() // Ensure all segments of the route are matched - && (best_match.is_none() || matched_segments > best_match.unwrap().1) - { - best_match = Some((r, matched_segments)); - } - } - - if let Some((route, matched_segments)) = best_match { - // Construct the remaining path by skipping matched segments and accumulating the rest - let remaining_path: String = path_segments.iter().skip(matched_segments).fold( - String::new(), - |mut acc, segment| { - acc.push('/'); - acc.push_str(segment); - acc - }, - ); - - // Construct the new path and query - let new_path_and_query = match uri.query() { - Some(query) => format!("{}?{}", remaining_path, query), - None => remaining_path, - }; - - let mut parts = uri.clone().into_parts(); - parts.path_and_query = Some(new_path_and_query.parse()?); - let new_uri = Uri::from_parts(parts)?; - *uri = new_uri; - - return Ok(Some(route)); - } - - Ok(None) + #[inline(always)] + /// Finds the best matching proxy route based on the provided URI's. + pub(crate) fn get_proxy_route_by_uri(&self, uri: &mut Uri) -> Option<&ProxyRoute> { + self.proxy_routes + .iter() + .filter(|proxy_route| uri.path().starts_with(&proxy_route.inbound_route)) + .max_by_key(|proxy_route| proxy_route.inbound_route.len()) } } diff --git a/src/net/http.rs b/src/net/http.rs index b37d40b..41a5c31 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -94,9 +94,9 @@ pub(crate) async fn http_handler( } let proxy_route = match req.method() { - &Method::GET => match cfg.get_proxy_route_extracting_uri_inbound(req.uri_mut()) { - Ok(Some(route)) => route, - Ok(None) => { + &Method::GET => match cfg.get_proxy_route_by_uri(req.uri_mut()) { + Some(proxy_route) => proxy_route, + None => { log::warn!( "{}", log_format!( @@ -108,19 +108,6 @@ pub(crate) async fn http_handler( ); return response_by_status(StatusCode::NOT_FOUND); } - Err(e) => { - log::error!( - "{}", - log_format!( - remote_addr.ip(), - String::from("-"), - req_uri, - "Error finding proxy route for GET request: {}, returning 500.", - e - ) - ); - return response_by_status(StatusCode::INTERNAL_SERVER_ERROR); - } }, _ => match cfg.get_proxy_route_by_inbound(req.uri().path()) { Some(proxy_route) => proxy_route, @@ -241,33 +228,18 @@ fn test_get_proxy_route_by_uri_inbound() { // test "/nft-test" inbound case let mut url = Uri::from_str("https://komodo.proxy:5535/nft-test/nft/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); - let proxy_route = cfg - .get_proxy_route_extracting_uri_inbound(&mut url) - .unwrap() - .unwrap(); + let proxy_route = cfg.get_proxy_route_by_uri(&mut url).unwrap(); assert_eq!(proxy_route.outbound_route, "https://nft.proxy"); - let expected = Uri::from_str("https://komodo.proxy:5535/nft/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); - assert_eq!(expected, url); // test "/nft-test/special" inbound case let mut url = Uri::from_str("https://komodo.proxy:3333/nft-test/special/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); - let proxy_route = cfg - .get_proxy_route_extracting_uri_inbound(&mut url) - .unwrap() - .unwrap(); + let proxy_route = cfg.get_proxy_route_by_uri(&mut url).unwrap(); assert_eq!(proxy_route.outbound_route, "https://nft.special"); - let expected = Uri::from_str("https://komodo.proxy:3333/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); - assert_eq!(expected, url); // test "/" inbound case let mut url = Uri::from_str("https://komodo.proxy:0333/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); - let proxy_route = cfg - .get_proxy_route_extracting_uri_inbound(&mut url) - .unwrap() - .unwrap(); + let proxy_route = cfg.get_proxy_route_by_uri(&mut url).unwrap(); assert_eq!(proxy_route.outbound_route, "https://adex.io"); - let expected = Uri::from_str("https://komodo.proxy:0333/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC").unwrap(); - assert_eq!(expected, url); } #[test] diff --git a/src/proxy/moralis.rs b/src/proxy/moralis.rs index f491b9d..a997f05 100644 --- a/src/proxy/moralis.rs +++ b/src/proxy/moralis.rs @@ -39,7 +39,7 @@ pub(crate) async fn proxy_moralis( let original_req_uri = req.uri().clone(); - if let Err(e) = modify_request_base_uri(&mut req, proxy_route) { + if let Err(e) = modify_request_uri(&mut req, proxy_route) { log::error!( "{}", log_format!( @@ -85,12 +85,27 @@ pub(crate) async fn proxy_moralis( Ok(res) } -/// Modifies the request URI to use the proxy base URI from `outbound_route`. -fn modify_request_base_uri(req: &mut Request, proxy_route: &ProxyRoute) -> GenericResult<()> { +/// This function removes the matched inbound route from the request URI and +/// replaces request base URI with the outbound route specified in the proxy route. +fn modify_request_uri(req: &mut Request, proxy_route: &ProxyRoute) -> GenericResult<()> { let proxy_base_uri = proxy_route.outbound_route.parse::()?; let original_uri = req.uri(); + + let original_path_and_query = original_uri + .path_and_query() + .map(|pq| pq.as_str()) + .unwrap_or(""); + // Remove the "inbound_route" part from the original path and query + let remaining_path_and_query = if proxy_route.inbound_route == "/" { + original_path_and_query + } else { + original_path_and_query + .strip_prefix(&proxy_route.inbound_route) + .ok_or("Route doesn't match with the given inbound URL.")? + }; + let mut base_uri_parts = proxy_base_uri.into_parts(); - base_uri_parts.path_and_query = original_uri.path_and_query().cloned(); + base_uri_parts.path_and_query = Some(remaining_path_and_query.parse()?); let new_uri = Uri::from_parts(base_uri_parts)?; *req.uri_mut() = new_uri; Ok(()) @@ -254,24 +269,53 @@ async fn test_modify_request_uri() { use super::ProxyType; use std::str::FromStr; + const EXPECTED_URI: &str = "http://localhost:8000/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC"; + let mut req = Request::builder() - .uri("https://komodoproxy.com/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC") + .uri("https://komodo.proxy:5535/nft-test/nft/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC") .body(Body::empty()) .unwrap(); - let proxy_route = ProxyRoute { - inbound_route: String::from_str("/nft-proxy").unwrap(), + inbound_route: String::from_str("/nft-test").unwrap(), outbound_route: "http://localhost:8000".to_string(), proxy_type: ProxyType::Moralis, authorized: false, allowed_rpc_methods: vec![], rate_limiter: None, }; - - modify_request_base_uri(&mut req, &proxy_route).unwrap(); - + modify_request_uri(&mut req, &proxy_route).unwrap(); assert_eq!( req.uri(), - "http://localhost:8000/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC" + "http://localhost:8000/nft/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC" ); + + let mut req = Request::builder() + .uri("https://komodo.proxy:5535/nft-test/special/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC") + .body(Body::empty()) + .unwrap(); + let proxy_route = ProxyRoute { + inbound_route: String::from_str("/nft-test/special").unwrap(), + outbound_route: "http://localhost:8000".to_string(), + proxy_type: ProxyType::Moralis, + authorized: false, + allowed_rpc_methods: vec![], + rate_limiter: None, + }; + modify_request_uri(&mut req, &proxy_route).unwrap(); + assert_eq!(req.uri(), EXPECTED_URI); + + let mut req = Request::builder() + .uri("https://komodo.proxy:5535/api/v2.2/0x1f9090aaE28b8a3dCeaDf281B0F12828e676c326/nft/transfers?chain=eth&format=decimal&order=DESC") + .body(Body::empty()) + .unwrap(); + let proxy_route = ProxyRoute { + inbound_route: String::from_str("/").unwrap(), + outbound_route: "http://localhost:8000".to_string(), + proxy_type: ProxyType::Moralis, + authorized: false, + allowed_rpc_methods: vec![], + rate_limiter: None, + }; + modify_request_uri(&mut req, &proxy_route).unwrap(); + assert_eq!(req.uri(), EXPECTED_URI); }