feat: Require NEAR regular RPC URL additionally in configs (#137)

* feat: Require NEAR regular RPC URL additionally in configs

* improvement according comments

* improve naming

rpc-server/src/config.rs

@@ -8,6 +8,15 @@ pub struct Opts {
     #[clap(long, env = "NEAR_RPC_URL")]
     pub rpc_url: http::Uri,
 
+    // near network archival rpc url
+    #[clap(long, env = "ARCHIVAL_NEAR_RPC_URL")]
+    pub archival_rpc_url: Option<http::Uri>,
+
+    // Referer header value
+    // We want to set a custom referer to let NEAR JSON RPC nodes know that we are a read-rpc instance
+    #[clap(long, env = "REFERER_HEADER_VALUE", default_value = "read-rpc")]
+    pub referer_header_value: String,
+
     // Indexer bucket name
     #[clap(long, env = "AWS_BUCKET_NAME")]
     pub s3_bucket_name: String,
@@ -37,12 +46,12 @@ pub struct Opts {
     pub region: String,
 
     // AWS default region
-    #[clap(long, env, default_value = "8000")]
+    #[clap(long, env, default_value_t = 8000)]
     pub server_port: u16,
 
     /// Max gas burnt for contract function call
     /// Default value is 300_000_000_000_000
-    #[clap(long, env, default_value = "300000000000000")]
+    #[clap(long, env, default_value_t = 300_000_000_000_000)]
     pub max_gas_burnt: near_primitives_core::types::Gas,
 
     /// Max available memory for `block_cache` and `contract_code_cache` in gigabytes
@@ -52,14 +61,14 @@ pub struct Opts {
 
     /// Reserved memory for running the application in gigabytes
     /// By default we use 0.25 gigabyte (256MB or 268_435_456 bytes)
-    #[clap(long, env, default_value = "0.25")]
+    #[clap(long, env, default_value_t = 0.25)]
     pub reserved_memory: f64,
 
     /// Block cache size in gigabytes
     /// By default we use 0.125 gigabyte (128MB or 134_217_728 bytes)
     /// One cache_block size is ≈ 96 bytes
     /// In 128MB we can put 1_398_101 cache_blocks
-    #[clap(long, env, default_value = "0.125")]
+    #[clap(long, env, default_value_t = 0.125)]
     pub block_cache_size: f64,
 
     /// ScyllaDB preferred DataCenter
@@ -140,7 +149,7 @@ impl Opts {
 pub struct ServerContext {
     pub s3_client: near_lake_framework::s3_fetchers::LakeS3Client,
     pub db_manager: std::sync::Arc<Box<dyn database::ReaderDbManager + Sync + Send + 'static>>,
-    pub near_rpc_client: near_jsonrpc_client::JsonRpcClient,
+    pub near_rpc_client: crate::utils::JsonRpcClient,
     pub s3_bucket_name: String,
     pub genesis_config: near_chain_configs::GenesisConfig,
     pub blocks_cache:
@@ -158,7 +167,7 @@ impl ServerContext {
     pub fn new(
         s3_client: near_lake_framework::s3_fetchers::LakeS3Client,
         db_manager: impl database::ReaderDbManager + Sync + Send + 'static,
-        near_rpc_client: near_jsonrpc_client::JsonRpcClient,
+        near_rpc_client: crate::utils::JsonRpcClient,
         s3_bucket_name: String,
         genesis_config: near_chain_configs::GenesisConfig,
         blocks_cache: std::sync::Arc<

rpc-server/src/main.rs

@@ -74,9 +74,11 @@ async fn main() -> anyhow::Result<()> {
     #[cfg(not(feature = "tracing-instrumentation"))]
     init_logging(false)?;
 
-    let near_rpc_client = near_jsonrpc_client::JsonRpcClient::connect(opts.rpc_url.to_string());
+    let near_rpc_client =
+        utils::JsonRpcClient::new(opts.rpc_url.clone(), opts.archival_rpc_url.clone());
     // We want to set a custom referer to let NEAR JSON RPC nodes know that we are a read-rpc instance
-    let near_rpc_client = near_rpc_client.header(("Referer", "read-rpc"))?; // TODO: make it configurable
+    let near_rpc_client =
+        near_rpc_client.header("Referer".to_string(), opts.referer_header_value.clone())?;
 
     let final_block = get_final_cache_block(&near_rpc_client)
         .await
@@ -149,7 +151,7 @@ async fn main() -> anyhow::Result<()> {
             s3_config,
         )),
         db_manager,
-        near_rpc_client.clone(),
+        near_rpc_client,
         opts.s3_bucket_name.clone(),
         genesis_config,
         std::sync::Arc::clone(&blocks_cache),

rpc-server/src/modules/blocks/methods.rs

@@ -6,8 +6,6 @@ use crate::modules::blocks::utils::{
 #[cfg(feature = "shadow_data_consistency")]
 use crate::utils::shadow_compare_results;
 use jsonrpc_v2::{Data, Params};
-
-use crate::utils::proxy_rpc_call;
 use near_primitives::trie_key::TrieKey;
 use near_primitives::views::StateChangeValueView;
 
@@ -27,7 +25,7 @@ pub async fn block(
             // genesis sync_checkpoint or earliest_available sync_checkpoint
             // and proxy to near-rpc
             crate::metrics::SYNC_CHECKPOINT_REQUESTS_TOTAL.inc();
-            let block_view = proxy_rpc_call(&data.near_rpc_client, params).await?;
+            let block_view = data.near_rpc_client.call(params).await?;
             Ok(near_jsonrpc_primitives::types::blocks::RpcBlockResponse { block_view })
         }
         near_primitives::types::BlockReference::Finality(finality) => {
@@ -36,7 +34,7 @@ pub async fn block(
                 // genesis sync_checkpoint or earliest_available sync_checkpoint
                 // and proxy to near-rpc
                 crate::metrics::OPTIMISTIC_REQUESTS_TOTAL.inc();
-                let block_view = proxy_rpc_call(&data.near_rpc_client, params).await?;
+                let block_view = data.near_rpc_client.call(params).await?;
                 Ok(near_jsonrpc_primitives::types::blocks::RpcBlockResponse { block_view })
             } else {
                 block_call(data, Params(params)).await
@@ -100,15 +98,15 @@ pub async fn changes_in_block_by_type(
             // genesis sync_checkpoint or earliest_available sync_checkpoint
             // and proxy to near-rpc
             crate::metrics::SYNC_CHECKPOINT_REQUESTS_TOTAL.inc();
-            Ok(proxy_rpc_call(&data.near_rpc_client, params).await?)
+            Ok(data.near_rpc_client.call(params).await?)
         }
         near_primitives::types::BlockReference::Finality(finality) => {
             if finality != &near_primitives::types::Finality::Final {
                 // Increase the OPTIMISTIC_REQUESTS_TOTAL metric if the request has
                 // optimistic finality or doom_slug finality
                 // and proxy to near-rpc
                 crate::metrics::OPTIMISTIC_REQUESTS_TOTAL.inc();
-                Ok(proxy_rpc_call(&data.near_rpc_client, params).await?)
+                Ok(data.near_rpc_client.call(params).await?)
             } else {
                 changes_in_block_by_type_call(data, Params(params)).await
             }
@@ -138,15 +136,15 @@ pub async fn changes_in_block(
             // genesis sync_checkpoint or earliest_available sync_checkpoint
             // and proxy to near-rpc
             crate::metrics::SYNC_CHECKPOINT_REQUESTS_TOTAL.inc();
-            Ok(proxy_rpc_call(&data.near_rpc_client, params).await?)
+            Ok(data.near_rpc_client.call(params).await?)
         }
         near_primitives::types::BlockReference::Finality(finality) => {
             if finality != &near_primitives::types::Finality::Final {
                 // Increase the OPTIMISTIC_REQUESTS_TOTAL metric if the request has
                 // optimistic finality or doom_slug finality
                 // and proxy to near-rpc
                 crate::metrics::OPTIMISTIC_REQUESTS_TOTAL.inc();
-                Ok(proxy_rpc_call(&data.near_rpc_client, params).await?)
+                Ok(data.near_rpc_client.call(params).await?)
             } else {
                 changes_in_block_call(data, Params(params)).await
             }

rpc-server/src/modules/clients/methods.rs

@@ -1,6 +1,5 @@
 use crate::config::ServerContext;
 use crate::errors::RPCError;
-use crate::utils::proxy_rpc_call;
 use jsonrpc_v2::{Data, Params};
 
 pub async fn light_client_proof(
@@ -12,7 +11,7 @@ pub async fn light_client_proof(
     near_jsonrpc_primitives::types::light_client::RpcLightClientExecutionProofResponse,
     RPCError,
 > {
-    Ok(proxy_rpc_call(&data.near_rpc_client, params).await?)
+    Ok(data.near_rpc_client.call(params).await?)
 }
 
 pub async fn next_light_client_block(
@@ -22,7 +21,7 @@ pub async fn next_light_client_block(
     >,
 ) -> Result<near_jsonrpc_primitives::types::light_client::RpcLightClientNextBlockResponse, RPCError>
 {
-    match proxy_rpc_call(&data.near_rpc_client, params).await? {
+    match data.near_rpc_client.call(params).await? {
         Some(light_client_block) => Ok(
             near_jsonrpc_primitives::types::light_client::RpcLightClientNextBlockResponse {
                 light_client_block: Some(std::sync::Arc::new(light_client_block)),

rpc-server/src/modules/gas/methods.rs

@@ -1,12 +1,11 @@
 use crate::config::ServerContext;
 use crate::errors::RPCError;
-use crate::utils::proxy_rpc_call;
 use jsonrpc_v2::{Data, Params};
 
 pub async fn gas_price(
     data: Data<ServerContext>,
     Params(params): Params<near_jsonrpc_primitives::types::gas_price::RpcGasPriceRequest>,
 ) -> Result<near_jsonrpc_primitives::types::gas_price::RpcGasPriceResponse, RPCError> {
-    let gas_price_view = proxy_rpc_call(&data.near_rpc_client, params).await?;
+    let gas_price_view = data.near_rpc_client.call(params).await?;
     Ok(near_jsonrpc_primitives::types::gas_price::RpcGasPriceResponse { gas_price_view })
 }

rpc-server/src/modules/network/methods.rs

@@ -3,7 +3,6 @@ use crate::errors::RPCError;
 use crate::modules::network::{
     friendly_memory_size_format, parse_validator_request, StatusResponse,
 };
-use crate::utils::proxy_rpc_call;
 use jsonrpc_v2::{Data, Params};
 use serde_json::Value;
 use sysinfo::{System, SystemExt};
@@ -66,7 +65,7 @@ pub async fn validators(
 ) -> Result<near_jsonrpc_primitives::types::validator::RpcValidatorResponse, RPCError> {
     match parse_validator_request(params).await {
         Ok(request) => {
-            let validator_info = proxy_rpc_call(&data.near_rpc_client, request).await?;
+            let validator_info = data.near_rpc_client.call(request).await?;
             Ok(near_jsonrpc_primitives::types::validator::RpcValidatorResponse { validator_info })
         }
         Err(err) => Err(RPCError::parse_error(&err.to_string())),
@@ -77,7 +76,7 @@ pub async fn validators_ordered(
     data: Data<ServerContext>,
     Params(params): Params<near_jsonrpc_primitives::types::validator::RpcValidatorsOrderedRequest>,
 ) -> Result<near_jsonrpc_primitives::types::validator::RpcValidatorsOrderedResponse, RPCError> {
-    Ok(proxy_rpc_call(&data.near_rpc_client, params).await?)
+    Ok(data.near_rpc_client.call(params).await?)
 }
 
 pub async fn genesis_config(
@@ -91,6 +90,6 @@ pub async fn protocol_config(
     data: Data<ServerContext>,
     Params(params): Params<near_jsonrpc_primitives::types::config::RpcProtocolConfigRequest>,
 ) -> Result<near_jsonrpc_primitives::types::config::RpcProtocolConfigResponse, RPCError> {
-    let config_view = proxy_rpc_call(&data.near_rpc_client, params).await?;
+    let config_view = data.near_rpc_client.call(params).await?;
     Ok(near_jsonrpc_primitives::types::config::RpcProtocolConfigResponse { config_view })
 }

rpc-server/src/modules/queries/methods.rs

@@ -5,7 +5,6 @@ use crate::modules::blocks::CacheBlock;
 #[cfg(feature = "account_access_keys")]
 use crate::modules::queries::utils::fetch_list_access_keys_from_db;
 use crate::modules::queries::utils::{fetch_state_from_db, run_contract};
-use crate::utils::proxy_rpc_call;
 #[cfg(feature = "shadow_data_consistency")]
 use crate::utils::shadow_compare_results;
 use jsonrpc_v2::{Data, Params};
@@ -26,15 +25,15 @@ pub async fn query(
             // genesis sync_checkpoint or earliest_available sync_checkpoint
             // and proxy to near-rpc
             crate::metrics::SYNC_CHECKPOINT_REQUESTS_TOTAL.inc();
-            Ok(proxy_rpc_call(&data.near_rpc_client, params).await?)
+            Ok(data.near_rpc_client.call(params).await?)
         }
         near_primitives::types::BlockReference::Finality(finality) => {
             if finality != &near_primitives::types::Finality::Final {
                 // Increase the OPTIMISTIC_REQUESTS_TOTAL metric if the request has
                 // optimistic finality or doom_slug finality
                 // and proxy to near-rpc
                 crate::metrics::OPTIMISTIC_REQUESTS_TOTAL.inc();
-                Ok(proxy_rpc_call(&data.near_rpc_client, params).await?)
+                Ok(data.near_rpc_client.call(params).await?)
             } else {
                 query_call(data, Params(params)).await
             }
@@ -76,10 +75,18 @@ async fn query_call(
         near_primitives::views::QueryRequest::ViewState {
             account_id,
             prefix,
-            include_proof: _,
+            include_proof,
         } => {
             crate::metrics::QUERY_VIEW_STATE_REQUESTS_TOTAL.inc();
-            view_state(&data, block, &account_id, prefix.as_ref()).await
+            if include_proof {
+                // TODO: We can calculate the proof for state only on regular or archival nodes.
+                // After indexing the epochs and validators,
+                // we will be able to separate proxies queries to regular and archival nodes.
+                // For now we will proxy all requests with `include_proof` to archival nodes.
+                return Ok(data.near_rpc_client.archival_call(params).await?);
+            } else {
+                view_state(&data, block, &account_id, prefix.as_ref()).await
+            }
         }
         near_primitives::views::QueryRequest::CallFunction {
             account_id,
@@ -94,7 +101,7 @@ async fn query_call(
         near_primitives::views::QueryRequest::ViewAccessKeyList { account_id } => {
             crate::metrics::QUERY_VIEW_ACCESS_KEYS_LIST_REQUESTS_TOTAL.inc();
             #[cfg(not(feature = "account_access_keys"))]
-            return Ok(crate::utils::proxy_rpc_call(&data.near_rpc_client, params).await?);
+            return Ok(data.near_rpc_client.call(params).await?);
             #[cfg(feature = "account_access_keys")]
             {
                 view_access_keys_list(&data, block, &account_id).await

rpc-server/src/modules/transactions/methods.rs

@@ -3,7 +3,6 @@ use crate::errors::RPCError;
 use crate::modules::transactions::{
     parse_signed_transaction, parse_transaction_status_common_request,
 };
-use crate::utils::proxy_rpc_call;
 #[cfg(feature = "shadow_data_consistency")]
 use crate::utils::shadow_compare_results;
 use jsonrpc_v2::{Data, Params};
@@ -128,7 +127,7 @@ pub async fn send_tx_async(
             near_jsonrpc_client::methods::broadcast_tx_async::RpcBroadcastTxAsyncRequest {
                 signed_transaction,
             };
-        match proxy_rpc_call(&data.near_rpc_client, proxy_params).await {
+        match data.near_rpc_client.call(proxy_params).await {
             Ok(resp) => Ok(resp),
             Err(err) => Err(RPCError::internal_error(&err.to_string())),
         }
@@ -154,7 +153,7 @@ pub async fn send_tx_commit(
             near_jsonrpc_client::methods::broadcast_tx_commit::RpcBroadcastTxCommitRequest {
                 signed_transaction,
             };
-        match proxy_rpc_call(&data.near_rpc_client, proxy_params).await {
+        match data.near_rpc_client.call(proxy_params).await {
             Ok(resp) => Ok(
                 near_jsonrpc_primitives::types::transactions::RpcTransactionResponse {
                     final_execution_outcome: FinalExecutionOutcome(resp),