Implement TOML Configuration for All Services (#152)

* add configuration

* add state indexer configuration

* parse config improvement

* example config improvement

* add configuration for state indexer

* add configuration for epoch indexer

* add configuration to tx-indexer

* fmt

* add configuration for rpc-server

* config improvement

* add readme for configuration

* fix docker build

* parse config improvement

* add rightsizing for tx indexer

* move init tracing into the configuration

* improvement according pr comments

* parse config inprovement
add config.toml around env variables
fix docker build

* small improvement epoch indexer config

Cargo.toml

@@ -3,6 +3,7 @@
 resolver = "2"
 
 members = [
+    "configuration",
     "database",
     "epoch-indexer",
     "perf-testing",

README.md

@@ -20,20 +20,18 @@ The indexer built on top of Lake Framework that watches the network and stores t
 
 The indexer built on top of Lake Framework that watches the network and stores the `Transactions` along with all the related entities (`Receipts`, `ExecutionOutcomes`) into the Storage (ScyllaDB) using the specifically defined `TransactionDetails` structure in a dumped way (using the simplest key-value schema)
 
+### [config](configuration/README.md)
+
+The configuration module is responsible for managing the configuration settings of the NEAR ReadRPC project.
 
 ## Docker compose
 
 **Note!** The docker compose is not fully ready yet. It's still in progress. However, you can run the entire project to play around with it. It is still not convenient for development or debugging purposes. We are working on improving it.
 
 ### Run the entire project
 
-Add the `.env` file (update the credentials with your own to access the S3 bucket)
-
-```
-AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
-AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
-AWS_DEFAULT_REGION=eu-central-1
-```
+Put TOML file `config.toml` with configuration in the home root of the project.
+See the example [here](./configuration/example.config.toml).
 
 Run the docker compose:
 

config.toml

@@ -0,0 +1,61 @@
+# This file to present all confiuration around th environment variables
+# Not present environment variables will be set to default values
+# See more details and information about each parameter in configuration/README.md and configuration/example.config.toml
+[general]
+chain_id = "${CHAIN_ID}"
+near_rpc_url = "${NEAR_RPC_URL}"
+near_archival_rpc_url = "${ARCHIVAL_NEAR_RPC_URL}"
+
+[general.rpc_server]
+referer_header_value = "${REFERER_HEADER_VALUE}"
+server_port = "${SERVER_PORT}"
+max_gas_burnt = "${MAX_GAS_BURNT}"
+limit_memory_cache = "${LIMIT_MEMORY_CACHE}"
+reserved_memory = "${RESERVED_MEMORY}"
+block_cache_size = "${BLOCK_CACHE_SIZE}"
+
+[general.tx_indexer]
+indexer_id = "${TX_INDEXER_ID}"
+metrics_server_port = "${TX_SERVER_PORT}"
+cache_restore_blocks_range = "${CACHE_RESTORE_BLOCKS_RANGE}"
+
+[general.state_indexer]
+indexer_id = "${STATE_INDEXER_ID}"
+metrics_server_port = "${STATE_SERVER_PORT}"
+concurrency = "${CONCURRENCY}"
+
+[general.epoch_indexer]
+indexer_id = "${EPOCH_INDEXER_ID}"
+
+[rightsizing]
+tracked_accounts = "${TRACKED_ACCOUNTS}"
+tracked_changes = "${TRACKED_CHANGES}"
+
+[lake_config]
+aws_access_key_id = "${AWS_ACCESS_KEY_ID}"
+aws_secret_access_key = "${AWS_SECRET_ACCESS_KEY}"
+aws_default_region = "${AWS_DEFAULT_REGION}"
+aws_bucket_name = "${AWS_BUCKET_NAME}"
+
+[database]
+database_url = "${DATABASE_URL}"
+database_user = "${DATABASE_USER}"
+database_password = "${DATABASE_PASSWORD}"
+database_name = "${DATABASE_NAME}"
+
+[database.rpc_server]
+preferred_dc = "${RPC_PREFERRED_DC}"
+max_retry = "${RPC_MAX_RETRY}"
+strict_mode = "${RPC_STRICT_MODE}"
+keepalive_interval = "${KEEPALIVE_INTERVAL}"
+
+[database.tx_indexer]
+preferred_dc = "${TX_PREFERRED_DC}"
+max_retry = "${TX_MAX_RETRY}"
+strict_mode = "${TX_STRICT_MODE}"
+max_db_parallel_queries = "${MAX_DB_PARALLEL_QUERIES}"
+
+[database.state_indexer]
+preferred_dc = "${STATE_PREFERRED_DC}"
+max_retry = "${STATE_MAX_RETRY}"
+strict_mode = "${STATE_STRICT_MODE}"

configuration/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "configuration"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1.0.70"
+aws-credential-types = "1.0.0"
+aws-sdk-s3 = { version = "0.39.1", features = ["behavior-version-latest"] }
+aws-types = "1.0.0"
+dotenv = "0.15.0"
+lazy_static = "1.4.0"
+regex = "1.10.2"
+serde = "1.0.145"
+serde_derive = "1.0.145"
+serde_json = "1.0.108"
+opentelemetry = { version = "0.19", features = ["rt-tokio-current-thread"] }
+opentelemetry-jaeger = { version = "0.18", features = [
+    "rt-tokio-current-thread",
+    "collector_client",
+    "isahc_collector_client",
+], optional = true }
+toml = "0.8.4"
+tracing = "0.1.34"
+tracing-subscriber = { version = "0.3.15", features = [
+    "fmt",
+    "env-filter",
+    "std",
+    "json",
+] }
+tracing-opentelemetry = { version = "0.19", optional = true }
+tracing-stackdriver = "0.7.2" # GCP logs
+
+near-lake-framework = "0.7.6"
+
+[features]
+tracing-instrumentation = ["dep:opentelemetry-jaeger", "dep:tracing-opentelemetry"]

configuration/README.md

@@ -0,0 +1,41 @@
+# Configuration
+
+The configuration module is responsible for managing the configuration settings of the NEAR ReadRPC project.
+It reads and parses the configuration from a TOML file and provides these settings to other parts of the application.
+
+## Environment Variables
+
+The configuration settings can also be provided through environment variables. 
+This is useful when you want to change the settings without modifying the TOML file, 
+such as when deploying the application in different environments. 
+The environment variables are specified in the TOML file using the syntax `${VARIABLE_NAME}`.
+For example, `${DATABASE_URL}` specifies the `DATABASE_URL` environment variable.
+
+## Files
+
+- `example.config.toml`: This file contains an example configuration for the NEAR ReadRPC. 
+It includes settings for the general configuration, 
+RPC server, transaction indexer, state indexer, epoch indexer, rightsizing, lake framework, and database.
+
+
+## Configuration
+
+The configuration settings are stored in a TOML file. The settings include:
+
+- General settings like the chain ID and NEAR network RPC URL.
+- RPC server settings like the server port and max gas burnt for contract function call.
+- Transaction indexer settings like the indexer ID and port for the metrics server.
+- State indexer settings like the indexer ID and port for the metrics server.
+- Epoch indexer settings like the indexer ID.
+- Rightsizing settings like the accounts and state changes to track.
+- Lake framework settings like the AWS access key ID and secret access key.
+- Database settings like the database connection string and user credentials.
+
+## Usage
+
+Put TOML file `config.toml` with configuration in the home root of the project.
+
+## Note
+
+Please ensure that the configuration file is correctly formatted and all the required settings are provided. 
+Incorrect or missing settings may cause the application to behave unexpectedly or fail.