diff --git a/Cargo.lock b/Cargo.lock index 3242d60d..987f4edb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,7 +90,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -203,7 +203,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -315,43 +315,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "arbitrary" @@ -420,7 +420,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -431,7 +431,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -1241,9 +1241,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.10" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive 4.4.7", @@ -1251,9 +1251,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.9" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -1283,7 +1283,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -1325,6 +1325,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "configuration" +version = "0.1.0" +dependencies = [ + "anyhow", + "aws-credential-types", + "aws-sdk-s3", + "aws-types", + "dotenv", + "lazy_static", + "near-lake-framework", + "opentelemetry", + "opentelemetry-jaeger", + "regex", + "serde", + "serde_derive", + "serde_json", + "toml 0.8.8", + "tracing", + "tracing-opentelemetry", + "tracing-stackdriver", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1516,9 +1540,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -1526,9 +1550,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", @@ -1537,22 +1561,21 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" dependencies = [ "autocfg", "cfg-if 1.0.0", "crossbeam-utils", "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" dependencies = [ "cfg-if 1.0.0", ] @@ -1643,9 +1666,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.68+curl-8.4.0" +version = "0.4.70+curl-8.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a0d18d88360e374b16b2273c832b5e57258ffc1d4aa4f96b108e0738d5752f" +checksum = "3c0333d8849afe78a4c8102a429a446bfdd055832af071945520e835ae2d841e" dependencies = [ "cc", "libc", @@ -1714,7 +1737,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -1736,7 +1759,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -1761,6 +1784,7 @@ dependencies = [ "bigdecimal 0.4.2", "borsh 0.10.3", "bytes", + "configuration", "diesel", "diesel-async", "futures", @@ -1837,7 +1861,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -1926,7 +1950,7 @@ dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -1935,7 +1959,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -2152,7 +2176,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -2173,7 +2197,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -2181,23 +2205,17 @@ name = "epoch-indexer" version = "0.1.0" dependencies = [ "anyhow", - "aws-credential-types", - "aws-sdk-s3", - "aws-types", "clap 3.2.25", + "configuration", "database", - "dotenv", "futures", "near-chain-configs", "near-indexer-primitives", "near-jsonrpc-client", "near-lake-framework", - "opentelemetry 0.19.0", "readnode-primitives", "tokio", "tracing", - "tracing-opentelemetry 0.19.0", - "tracing-subscriber", ] [[package]] @@ -2461,7 +2479,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -2708,9 +2726,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -2737,9 +2755,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -2752,7 +2770,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -2888,7 +2906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.3", - "rustix 0.38.26", + "rustix 0.38.28", "windows-sys 0.48.0", ] @@ -2937,9 +2955,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" @@ -3013,9 +3031,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libm" @@ -3187,7 +3205,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "rustix 0.38.26", + "rustix 0.38.28", ] [[package]] @@ -3253,9 +3271,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "log", @@ -3500,7 +3518,7 @@ source = "git+https://github.com/near/nearcore.git?branch=1.36.0#b0c5d14557b0e24 dependencies = [ "quote", "serde", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -3511,7 +3529,7 @@ dependencies = [ "fs2", "near-rpc-error-core", "serde", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -3775,7 +3793,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -3801,9 +3819,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" @@ -3813,9 +3831,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.60" +version = "0.10.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" +checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" dependencies = [ "bitflags 2.4.1", "cfg-if 1.0.0", @@ -3834,7 +3852,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -3845,9 +3863,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.96" +version = "0.9.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" dependencies = [ "cc", "libc", @@ -3855,27 +3873,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "opentelemetry" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" -dependencies = [ - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "js-sys", - "lazy_static", - "percent-encoding", - "pin-project", - "rand 0.8.5", - "thiserror", - "tokio", - "tokio-stream", -] - [[package]] name = "opentelemetry" version = "0.19.0" @@ -3899,21 +3896,6 @@ dependencies = [ "opentelemetry_api", ] -[[package]] -name = "opentelemetry-jaeger" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c0b12cd9e3f9b35b52f6e0dac66866c519b26f424f4bbf96e3fe8bfbdc5229" -dependencies = [ - "async-trait", - "lazy_static", - "opentelemetry 0.17.0", - "opentelemetry-semantic-conventions 0.9.0", - "thiserror", - "thrift 0.15.0", - "tokio", -] - [[package]] name = "opentelemetry-jaeger" version = "0.18.0" @@ -3926,30 +3908,21 @@ dependencies = [ "http", "isahc", "once_cell", - "opentelemetry 0.19.0", + "opentelemetry", "opentelemetry-http", - "opentelemetry-semantic-conventions 0.11.0", + "opentelemetry-semantic-conventions", "thiserror", - "thrift 0.17.0", + "thrift", "tokio", ] -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" -dependencies = [ - "opentelemetry 0.17.0", -] - [[package]] name = "opentelemetry-semantic-conventions" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24e33428e6bf08c6f7fcea4ddb8e358fab0fe48ab877a87c70c6ebe20f673ce5" dependencies = [ - "opentelemetry 0.19.0", + "opentelemetry", ] [[package]] @@ -3990,15 +3963,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "ordered-float" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" -dependencies = [ - "num-traits", -] - [[package]] name = "ordered-float" version = "2.10.1" @@ -4130,7 +4094,7 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", - "clap 4.4.10", + "clap 4.4.18", "dotenv", "futures", "http", @@ -4178,7 +4142,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -4311,7 +4275,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" dependencies = [ - "toml", + "toml 0.5.11", ] [[package]] @@ -4321,7 +4285,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit", + "toml_edit 0.19.15", ] [[package]] @@ -4350,9 +4314,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708" dependencies = [ "unicode-ident", ] @@ -4420,9 +4384,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -4541,18 +4505,13 @@ dependencies = [ "actix-web", "anyhow", "assert-json-diff", - "aws-credential-types", - "aws-sdk-s3", - "aws-types", "borsh 0.10.3", - "clap 3.2.25", + "configuration", "database", - "dotenv", "erased-serde", "futures", "futures-locks", "hex", - "http", "jsonrpc-v2", "lazy_static", "lru 0.11.1", @@ -4564,8 +4523,6 @@ dependencies = [ "near-lake-framework", "near-primitives", "near-vm-runner", - "opentelemetry 0.17.0", - "opentelemetry-jaeger 0.16.0", "paste", "prometheus", "readnode-primitives", @@ -4577,9 +4534,6 @@ dependencies = [ "tokio-stream", "tracing", "tracing-actix-web", - "tracing-opentelemetry 0.17.4", - "tracing-stackdriver", - "tracing-subscriber", ] [[package]] @@ -4717,9 +4671,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64", "bytes", @@ -4772,9 +4726,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.6" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom 0.2.11", @@ -4795,12 +4749,13 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.42" +version = "0.7.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +checksum = "527a97cdfef66f65998b5f3b637c26f5a5ec09cc52a3f9932313ac645f4190f5" dependencies = [ "bitvec", "bytecheck", + "bytes", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -4812,9 +4767,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.42" +version = "0.7.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +checksum = "b5c462a1328c8e67e4d6dbad1eb0355dd43e8ab432c6e227a43657f16ade5033" dependencies = [ "proc-macro2", "quote", @@ -4873,9 +4828,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.26" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno 0.3.8", @@ -4935,9 +4890,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" @@ -5031,13 +4986,14 @@ dependencies = [ [[package]] name = "scylla-macros" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5757ded3dfb10967ca7d1ff1084d072d565b5e10b2b21c286d5335c245425a7e" +checksum = "71da9865153917f9ae6e9e2ba2b3dc52edd3ef7487a4efbe93cbcb92841c1cdd" dependencies = [ + "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -5159,7 +5115,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -5181,7 +5137,16 @@ checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", +] + +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", ] [[package]] @@ -5222,7 +5187,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -5368,9 +5333,9 @@ dependencies = [ [[package]] name = "snap" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" @@ -5426,12 +5391,10 @@ version = "0.1.0" dependencies = [ "actix-web", "anyhow", - "aws-sdk-s3", - "aws-types", "borsh 0.10.3", "clap 3.2.25", + "configuration", "database", - "dotenv", "epoch-indexer", "futures", "hex", @@ -5442,15 +5405,10 @@ dependencies = [ "near-lake-framework", "near-primitives", "openssl-probe", - "opentelemetry 0.19.0", - "opentelemetry-jaeger 0.18.0", "prometheus", "tokio", "tokio-stream", "tracing", - "tracing-opentelemetry 0.19.0", - "tracing-stackdriver", - "tracing-subscriber", ] [[package]] @@ -5536,9 +5494,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -5608,7 +5566,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand 2.0.1", "redox_syscall 0.4.1", - "rustix 0.38.26", + "rustix 0.38.28", "windows-sys 0.48.0", ] @@ -5640,22 +5598,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -5677,19 +5635,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "thrift" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82ca8f46f95b3ce96081fe3dd89160fdea970c254bb72925255d1b62aae692e" -dependencies = [ - "byteorder", - "integer-encoding", - "log", - "ordered-float 1.1.1", - "threadpool", -] - [[package]] name = "thrift" version = "0.17.0" @@ -5699,15 +5644,15 @@ dependencies = [ "byteorder", "integer-encoding", "log", - "ordered-float 2.10.1", + "ordered-float", "threadpool", ] [[package]] name = "time" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", @@ -5725,9 +5670,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -5749,9 +5694,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -5775,7 +5720,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -5858,11 +5803,26 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.21.0", +] + [[package]] name = "toml_datetime" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -5875,6 +5835,19 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap 2.1.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -5913,7 +5886,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -5958,20 +5931,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-opentelemetry" -version = "0.17.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" -dependencies = [ - "once_cell", - "opentelemetry 0.17.0", - "tracing", - "tracing-core", - "tracing-log 0.1.4", - "tracing-subscriber", -] - [[package]] name = "tracing-opentelemetry" version = "0.19.0" @@ -5979,7 +5938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00a39dcf9bfc1742fa4d6215253b33a6e474be78275884c216fc2a06267b3600" dependencies = [ "once_cell", - "opentelemetry 0.19.0", + "opentelemetry", "tracing", "tracing-core", "tracing-log 0.1.4", @@ -6034,9 +5993,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" @@ -6055,12 +6014,9 @@ dependencies = [ "actix-web", "anyhow", "async-trait", - "aws-credential-types", - "aws-sdk-s3", - "aws-types", "clap 3.2.25", + "configuration", "database", - "dotenv", "futures", "futures-locks", "humantime", @@ -6068,16 +6024,11 @@ dependencies = [ "near-indexer-primitives", "near-jsonrpc-client", "near-lake-framework", - "opentelemetry 0.19.0", - "opentelemetry-jaeger 0.18.0", "prometheus", "readnode-primitives", "tokio", "tokio-stream", "tracing", - "tracing-opentelemetry 0.19.0", - "tracing-stackdriver", - "tracing-subscriber", ] [[package]] @@ -6100,9 +6051,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" @@ -6133,9 +6084,9 @@ checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" [[package]] name = "untrusted" @@ -6253,7 +6204,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -6287,7 +6238,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6896,9 +6847,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.19" +version = "0.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" dependencies = [ "memchr", ] @@ -6930,22 +6881,22 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "zerocopy" -version = "0.7.28" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" +checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.28" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" +checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] @@ -6965,7 +6916,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6246dc38..12df50b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ + "configuration", "database", "epoch-indexer", "perf-testing", diff --git a/README.md b/README.md index b46135eb..5b89ff87 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ The indexer built on top of Lake Framework that watches the network and stores t The indexer built on top of Lake Framework that watches the network and stores the `Transactions` along with all the related entities (`Receipts`, `ExecutionOutcomes`) into the Storage (ScyllaDB) using the specifically defined `TransactionDetails` structure in a dumped way (using the simplest key-value schema) +### [config](configuration/README.md) + +The configuration module is responsible for managing the configuration settings of the NEAR ReadRPC project. ## Docker compose @@ -27,13 +30,8 @@ The indexer built on top of Lake Framework that watches the network and stores t ### Run the entire project -Add the `.env` file (update the credentials with your own to access the S3 bucket) - -``` -AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE -AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY -AWS_DEFAULT_REGION=eu-central-1 -``` +Put TOML file `config.toml` with configuration in the home root of the project. +See the example [here](./configuration/example.config.toml). Run the docker compose: diff --git a/config.toml b/config.toml new file mode 100644 index 00000000..cb28f292 --- /dev/null +++ b/config.toml @@ -0,0 +1,61 @@ +# This file to present all confiuration around th environment variables +# Not present environment variables will be set to default values +# See more details and information about each parameter in configuration/README.md and configuration/example.config.toml +[general] +chain_id = "${CHAIN_ID}" +near_rpc_url = "${NEAR_RPC_URL}" +near_archival_rpc_url = "${ARCHIVAL_NEAR_RPC_URL}" + +[general.rpc_server] +referer_header_value = "${REFERER_HEADER_VALUE}" +server_port = "${SERVER_PORT}" +max_gas_burnt = "${MAX_GAS_BURNT}" +limit_memory_cache = "${LIMIT_MEMORY_CACHE}" +reserved_memory = "${RESERVED_MEMORY}" +block_cache_size = "${BLOCK_CACHE_SIZE}" + +[general.tx_indexer] +indexer_id = "${TX_INDEXER_ID}" +metrics_server_port = "${TX_SERVER_PORT}" +cache_restore_blocks_range = "${CACHE_RESTORE_BLOCKS_RANGE}" + +[general.state_indexer] +indexer_id = "${STATE_INDEXER_ID}" +metrics_server_port = "${STATE_SERVER_PORT}" +concurrency = "${CONCURRENCY}" + +[general.epoch_indexer] +indexer_id = "${EPOCH_INDEXER_ID}" + +[rightsizing] +tracked_accounts = "${TRACKED_ACCOUNTS}" +tracked_changes = "${TRACKED_CHANGES}" + +[lake_config] +aws_access_key_id = "${AWS_ACCESS_KEY_ID}" +aws_secret_access_key = "${AWS_SECRET_ACCESS_KEY}" +aws_default_region = "${AWS_DEFAULT_REGION}" +aws_bucket_name = "${AWS_BUCKET_NAME}" + +[database] +database_url = "${DATABASE_URL}" +database_user = "${DATABASE_USER}" +database_password = "${DATABASE_PASSWORD}" +database_name = "${DATABASE_NAME}" + +[database.rpc_server] +preferred_dc = "${RPC_PREFERRED_DC}" +max_retry = "${RPC_MAX_RETRY}" +strict_mode = "${RPC_STRICT_MODE}" +keepalive_interval = "${KEEPALIVE_INTERVAL}" + +[database.tx_indexer] +preferred_dc = "${TX_PREFERRED_DC}" +max_retry = "${TX_MAX_RETRY}" +strict_mode = "${TX_STRICT_MODE}" +max_db_parallel_queries = "${MAX_DB_PARALLEL_QUERIES}" + +[database.state_indexer] +preferred_dc = "${STATE_PREFERRED_DC}" +max_retry = "${STATE_MAX_RETRY}" +strict_mode = "${STATE_STRICT_MODE}" diff --git a/configuration/Cargo.toml b/configuration/Cargo.toml new file mode 100644 index 00000000..323e63b0 --- /dev/null +++ b/configuration/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "configuration" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.70" +aws-credential-types = "1.0.0" +aws-sdk-s3 = { version = "0.39.1", features = ["behavior-version-latest"] } +aws-types = "1.0.0" +dotenv = "0.15.0" +lazy_static = "1.4.0" +regex = "1.10.2" +serde = "1.0.145" +serde_derive = "1.0.145" +serde_json = "1.0.108" +opentelemetry = { version = "0.19", features = ["rt-tokio-current-thread"] } +opentelemetry-jaeger = { version = "0.18", features = [ + "rt-tokio-current-thread", + "collector_client", + "isahc_collector_client", +], optional = true } +toml = "0.8.4" +tracing = "0.1.34" +tracing-subscriber = { version = "0.3.15", features = [ + "fmt", + "env-filter", + "std", + "json", +] } +tracing-opentelemetry = { version = "0.19", optional = true } +tracing-stackdriver = "0.7.2" # GCP logs + +near-lake-framework = "0.7.6" + +[features] +tracing-instrumentation = ["dep:opentelemetry-jaeger", "dep:tracing-opentelemetry"] diff --git a/configuration/README.md b/configuration/README.md new file mode 100644 index 00000000..bda68b95 --- /dev/null +++ b/configuration/README.md @@ -0,0 +1,41 @@ +# Configuration + +The configuration module is responsible for managing the configuration settings of the NEAR ReadRPC project. +It reads and parses the configuration from a TOML file and provides these settings to other parts of the application. + +## Environment Variables + +The configuration settings can also be provided through environment variables. +This is useful when you want to change the settings without modifying the TOML file, +such as when deploying the application in different environments. +The environment variables are specified in the TOML file using the syntax `${VARIABLE_NAME}`. +For example, `${DATABASE_URL}` specifies the `DATABASE_URL` environment variable. + +## Files + +- `example.config.toml`: This file contains an example configuration for the NEAR ReadRPC. +It includes settings for the general configuration, +RPC server, transaction indexer, state indexer, epoch indexer, rightsizing, lake framework, and database. + + +## Configuration + +The configuration settings are stored in a TOML file. The settings include: + +- General settings like the chain ID and NEAR network RPC URL. +- RPC server settings like the server port and max gas burnt for contract function call. +- Transaction indexer settings like the indexer ID and port for the metrics server. +- State indexer settings like the indexer ID and port for the metrics server. +- Epoch indexer settings like the indexer ID. +- Rightsizing settings like the accounts and state changes to track. +- Lake framework settings like the AWS access key ID and secret access key. +- Database settings like the database connection string and user credentials. + +## Usage + +Put TOML file `config.toml` with configuration in the home root of the project. + +## Note + +Please ensure that the configuration file is correctly formatted and all the required settings are provided. +Incorrect or missing settings may cause the application to behave unexpectedly or fail. diff --git a/configuration/example.config.toml b/configuration/example.config.toml new file mode 100644 index 00000000..03d9f6ea --- /dev/null +++ b/configuration/example.config.toml @@ -0,0 +1,234 @@ +### General configuration for NEAR ReadRPC +[general] + +## Chain ID: testnet or mainnet +chain_id = "mainnet" + +## Near network rpc url +## Using for proxying some requests to near network and to handle finality block +near_rpc_url = "https://beta.rpc.mainnet.near.org" + +## near network archival rpc url +## Using under the hood in near network rpc +## Not needed for regular users. +## Please, don't uncoment this parameter or use same as near_rpc_url +## default value is None +#near_archival_rpc_url = "https://beta.rpc.mainnet.near.org" + +### Rpc server general configuration +[general.rpc_server] + +## Referer header value +## We want to set a custom referer to let NEAR JSON RPC nodes know that we are a read-rpc instance +## Default value is "read-rpc" +#referer_header_value = "read-rpc" + +## Port for RPC server +## Default port is 8000 +#server_port = 8000 + +## Max gas burnt for contract function call +## We allow to use max gas bunt to run contract function call +## Default value is 300_000_000_000_000 +#max_gas_burnt = 300_000_000_000_000 + +## Max available memory for `block_cache` and `contract_code_cache` in gigabytes +## By default we use all available memory +#limit_memory_cache = 0 + +## Reserved memory for running the application in gigabytes +## By default we use 0.25 gigabyte (256MB or 268_435_456 bytes) +#reserved_memory = 0.25 + +## Block cache size in gigabytes +## By default we use 0.125 gigabyte (128MB or 134_217_728 bytes) +## One cache_block size is ≈ 96 bytes +## In 128MB we can put 1_398_101 cache_blocks +#block_cache_size = 0.125 + +### Tx indexer general configuration +[general.tx_indexer] + +## Indexer ID to handle meta data about the instance +## Unique indexer ID +## Default value is "tx-indexer" +## If you run multiple instances of the indexer, you should change this value for each instance +#indexer_id = "tx-indexer" + +## Port for metrics server +## By default it 8080 for tx-indexer and 8081 for state-indexer +#metrics_server_port = 8080 + +## To restore cache from db we use smart range blocks +## Regular transaction takes some blocks to be finalized +## We don't need to restore too old transactions for the indexer because we will probably never be able to reassemble them. +## We use a range of 1000 blocks for our peace of mind. We also leave the option to increase or decrease this range +#cache_restore_blocks_range = 1000 + +### State indexer general configuration +[general.state_indexer] + +## Indexer ID to handle meta data about the instance +## Unique indexer ID +## Default value is "state-indexer" +## If you run multiple instances of the indexer, you should change this value for each instance +#indexer_id = "state-indexer" + +## Port for metrics server +## By default it 8080 for tx-indexer and 8081 for state-indexer +#metrics_server_port = 8081 + +## Concurrency for state-indexer +## Default value is 1 +#concurrency = 1 + +## Epoch indexer general configuration +[general.epoch_indexer] + +## Indexer ID to handle meta data about the instance +## Unique indexer ID +## Epoch indexer is a single instance. it should run once to collect historical data +## If you run state-indexer from genesis, you don't need to run epoch-indexer +#indexer_id = "epoch-indexer" + +### Tracking acconunts and state changes configuration +[rightsizing] + +## Accounts to track. By default we track all accounts. +## You can specify a list of accounts to track. +## tracked_accounts = ["test.near"] +#tracked_accounts = [] + +## State changes to track. By default we track all state changes. +## You can specify a list of state changes to track. +## Possible values: "state", "access_key", "contract_code" +## "accounts" are tracked from the `tracked_accounts` section +#tracked_changes = ["state", "access_key", "contract_code"] + +### Lake framework configuration +[lake_config] + +## Lake framework AWS access key id +aws_access_key_id = "${AWS_ACCESS_KEY_ID}" + +## Lake framework AWS secret access key +aws_secret_access_key = "${AWS_SECRET_ACCESS_KEY}" + +## Lake framework AWS default region +aws_default_region = "eu-central-1" + +## Lake framework bucket name +aws_bucket_name = "near-lake-data-mainnet" + +## Database configuration +[database] + +## Database connection string + +## Postgres Example: +## You can use database connection URL +## postgresql://{user}:{password}@localhost:5432/{db_name} +## Or you can use split configuration +## Put postgres database host and put credentials and db_name in database_user, database_password and database_name +#database_url = "localhost:5432" +database_url = "postgresql://postgres:password@localhost:5432/near-indexer" + +## Database user(login) +## Optional database user +#database_user = "postgres" + +## Database password +## Optional database password +#database_password = "password" + +## Postgres database name +## Optional database name +#database_name = "near-indexer" + +## ScyllaDB Example: +## ScyllaDB database connection URL +#database_url = "127.0.0.1:9042" + +## Database user(login) +## Optional database user +#database_user = "cassandra" + +## Database password +## Optional database password +#database_password = "password" + +### Database configuration for rpc server +[database.rpc_server] + +## ScyllaDB preferred DataCenter +## Accepts the DC name of the ScyllaDB to filter the connection to that DC only (preferrably). +## If you connect to multi-DC cluter, you might experience big latencies while working with the DB. +## This is due to the fact that ScyllaDB driver tries to connect to any of the nodes in the cluster disregarding of the location of the DC. +## This option allows to filter the connection to the DC you need. +## Example: "DC1" where DC1 is located in the same region as the application. +## Default value is None +#preferred_dc = "DC1" + +## Max retry count for ScyllaDB if `strict_mode` is `false` +## By default for rpc server we use 2 +#max_retry = 2 + +## Attempts to store data in the database should be infinite to ensure no data is missing. +## Disable it to perform a limited write attempts (`max_retry`) +## before giving up, and moving to the next piece of data +## By default for rpc server we use false +#strict_mode = false + +## ScyllaDB keepalive interval +## This parameter need only for ScyllaDB and only for rpc server +## By default we use 60 seconds +#keepalive_interval = 60 + +### Database configuration for tx indexer +[database.tx_indexer] + +## ScyllaDB preferred DataCenter +## Accepts the DC name of the ScyllaDB to filter the connection to that DC only (preferrably). +## If you connect to multi-DC cluter, you might experience big latencies while working with the DB. +## This is due to the fact that ScyllaDB driver tries to connect to any of the nodes in the cluster disregarding of the location of the DC. +## This option allows to filter the connection to the DC you need. +## Example: "DC1" where DC1 is located in the same region as the application. +## Default value is None +#preferred_dc = "DC1" + +## Max retry count for ScyllaDB if `strict_mode` is `false` +## For indexers we use 5 attempts by default +#max_retry = 5 + +## Attempts to store data in the database should be infinite to ensure no data is missing. +## Disable it to perform a limited write attempts (`max_retry`) +## before giving up, and moving to the next piece of data +## For indexers we use true by default +#strict_mode = true + +## Parallel queries = (nodes in cluster) ✕ (cores in node) ✕ 3 +## Current we have 6 - nodes with 8 - cpus +## 6 ✕ 8 ✕ 3 = 144 +#max_db_parallel_queries = 144 + +## Database configuration for state indexer +[database.state_indexer] + +## ScyllaDB preferred DataCenter +## Accepts the DC name of the ScyllaDB to filter the connection to that DC only (preferrably). +## If you connect to multi-DC cluter, you might experience big latencies while working with the DB. +## This is due to the fact that ScyllaDB driver tries to connect to any of the nodes in the cluster disregarding of the location of the DC. +## This option allows to filter the connection to the DC you need. +## Example: "DC1" where DC1 is located in the same region as the application. +## Default value is None +#preferred_dc = "DC1" + +## Max retry count for ScyllaDB if `strict_mode` is `false` +## For indexers we use 5 attempts by default +#max_retry = 5 + +## Attempts to store data in the database should be infinite to ensure no data is missing. +## Disable it to perform a limited write attempts (`max_retry`) +## before giving up, and moving to the next piece of data +## For indexers we use true by default +#strict_mode = true diff --git a/configuration/src/configs/database.rs b/configuration/src/configs/database.rs new file mode 100644 index 00000000..42b7a5bd --- /dev/null +++ b/configuration/src/configs/database.rs @@ -0,0 +1,286 @@ +use serde_derive::Deserialize; + +use crate::configs::{deserialize_data_or_env, deserialize_optional_data_or_env}; + +#[derive(Debug, Clone)] +pub struct DatabaseConfig { + pub database_url: String, + pub database_user: Option, + pub database_password: Option, + pub database_name: Option, + pub preferred_dc: Option, + pub max_retry: u8, + pub strict_mode: bool, + pub keepalive_interval: Option, + pub max_db_parallel_queries: i64, +} + +pub struct DatabaseRpcServerConfig { + pub database_url: String, + pub database_user: Option, + pub database_password: Option, + pub database_name: Option, + pub preferred_dc: Option, + pub max_retry: u8, + pub strict_mode: bool, + pub keepalive_interval: u64, +} + +pub struct DatabaseTxIndexerConfig { + pub database_url: String, + pub database_user: Option, + pub database_password: Option, + pub database_name: Option, + pub preferred_dc: Option, + pub max_retry: u8, + pub strict_mode: bool, + pub max_db_parallel_queries: i64, +} + +pub struct DatabaseStateIndexerConfig { + pub database_url: String, + pub database_user: Option, + pub database_password: Option, + pub database_name: Option, + pub preferred_dc: Option, + pub max_retry: u8, + pub strict_mode: bool, +} + +#[derive(Deserialize, Debug, Clone, Default)] +pub struct CommonDatabaseConfig { + #[serde(deserialize_with = "deserialize_data_or_env")] + pub database_url: String, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub database_user: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub database_password: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub database_name: Option, + #[serde(default)] + pub rpc_server: CommonDatabaseRpcServerConfig, + #[serde(default)] + pub tx_indexer: CommonDatabaseTxIndexerConfig, + #[serde(default)] + pub state_indexer: CommonDatabaseStateIndexerConfig, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CommonDatabaseRpcServerConfig { + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub preferred_dc: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub max_retry: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub strict_mode: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub keepalive_interval: Option, +} + +impl CommonDatabaseRpcServerConfig { + pub fn default_max_retry() -> u8 { + 2 + } + + pub fn default_strict_mode() -> bool { + false + } + + pub fn default_keepalive_interval() -> u64 { + 60 + } +} + +impl Default for CommonDatabaseRpcServerConfig { + fn default() -> Self { + Self { + preferred_dc: Default::default(), + max_retry: Some(Self::default_max_retry()), + strict_mode: Some(Self::default_strict_mode()), + keepalive_interval: Some(Self::default_keepalive_interval()), + } + } +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CommonDatabaseTxIndexerConfig { + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub preferred_dc: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub max_retry: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub strict_mode: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub max_db_parallel_queries: Option, +} + +impl CommonDatabaseTxIndexerConfig { + pub fn default_max_retry() -> u8 { + 5 + } + + pub fn default_strict_mode() -> bool { + true + } + + pub fn default_max_db_parallel_queries() -> i64 { + 144 + } +} + +impl Default for CommonDatabaseTxIndexerConfig { + fn default() -> Self { + Self { + preferred_dc: Default::default(), + max_retry: Some(Self::default_max_retry()), + strict_mode: Some(Self::default_strict_mode()), + max_db_parallel_queries: Some(Self::default_max_db_parallel_queries()), + } + } +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CommonDatabaseStateIndexerConfig { + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub preferred_dc: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub max_retry: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub strict_mode: Option, +} + +impl CommonDatabaseStateIndexerConfig { + pub fn default_max_retry() -> u8 { + 5 + } + + pub fn default_strict_mode() -> bool { + true + } +} + +impl Default for CommonDatabaseStateIndexerConfig { + fn default() -> Self { + Self { + preferred_dc: Default::default(), + max_retry: Some(Self::default_max_retry()), + strict_mode: Some(Self::default_strict_mode()), + } + } +} + +impl From for DatabaseRpcServerConfig { + fn from(common_config: CommonDatabaseConfig) -> Self { + Self { + database_url: common_config.database_url, + database_user: common_config.database_user, + database_password: common_config.database_password, + database_name: common_config.database_name, + preferred_dc: common_config.rpc_server.preferred_dc, + max_retry: common_config + .rpc_server + .max_retry + .unwrap_or_else(CommonDatabaseRpcServerConfig::default_max_retry), + strict_mode: common_config + .rpc_server + .strict_mode + .unwrap_or_else(CommonDatabaseRpcServerConfig::default_strict_mode), + keepalive_interval: common_config + .rpc_server + .keepalive_interval + .unwrap_or_else(CommonDatabaseRpcServerConfig::default_keepalive_interval), + } + } +} + +impl From for DatabaseTxIndexerConfig { + fn from(common_config: CommonDatabaseConfig) -> Self { + Self { + database_url: common_config.database_url, + database_user: common_config.database_user, + database_password: common_config.database_password, + database_name: common_config.database_name, + preferred_dc: common_config.tx_indexer.preferred_dc, + max_retry: common_config + .tx_indexer + .max_retry + .unwrap_or_else(CommonDatabaseTxIndexerConfig::default_max_retry), + strict_mode: common_config + .tx_indexer + .strict_mode + .unwrap_or_else(CommonDatabaseTxIndexerConfig::default_strict_mode), + max_db_parallel_queries: common_config + .tx_indexer + .max_db_parallel_queries + .unwrap_or_else(CommonDatabaseTxIndexerConfig::default_max_db_parallel_queries), + } + } +} + +impl From for DatabaseStateIndexerConfig { + fn from(common_config: CommonDatabaseConfig) -> Self { + Self { + database_url: common_config.database_url, + database_user: common_config.database_user, + database_password: common_config.database_password, + database_name: common_config.database_name, + preferred_dc: common_config.state_indexer.preferred_dc, + max_retry: common_config + .state_indexer + .max_retry + .unwrap_or_else(CommonDatabaseStateIndexerConfig::default_max_retry), + strict_mode: common_config + .state_indexer + .strict_mode + .unwrap_or_else(CommonDatabaseStateIndexerConfig::default_strict_mode), + } + } +} + +impl From for DatabaseConfig { + fn from(rpc_server_config: DatabaseRpcServerConfig) -> Self { + Self { + database_url: rpc_server_config.database_url, + database_user: rpc_server_config.database_user, + database_password: rpc_server_config.database_password, + database_name: rpc_server_config.database_name, + preferred_dc: rpc_server_config.preferred_dc, + max_retry: rpc_server_config.max_retry, + strict_mode: rpc_server_config.strict_mode, + keepalive_interval: Some(rpc_server_config.keepalive_interval), + max_db_parallel_queries: Default::default(), + } + } +} + +impl From for DatabaseConfig { + fn from(tx_indexer_config: DatabaseTxIndexerConfig) -> Self { + Self { + database_url: tx_indexer_config.database_url, + database_user: tx_indexer_config.database_user, + database_password: tx_indexer_config.database_password, + database_name: tx_indexer_config.database_name, + preferred_dc: tx_indexer_config.preferred_dc, + max_retry: tx_indexer_config.max_retry, + strict_mode: tx_indexer_config.strict_mode, + keepalive_interval: Default::default(), + max_db_parallel_queries: tx_indexer_config.max_db_parallel_queries, + } + } +} + +impl From for DatabaseConfig { + fn from(state_indexer_config: DatabaseStateIndexerConfig) -> Self { + Self { + database_url: state_indexer_config.database_url, + database_user: state_indexer_config.database_user, + database_password: state_indexer_config.database_password, + database_name: state_indexer_config.database_name, + preferred_dc: state_indexer_config.preferred_dc, + max_retry: state_indexer_config.max_retry, + strict_mode: state_indexer_config.strict_mode, + keepalive_interval: Default::default(), + max_db_parallel_queries: Default::default(), + } + } +} diff --git a/configuration/src/configs/general.rs b/configuration/src/configs/general.rs new file mode 100644 index 00000000..9d0c408d --- /dev/null +++ b/configuration/src/configs/general.rs @@ -0,0 +1,312 @@ +use std::str::FromStr; + +use serde_derive::Deserialize; + +use crate::configs::{deserialize_data_or_env, deserialize_optional_data_or_env}; + +#[derive(Debug, Clone)] +pub struct GeneralRpcServerConfig { + pub chain_id: ChainId, + pub near_rpc_url: String, + pub near_archival_rpc_url: Option, + pub referer_header_value: String, + pub server_port: u16, + pub max_gas_burnt: u64, + pub limit_memory_cache: Option, + pub reserved_memory: f64, + pub block_cache_size: f64, +} + +#[derive(Debug, Clone)] +pub struct GeneralTxIndexerConfig { + pub chain_id: ChainId, + pub near_rpc_url: String, + pub near_archival_rpc_url: Option, + pub indexer_id: String, + pub metrics_server_port: u16, + pub cache_restore_blocks_range: u64, +} + +#[derive(Debug, Clone)] +pub struct GeneralStateIndexerConfig { + pub chain_id: ChainId, + pub near_rpc_url: String, + pub near_archival_rpc_url: Option, + pub indexer_id: String, + pub metrics_server_port: u16, + pub concurrency: usize, +} + +#[derive(Debug, Clone)] +pub struct GeneralEpochIndexerConfig { + pub chain_id: ChainId, + pub near_rpc_url: String, + pub near_archival_rpc_url: Option, + pub indexer_id: String, +} + +#[derive(Deserialize, Debug, Clone, Default)] +pub struct CommonGeneralConfig { + #[serde(deserialize_with = "deserialize_data_or_env")] + pub chain_id: ChainId, + #[serde(deserialize_with = "deserialize_data_or_env")] + pub near_rpc_url: String, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub near_archival_rpc_url: Option, + #[serde(default)] + pub rpc_server: CommonGeneralRpcServerConfig, + #[serde(default)] + pub tx_indexer: CommonGeneralTxIndexerConfig, + #[serde(default)] + pub state_indexer: CommonGeneralStateIndexerConfig, + #[serde(default)] + pub epoch_indexer: CommonGeneralEpochIndexerConfig, +} + +#[derive(Deserialize, PartialEq, Debug, Clone, Default)] +#[serde(rename_all = "lowercase")] +pub enum ChainId { + #[default] + Mainnet, + Testnet, +} + +impl FromStr for ChainId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "mainnet" => Ok(ChainId::Mainnet), + "testnet" => Ok(ChainId::Testnet), + _ => Err(anyhow::anyhow!("Invalid chain id")), + } + } +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CommonGeneralRpcServerConfig { + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub referer_header_value: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub server_port: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub max_gas_burnt: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub limit_memory_cache: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub reserved_memory: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub block_cache_size: Option, +} + +impl CommonGeneralRpcServerConfig { + pub fn default_referer_header_value() -> String { + "read-rpc".to_string() + } + + pub fn default_server_port() -> u16 { + 8080 + } + + pub fn default_max_gas_burnt() -> u64 { + 300_000_000_000_000 + } + + pub fn default_reserved_memory() -> f64 { + 0.25 + } + + pub fn default_block_cache_size() -> f64 { + 0.125 + } +} + +impl Default for CommonGeneralRpcServerConfig { + fn default() -> Self { + Self { + referer_header_value: Some(Self::default_referer_header_value()), + server_port: Some(Self::default_server_port()), + max_gas_burnt: Some(Self::default_max_gas_burnt()), + limit_memory_cache: Default::default(), + reserved_memory: Some(Self::default_reserved_memory()), + block_cache_size: Some(Self::default_block_cache_size()), + } + } +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CommonGeneralTxIndexerConfig { + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub indexer_id: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub metrics_server_port: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub cache_restore_blocks_range: Option, +} + +impl CommonGeneralTxIndexerConfig { + pub fn default_indexer_id() -> String { + "tx-indexer".to_string() + } + + pub fn default_metrics_server_port() -> u16 { + 8080 + } + + pub fn default_cache_restore_blocks_range() -> u64 { + 1000 + } +} + +impl Default for CommonGeneralTxIndexerConfig { + fn default() -> Self { + Self { + indexer_id: Some(Self::default_indexer_id()), + metrics_server_port: Some(Self::default_metrics_server_port()), + cache_restore_blocks_range: Some(Self::default_cache_restore_blocks_range()), + } + } +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CommonGeneralStateIndexerConfig { + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub indexer_id: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub metrics_server_port: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub concurrency: Option, +} + +impl CommonGeneralStateIndexerConfig { + pub fn default_indexer_id() -> String { + "state-indexer".to_string() + } + + pub fn default_metrics_server_port() -> u16 { + 8081 + } + + pub fn default_concurrency() -> usize { + 1 + } +} + +impl Default for CommonGeneralStateIndexerConfig { + fn default() -> Self { + Self { + indexer_id: Some(Self::default_indexer_id()), + metrics_server_port: Some(Self::default_metrics_server_port()), + concurrency: Some(Self::default_concurrency()), + } + } +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CommonGeneralEpochIndexerConfig { + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub indexer_id: Option, +} + +impl CommonGeneralEpochIndexerConfig { + pub fn default_indexer_id() -> String { + "epoch-indexer".to_string() + } +} + +impl Default for CommonGeneralEpochIndexerConfig { + fn default() -> Self { + Self { + indexer_id: Some(Self::default_indexer_id()), + } + } +} + +impl From for GeneralRpcServerConfig { + fn from(common_config: CommonGeneralConfig) -> Self { + Self { + chain_id: common_config.chain_id, + near_rpc_url: common_config.near_rpc_url, + near_archival_rpc_url: common_config.near_archival_rpc_url, + referer_header_value: common_config + .rpc_server + .referer_header_value + .unwrap_or_else(CommonGeneralRpcServerConfig::default_referer_header_value), + server_port: common_config + .rpc_server + .server_port + .unwrap_or_else(CommonGeneralRpcServerConfig::default_server_port), + max_gas_burnt: common_config + .rpc_server + .max_gas_burnt + .unwrap_or_else(CommonGeneralRpcServerConfig::default_max_gas_burnt), + limit_memory_cache: common_config.rpc_server.limit_memory_cache, + reserved_memory: common_config + .rpc_server + .reserved_memory + .unwrap_or_else(CommonGeneralRpcServerConfig::default_reserved_memory), + block_cache_size: common_config + .rpc_server + .block_cache_size + .unwrap_or_else(CommonGeneralRpcServerConfig::default_block_cache_size), + } + } +} + +impl From for GeneralTxIndexerConfig { + fn from(common_config: CommonGeneralConfig) -> Self { + Self { + chain_id: common_config.chain_id, + near_rpc_url: common_config.near_rpc_url, + near_archival_rpc_url: common_config.near_archival_rpc_url, + indexer_id: common_config + .tx_indexer + .indexer_id + .unwrap_or_else(CommonGeneralTxIndexerConfig::default_indexer_id), + metrics_server_port: common_config + .tx_indexer + .metrics_server_port + .unwrap_or_else(CommonGeneralTxIndexerConfig::default_metrics_server_port), + cache_restore_blocks_range: common_config + .tx_indexer + .cache_restore_blocks_range + .unwrap_or_else(CommonGeneralTxIndexerConfig::default_cache_restore_blocks_range), + } + } +} + +impl From for GeneralStateIndexerConfig { + fn from(common_config: CommonGeneralConfig) -> Self { + Self { + chain_id: common_config.chain_id, + near_rpc_url: common_config.near_rpc_url, + near_archival_rpc_url: common_config.near_archival_rpc_url, + indexer_id: common_config + .state_indexer + .indexer_id + .unwrap_or_else(CommonGeneralStateIndexerConfig::default_indexer_id), + metrics_server_port: common_config + .state_indexer + .metrics_server_port + .unwrap_or_else(CommonGeneralStateIndexerConfig::default_metrics_server_port), + concurrency: common_config + .state_indexer + .concurrency + .unwrap_or_else(CommonGeneralStateIndexerConfig::default_concurrency), + } + } +} + +impl From for GeneralEpochIndexerConfig { + fn from(common_config: CommonGeneralConfig) -> Self { + Self { + chain_id: common_config.chain_id, + near_rpc_url: common_config.near_rpc_url, + near_archival_rpc_url: common_config.near_archival_rpc_url, + indexer_id: common_config + .epoch_indexer + .indexer_id + .unwrap_or_else(CommonGeneralEpochIndexerConfig::default_indexer_id), + } + } +} diff --git a/configuration/src/configs/lake.rs b/configuration/src/configs/lake.rs new file mode 100644 index 00000000..15b1d351 --- /dev/null +++ b/configuration/src/configs/lake.rs @@ -0,0 +1,55 @@ +use near_lake_framework::near_indexer_primitives::near_primitives; +use serde_derive::Deserialize; + +use crate::configs::deserialize_data_or_env; + +#[derive(Deserialize, Debug, Clone, Default)] +pub struct LakeConfig { + #[serde(deserialize_with = "deserialize_data_or_env")] + pub aws_access_key_id: String, + #[serde(deserialize_with = "deserialize_data_or_env")] + pub aws_secret_access_key: String, + #[serde(deserialize_with = "deserialize_data_or_env")] + pub aws_default_region: String, + #[serde(deserialize_with = "deserialize_data_or_env")] + pub aws_bucket_name: String, +} + +impl LakeConfig { + pub async fn s3_config(&self) -> aws_sdk_s3::Config { + let credentials = aws_credential_types::Credentials::new( + &self.aws_access_key_id, + &self.aws_secret_access_key, + None, + None, + "", + ); + aws_sdk_s3::Config::builder() + .credentials_provider(credentials) + .region(aws_types::region::Region::new( + self.aws_default_region.clone(), + )) + .build() + } + + pub async fn lake_config( + &self, + start_block_height: near_primitives::types::BlockHeight, + ) -> anyhow::Result { + let config_builder = near_lake_framework::LakeConfigBuilder::default(); + Ok(config_builder + .s3_config(self.s3_config().await) + .s3_region_name(&self.aws_default_region) + .s3_bucket_name(&self.aws_bucket_name) + .start_block_height(start_block_height) + .build() + .expect("Failed to build LakeConfig")) + } + + pub async fn lake_s3_client(&self) -> near_lake_framework::s3_fetchers::LakeS3Client { + let s3_config = self.s3_config().await; + near_lake_framework::s3_fetchers::LakeS3Client::new(aws_sdk_s3::Client::from_conf( + s3_config, + )) + } +} diff --git a/configuration/src/configs/mod.rs b/configuration/src/configs/mod.rs new file mode 100644 index 00000000..841cb735 --- /dev/null +++ b/configuration/src/configs/mod.rs @@ -0,0 +1,166 @@ +use std::str::FromStr; + +use near_lake_framework::{ + near_indexer_primitives, near_indexer_primitives::views::StateChangeValueView, +}; +use serde::Deserialize; + +pub(crate) mod database; +pub(crate) mod general; +mod lake; +mod rightsizing; + +lazy_static::lazy_static! { + static ref RE_NAME_ENV: regex::Regex = regex::Regex::new(r"\$\{(?\w+)}").unwrap(); +} + +fn get_env_var(env_var_name: &str) -> anyhow::Result +where + T: FromStr, + T::Err: std::fmt::Debug, +{ + let var = dotenv::var(env_var_name).map_err(|err| { + anyhow::anyhow!( + "Failed to get env var: {:?}. Error: {:?}", + env_var_name, + err + ) + })?; + var.parse::().map_err(|err| { + anyhow::anyhow!( + "Failed to parse env var: {:?}. Error: {:?}", + env_var_name, + err + ) + }) +} + +fn deserialize_data_or_env<'de, D, T>(data: D) -> Result +where + D: serde::Deserializer<'de>, + T: serde::de::DeserializeOwned + FromStr, + ::Err: std::fmt::Debug, +{ + let value = serde_json::Value::deserialize(data)?; + if let serde_json::Value::String(value) = &value { + if let Some(caps) = RE_NAME_ENV.captures(value) { + return get_env_var::(&caps["env_name"]).map_err(serde::de::Error::custom); + } + } + serde_json::from_value::(value).map_err(serde::de::Error::custom) +} + +fn deserialize_optional_data_or_env<'de, D, T>(data: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, + T: serde::de::DeserializeOwned + FromStr, + ::Err: std::fmt::Debug, +{ + Ok(match deserialize_data_or_env(data) { + Ok(value) => Some(value), + Err(err) => { + tracing::warn!("Failed to deserialize_optional_data_or_env: {:?}", err); + None + } + }) +} + +#[derive(Deserialize, Debug, Clone, Default)] +pub struct CommonConfig { + pub general: general::CommonGeneralConfig, + #[serde(default)] + pub rightsizing: rightsizing::CommonRightsizingConfig, + pub lake_config: lake::LakeConfig, + pub database: database::CommonDatabaseConfig, +} + +pub trait Config { + fn from_common_config(common_config: CommonConfig) -> Self; +} + +#[derive(Debug, Clone)] +pub struct RpcServerConfig { + pub general: general::GeneralRpcServerConfig, + pub lake_config: lake::LakeConfig, + pub database: database::DatabaseConfig, +} + +impl Config for RpcServerConfig { + fn from_common_config(common_config: CommonConfig) -> Self { + Self { + general: common_config.general.into(), + lake_config: common_config.lake_config, + database: database::DatabaseRpcServerConfig::from(common_config.database).into(), + } + } +} + +#[derive(Debug, Clone)] +pub struct TxIndexerConfig { + pub general: general::GeneralTxIndexerConfig, + pub rightsizing: rightsizing::RightsizingConfig, + pub lake_config: lake::LakeConfig, + pub database: database::DatabaseConfig, +} + +impl TxIndexerConfig { + pub fn tx_should_be_indexed( + &self, + transaction: &near_indexer_primitives::IndexerTransactionWithOutcome, + ) -> bool { + self.rightsizing.tx_should_be_indexed(transaction) + } +} + +impl Config for TxIndexerConfig { + fn from_common_config(common_config: CommonConfig) -> Self { + Self { + general: common_config.general.into(), + rightsizing: common_config.rightsizing.into(), + lake_config: common_config.lake_config, + database: database::DatabaseTxIndexerConfig::from(common_config.database).into(), + } + } +} + +#[derive(Debug, Clone)] +pub struct StateIndexerConfig { + pub general: general::GeneralStateIndexerConfig, + pub rightsizing: rightsizing::RightsizingConfig, + pub lake_config: lake::LakeConfig, + pub database: database::DatabaseConfig, +} + +impl StateIndexerConfig { + pub fn state_should_be_indexed(&self, state_change_value: &StateChangeValueView) -> bool { + self.rightsizing.state_should_be_indexed(state_change_value) + } +} + +impl Config for StateIndexerConfig { + fn from_common_config(common_config: CommonConfig) -> Self { + Self { + general: common_config.general.into(), + rightsizing: common_config.rightsizing.into(), + lake_config: common_config.lake_config, + database: database::DatabaseStateIndexerConfig::from(common_config.database).into(), + } + } +} + +#[derive(Debug, Clone)] +pub struct EpochIndexerConfig { + pub general: general::GeneralEpochIndexerConfig, + pub lake_config: lake::LakeConfig, + pub database: database::DatabaseConfig, +} + +impl Config for EpochIndexerConfig { + fn from_common_config(common_config: CommonConfig) -> Self { + Self { + general: common_config.general.into(), + lake_config: common_config.lake_config, + database: database::DatabaseStateIndexerConfig::from(common_config.database).into(), + } + } +} diff --git a/configuration/src/configs/rightsizing.rs b/configuration/src/configs/rightsizing.rs new file mode 100644 index 00000000..ba0ceb41 --- /dev/null +++ b/configuration/src/configs/rightsizing.rs @@ -0,0 +1,120 @@ +use near_lake_framework::{ + near_indexer_primitives, near_indexer_primitives::views::StateChangeValueView, +}; +use serde::Deserialize; +use std::str::FromStr; + +use crate::configs::deserialize_optional_data_or_env; + +#[derive(Deserialize, Debug, Clone, Default)] +pub struct TrackedAccounts(pub Vec); + +impl FromStr for TrackedAccounts { + type Err = serde_json::Error; + + fn from_str(s: &str) -> Result { + Ok(Self(serde_json::from_str::< + Vec, + >(s)?)) + } +} + +#[derive(Deserialize, Debug, Clone, Default)] +pub struct TrackedChanges(pub Vec); + +impl FromStr for TrackedChanges { + type Err = serde_json::Error; + + fn from_str(s: &str) -> Result { + Ok(Self(serde_json::from_str::>(s)?)) + } +} + +#[derive(Deserialize, PartialEq, Debug, Clone)] +#[serde(rename_all = "snake_case")] +pub enum ChangeType { + State, + AccessKey, + ContractCode, +} + +#[derive(Deserialize, Debug, Clone, Default)] +pub struct CommonRightsizingConfig { + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub tracked_accounts: Option, + #[serde(deserialize_with = "deserialize_optional_data_or_env", default)] + pub tracked_changes: Option, +} + +#[derive(Debug, Clone)] +pub struct RightsizingConfig { + pub tracked_accounts: Vec, + pub tracked_changes: Vec, +} + +impl RightsizingConfig { + fn is_indexed_account(&self, account: &near_indexer_primitives::types::AccountId) -> bool { + if self.tracked_accounts.is_empty() { + true + } else { + self.tracked_accounts.contains(account) + } + } + + fn is_indexed_change(&self, change_type: ChangeType) -> bool { + if self.tracked_changes.is_empty() { + true + } else { + self.tracked_changes.contains(&change_type) + } + } + + pub fn state_should_be_indexed(&self, state_change_value: &StateChangeValueView) -> bool { + match state_change_value { + StateChangeValueView::DataUpdate { account_id, .. } + | StateChangeValueView::DataDeletion { account_id, .. } => { + self.is_indexed_account(account_id) && self.is_indexed_change(ChangeType::State) + } + StateChangeValueView::AccessKeyUpdate { account_id, .. } + | StateChangeValueView::AccessKeyDeletion { account_id, .. } => { + self.is_indexed_account(account_id) && self.is_indexed_change(ChangeType::AccessKey) + } + StateChangeValueView::ContractCodeUpdate { account_id, .. } + | StateChangeValueView::ContractCodeDeletion { account_id, .. } => { + self.is_indexed_account(account_id) + && self.is_indexed_change(ChangeType::ContractCode) + } + StateChangeValueView::AccountUpdate { account_id, .. } + | StateChangeValueView::AccountDeletion { account_id, .. } => { + self.is_indexed_account(account_id) + } + } + } + + /// For now we index only transactions that are related to indexed accounts as signer_id and receiver_id + /// But we know about transactions which include indexing accounts not only as signer_id and receiver_id + /// but also include indexing accounts in a args of a function call + /// So in future we should to index such transactions too if it will be needed + pub fn tx_should_be_indexed( + &self, + transaction: &near_indexer_primitives::IndexerTransactionWithOutcome, + ) -> bool { + self.is_indexed_account(&transaction.transaction.signer_id) + || self.is_indexed_account(&transaction.transaction.receiver_id) + } +} + +impl From for RightsizingConfig { + fn from(common_rightsizing_config: CommonRightsizingConfig) -> Self { + Self { + tracked_accounts: common_rightsizing_config + .tracked_accounts + .unwrap_or_default() + .0, + tracked_changes: common_rightsizing_config + .tracked_changes + .unwrap_or_default() + .0, + } + } +} diff --git a/configuration/src/lib.rs b/configuration/src/lib.rs new file mode 100644 index 00000000..6081c859 --- /dev/null +++ b/configuration/src/lib.rs @@ -0,0 +1,128 @@ +use std::path::PathBuf; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; + +mod configs; + +pub use crate::configs::database::DatabaseConfig; +pub use crate::configs::general::ChainId; +pub use crate::configs::{ + EpochIndexerConfig, RpcServerConfig, StateIndexerConfig, TxIndexerConfig, +}; + +pub async fn read_configuration() -> anyhow::Result +where + T: configs::Config + Send + Sync + 'static, +{ + let path_root = find_configs_root().await?; + load_env(path_root.clone()).await?; + let common_config = read_toml_file(path_root).await?; + Ok(T::from_common_config(common_config)) +} + +pub async fn init_tracing(service_name: &str) -> anyhow::Result<()> { + let path_root = find_configs_root().await?; + load_env(path_root.clone()).await?; + + let mut env_filter = tracing_subscriber::EnvFilter::new(format!("{}=info", service_name)); + + if let Ok(rust_log) = std::env::var("RUST_LOG") { + if !rust_log.is_empty() { + for directive in rust_log.split(',').filter_map(|s| match s.parse() { + Ok(directive) => Some(directive), + Err(err) => { + eprintln!("Ignoring directive `{}`: {}", s, err); + None + } + }) { + env_filter = env_filter.add_directive(directive); + } + } + } + + opentelemetry::global::shutdown_tracer_provider(); + + opentelemetry::global::set_text_map_propagator( + opentelemetry::sdk::propagation::TraceContextPropagator::new(), + ); + + #[cfg(feature = "tracing-instrumentation")] + let subscriber = { + let tracer = opentelemetry_jaeger::new_collector_pipeline() + .with_service_name(service_name) + .with_endpoint(std::env::var("OTEL_EXPORTER_JAEGER_ENDPOINT").unwrap_or_default()) + .with_isahc() + .with_batch_processor_config( + opentelemetry::sdk::trace::BatchConfig::default() + .with_max_queue_size(10_000) + .with_max_export_batch_size(10_000) + .with_max_concurrent_exports(100), + ) + .install_batch(opentelemetry::runtime::TokioCurrentThread)?; + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + + tracing_subscriber::Registry::default() + .with(env_filter) + .with(telemetry) + }; + + #[cfg(not(feature = "tracing-instrumentation"))] + let subscriber = tracing_subscriber::Registry::default().with(env_filter); + + if std::env::var("ENABLE_JSON_LOGS").is_ok() { + subscriber.with(tracing_stackdriver::layer()).try_init()?; + } else { + subscriber + .with(tracing_subscriber::fmt::Layer::default().compact()) + .try_init()?; + } + + Ok(()) +} + +async fn load_env(mut path_root: PathBuf) -> anyhow::Result<()> { + path_root.push(".env"); + if path_root.exists() { + dotenv::from_path(path_root.as_path()).ok(); + } else { + dotenv::dotenv().ok(); + } + Ok(()) +} + +async fn read_toml_file(mut path_root: PathBuf) -> anyhow::Result { + path_root.push("config.toml"); + match std::fs::read_to_string(path_root.as_path()) { + Ok(content) => match toml::from_str::(&content) { + Ok(config) => Ok(config), + Err(err) => { + anyhow::bail!( + "Unable to load data from: {:?}.\n Error: {}", + path_root.to_str(), + err + ); + } + }, + Err(err) => { + anyhow::bail!( + "Could not read file: {:?}.\n Error: {}", + path_root.to_str(), + err + ); + } + } +} + +async fn find_configs_root() -> anyhow::Result { + let current_path = std::env::current_dir()?; + + for path_config in current_path.as_path().ancestors() { + let has_config = std::fs::read_dir(path_config)?.any(|path| { + path.unwrap().file_name() == std::ffi::OsString::from(String::from("config.toml")) + }); + if has_config { + return Ok(PathBuf::from(path_config)); + } + } + anyhow::bail!("Ran out of places to find config.toml") +} diff --git a/database/Cargo.toml b/database/Cargo.toml index d4653b0e..aed1d27d 100644 --- a/database/Cargo.toml +++ b/database/Cargo.toml @@ -28,6 +28,7 @@ tokio = { version = "1.19.2", features = [ tracing = "0.1.34" uuid = { version = "1.3.0", optional = true } +configuration = { path = "../configuration" } readnode-primitives = { path = "../readnode-primitives" } near-chain-configs = "1.36.0" diff --git a/database/src/base/mod.rs b/database/src/base/mod.rs index 0ba54d67..476b2f96 100644 --- a/database/src/base/mod.rs +++ b/database/src/base/mod.rs @@ -46,10 +46,5 @@ pub type PageToken = Option; #[async_trait::async_trait] pub trait BaseDbManager { - async fn new( - database_url: &str, - database_user: Option<&str>, - database_password: Option<&str>, - database_options: AdditionalDatabaseOptions, - ) -> anyhow::Result>; + async fn new(config: &configuration::DatabaseConfig) -> anyhow::Result>; } diff --git a/database/src/lib.rs b/database/src/lib.rs index 1fddab0d..85ac89d2 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -18,20 +18,9 @@ pub use postgres::{models, schema}; pub mod primitives; -pub async fn prepare_db_manager( - database_url: &str, - database_user: Option<&str>, - database_password: Option<&str>, - database_options: AdditionalDatabaseOptions, -) -> anyhow::Result +pub async fn prepare_db_manager(config: &configuration::DatabaseConfig) -> anyhow::Result where T: BaseDbManager + Send + Sync + 'static, { - Ok(*T::new( - database_url, - database_user, - database_password, - database_options, - ) - .await?) + Ok(*T::new(config).await?) } diff --git a/database/src/postgres/mod.rs b/database/src/postgres/mod.rs index 2a7cec17..cd50edff 100644 --- a/database/src/postgres/mod.rs +++ b/database/src/postgres/mod.rs @@ -15,7 +15,7 @@ pub trait PostgresStorageManager { database_url: &str, database_user: Option<&str>, database_password: Option<&str>, - database_options: crate::AdditionalDatabaseOptions, + database_name: Option<&str>, ) -> anyhow::Result { let connection_string = if database_url.starts_with("postgres://") { database_url.to_string() @@ -25,7 +25,7 @@ pub trait PostgresStorageManager { database_user.unwrap(), database_password.unwrap(), database_url, - database_options.database_name.unwrap() + database_name.unwrap() ) }; let config = diesel_async::pooled_connection::AsyncDieselConnectionManager::< diff --git a/database/src/postgres/rpc_server.rs b/database/src/postgres/rpc_server.rs index 7c8d4429..c5071140 100644 --- a/database/src/postgres/rpc_server.rs +++ b/database/src/postgres/rpc_server.rs @@ -1,28 +1,21 @@ use std::str::FromStr; +use crate::postgres::PostgresStorageManager; use bigdecimal::ToPrimitive; use borsh::{BorshDeserialize, BorshSerialize}; -use crate::postgres::PostgresStorageManager; -use crate::AdditionalDatabaseOptions; - pub struct PostgresDBManager { pg_pool: crate::postgres::PgAsyncPool, } #[async_trait::async_trait] impl crate::BaseDbManager for PostgresDBManager { - async fn new( - database_url: &str, - database_user: Option<&str>, - database_password: Option<&str>, - database_options: AdditionalDatabaseOptions, - ) -> anyhow::Result> { + async fn new(config: &configuration::DatabaseConfig) -> anyhow::Result> { let pg_pool = Self::create_pool( - database_url, - database_user, - database_password, - database_options, + &config.database_url, + config.database_user.as_deref(), + config.database_password.as_deref(), + config.database_name.as_deref(), ) .await?; Ok(Box::new(Self { pg_pool })) diff --git a/database/src/postgres/state_indexer.rs b/database/src/postgres/state_indexer.rs index ba1c8a1e..3c122c77 100644 --- a/database/src/postgres/state_indexer.rs +++ b/database/src/postgres/state_indexer.rs @@ -1,5 +1,4 @@ use crate::postgres::PostgresStorageManager; -use crate::AdditionalDatabaseOptions; use bigdecimal::ToPrimitive; pub struct PostgresDBManager { @@ -8,17 +7,12 @@ pub struct PostgresDBManager { #[async_trait::async_trait] impl crate::BaseDbManager for PostgresDBManager { - async fn new( - database_url: &str, - database_user: Option<&str>, - database_password: Option<&str>, - database_options: AdditionalDatabaseOptions, - ) -> anyhow::Result> { + async fn new(config: &configuration::DatabaseConfig) -> anyhow::Result> { let pg_pool = Self::create_pool( - database_url, - database_user, - database_password, - database_options, + &config.database_url, + config.database_user.as_deref(), + config.database_password.as_deref(), + config.database_name.as_deref(), ) .await?; Ok(Box::new(Self { pg_pool })) diff --git a/database/src/postgres/tx_indexer.rs b/database/src/postgres/tx_indexer.rs index 2243c240..f1fdea29 100644 --- a/database/src/postgres/tx_indexer.rs +++ b/database/src/postgres/tx_indexer.rs @@ -1,5 +1,4 @@ use crate::postgres::PostgresStorageManager; -use crate::AdditionalDatabaseOptions; use bigdecimal::ToPrimitive; use borsh::{BorshDeserialize, BorshSerialize}; @@ -9,17 +8,12 @@ pub struct PostgresDBManager { #[async_trait::async_trait] impl crate::BaseDbManager for PostgresDBManager { - async fn new( - database_url: &str, - database_user: Option<&str>, - database_password: Option<&str>, - database_options: AdditionalDatabaseOptions, - ) -> anyhow::Result> { + async fn new(config: &configuration::DatabaseConfig) -> anyhow::Result> { let pg_pool = Self::create_pool( - database_url, - database_user, - database_password, - database_options, + &config.database_url, + config.database_user.as_deref(), + config.database_password.as_deref(), + config.database_name.as_deref(), ) .await?; Ok(Box::new(Self { pg_pool })) diff --git a/database/src/scylladb/rpc_server.rs b/database/src/scylladb/rpc_server.rs index f0672327..ab7062ae 100644 --- a/database/src/scylladb/rpc_server.rs +++ b/database/src/scylladb/rpc_server.rs @@ -30,21 +30,16 @@ pub struct ScyllaDBManager { #[async_trait::async_trait] impl crate::BaseDbManager for ScyllaDBManager { - async fn new( - database_url: &str, - database_user: Option<&str>, - database_password: Option<&str>, - database_options: crate::AdditionalDatabaseOptions, - ) -> anyhow::Result> { + async fn new(config: &configuration::DatabaseConfig) -> anyhow::Result> { let scylla_db_session = std::sync::Arc::new( Self::get_scylladb_session( - database_url, - database_user, - database_password, - database_options.preferred_dc.as_deref(), - database_options.keepalive_interval, - database_options.max_retry, - database_options.strict_mode, + &config.database_url, + config.database_user.as_deref(), + config.database_password.as_deref(), + config.preferred_dc.as_deref(), + config.keepalive_interval, + config.max_retry, + config.strict_mode, ) .await?, ); diff --git a/database/src/scylladb/state_indexer.rs b/database/src/scylladb/state_indexer.rs index ae3da910..272aac5f 100644 --- a/database/src/scylladb/state_indexer.rs +++ b/database/src/scylladb/state_indexer.rs @@ -39,21 +39,16 @@ pub struct ScyllaDBManager { #[async_trait::async_trait] impl crate::BaseDbManager for ScyllaDBManager { - async fn new( - database_url: &str, - database_user: Option<&str>, - database_password: Option<&str>, - database_options: crate::AdditionalDatabaseOptions, - ) -> anyhow::Result> { + async fn new(config: &configuration::DatabaseConfig) -> anyhow::Result> { let scylla_db_session = std::sync::Arc::new( Self::get_scylladb_session( - database_url, - database_user, - database_password, - database_options.preferred_dc.as_deref(), - database_options.keepalive_interval, - database_options.max_retry, - database_options.strict_mode, + &config.database_url, + config.database_user.as_deref(), + config.database_password.as_deref(), + config.preferred_dc.as_deref(), + config.keepalive_interval, + config.max_retry, + config.strict_mode, ) .await?, ); diff --git a/database/src/scylladb/tx_indexer.rs b/database/src/scylladb/tx_indexer.rs index 2e0c3cde..08a37bd2 100644 --- a/database/src/scylladb/tx_indexer.rs +++ b/database/src/scylladb/tx_indexer.rs @@ -23,21 +23,16 @@ pub struct ScyllaDBManager { #[async_trait::async_trait] impl crate::BaseDbManager for ScyllaDBManager { - async fn new( - database_url: &str, - database_user: Option<&str>, - database_password: Option<&str>, - database_options: crate::AdditionalDatabaseOptions, - ) -> anyhow::Result> { + async fn new(config: &configuration::DatabaseConfig) -> anyhow::Result> { let scylla_db_session = std::sync::Arc::new( Self::get_scylladb_session( - database_url, - database_user, - database_password, - database_options.preferred_dc.as_deref(), - database_options.keepalive_interval, - database_options.max_retry, - database_options.strict_mode, + &config.database_url, + config.database_user.as_deref(), + config.database_password.as_deref(), + config.preferred_dc.as_deref(), + config.keepalive_interval, + config.max_retry, + config.strict_mode, ) .await?, ); diff --git a/docker-compose.yml b/docker-compose.yml index 2f161273..357cec99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,10 @@ version: '3.4' x-common-variables: &common-variables + CHAIN_ID: mainnet DATABASE_URL: scylla:9042 - DATABASE_USER: ${SCYLLA_USER} - DATABASE_PASSWORD: ${SCYLLA_PASSWORD} + DATABASE_USER: ${DATABASE_USER} + DATABASE_PASSWORD: ${DATABASE_PASSWORD} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION} @@ -21,15 +22,14 @@ services: context: . dockerfile: rpc-server/Dockerfile args: - features: "tracing-instrumentation,shadow_data_consistency" + features: "scylla_db,tracing-instrumentation,shadow_data_consistency" environment: <<: *common-variables - AWS_BUCKET_NAME: near-lake-data-mainnet - SERVER_PORT: 8080 - RUST_LOG: "read_rpc_server=debug,shadow_data_consistency=debug" + SERVER_PORT: 8000 + RUST_LOG: "read_rpc_server=debug,shadow_data_consistency=debug,info" restart: on-failure ports: - - 8080:8080 + - 8000:8000 depends_on: - scylla - jaeger @@ -42,8 +42,11 @@ services: features: tracing-instrumentation environment: <<: *common-variables - INDEXER_ID: state-indexer-local - command: [ "mainnet", "from-interruption" ] + STATE_INDEXER_ID: state-indexer-local + RUST_LOG: "info" + command: [ "from-interruption" ] + ports: + - 8080:8080 depends_on: - scylla - jaeger @@ -57,8 +60,11 @@ services: features: tracing-instrumentation environment: <<: *common-variables - INDEXER_ID: tx-indexer-local - command: [ "mainnet", "from-interruption" ] + TX_INDEXER_ID: tx-indexer-local + RUST_LOG: "info" + command: [ "from-interruption" ] + ports: + - 8081:8081 depends_on: - scylla - jaeger diff --git a/epoch-indexer/Cargo.toml b/epoch-indexer/Cargo.toml index 01cfc60e..989f8af3 100644 --- a/epoch-indexer/Cargo.toml +++ b/epoch-indexer/Cargo.toml @@ -4,12 +4,8 @@ version = "0.1.0" edition = "2021" [dependencies] -aws-credential-types = "1.0.0" -aws-sdk-s3 = { version = "0.39.1", features = ["behavior-version-latest"] } -aws-types = "1.0.0" anyhow = "1.0.70" clap = { version = "3.1.6", features = ["color", "derive", "env"] } -dotenv = "0.15.0" futures = "0.3.5" tokio = { version = "1.28.2", features = [ "sync", @@ -18,15 +14,8 @@ tokio = { version = "1.28.2", features = [ "rt-multi-thread", ] } tracing = "0.1.34" -tracing-subscriber = { version = "0.3.15", features = [ - "fmt", - "env-filter", - "std", - "json", -] } -opentelemetry = { version = "0.19", features = ["rt-tokio-current-thread"] } -tracing-opentelemetry = { version = "0.19" } +configuration = { path = "../configuration" } database = { path = "../database" } readnode-primitives = { path = "../readnode-primitives" } diff --git a/epoch-indexer/src/config.rs b/epoch-indexer/src/config.rs index aaf836a5..b6d1e569 100644 --- a/epoch-indexer/src/config.rs +++ b/epoch-indexer/src/config.rs @@ -10,66 +10,8 @@ use clap::{Parser, Subcommand}; setting(clap::AppSettings::NextLineHelp) )] pub struct Opts { - /// Indexer ID to handle meta data about the instance - #[clap(long, env)] - pub indexer_id: String, - // Indexer bucket name - #[clap(long, env = "AWS_BUCKET_NAME")] - pub s3_bucket_name: String, - // AWS access key id - #[clap(long, env = "AWS_ACCESS_KEY_ID")] - pub access_key_id: String, - // AWS secret access key - #[clap(long, env = "AWS_SECRET_ACCESS_KEY")] - pub secret_access_key: String, - // AWS default region - #[clap(long, env = "AWS_DEFAULT_REGION")] - pub region: String, - - /// DB connection string - #[clap(long, default_value = "127.0.0.1:9042", env)] - pub database_url: String, - /// DB user(login) - #[clap(long, env)] - pub database_user: Option, - /// DB password - #[clap(long, env)] - pub database_password: Option, - - #[clap(subcommand)] - pub chain_id: ChainId, - - /// ScyllaDB preferred DataCenter - /// Accepts the DC name of the ScyllaDB to filter the connection to that DC only (preferrably). - /// If you connect to multi-DC cluter, you might experience big latencies while working with the DB. This is due to the fact that ScyllaDB driver tries to connect to any of the nodes in the cluster disregarding of the location of the DC. This option allows to filter the connection to the DC you need. Example: "DC1" where DC1 is located in the same region as the application. - #[cfg(feature = "scylla_db")] - #[clap(long, env)] - pub preferred_dc: Option, - - /// Max retry count for ScyllaDB if `strict_mode` is `false` - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = 5)] - pub max_retry: u8, - - /// Attempts to store data in the database should be infinite to ensure no data is missing. - /// Disable it to perform a limited write attempts (`max_retry`) - /// before giving up, and moving to the next piece of data - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = true)] - pub strict_mode: bool, - - /// Postgres database name - #[cfg(feature = "postgres_db")] - #[clap(long, env)] - pub database_name: Option, -} - -#[derive(Subcommand, Debug, Clone)] -pub enum ChainId { #[clap(subcommand)] - Mainnet(StartOptions), - #[clap(subcommand)] - Testnet(StartOptions), + pub start_options: StartOptions, } #[allow(clippy::enum_variant_names)] @@ -78,50 +20,3 @@ pub enum StartOptions { FromGenesis, FromInterruption, } - -impl Opts { - pub fn start_options(&self) -> &StartOptions { - match &self.chain_id { - ChainId::Mainnet(start_options) | ChainId::Testnet(start_options) => start_options, - } - } - - pub async fn to_additional_database_options(&self) -> database::AdditionalDatabaseOptions { - database::AdditionalDatabaseOptions { - #[cfg(feature = "scylla_db")] - preferred_dc: self.preferred_dc.clone(), - #[cfg(feature = "scylla_db")] - keepalive_interval: None, - #[cfg(feature = "scylla_db")] - max_retry: self.max_retry, - #[cfg(feature = "scylla_db")] - strict_mode: self.strict_mode, - #[cfg(feature = "postgres_db")] - database_name: self.database_name.clone(), - } - } - - pub fn rpc_url(&self) -> &str { - match &self.chain_id { - ChainId::Mainnet(_) => "https://rpc.mainnet.near.org", - ChainId::Testnet(_) => "https://rpc.testnet.near.org", - } - } - pub async fn to_s3_client(&self) -> near_lake_framework::s3_fetchers::LakeS3Client { - let credentials = aws_credential_types::Credentials::new( - &self.access_key_id, - &self.secret_access_key, - None, - None, - "", - ); - let s3_config = aws_sdk_s3::Config::builder() - .credentials_provider(credentials) - .region(aws_types::region::Region::new(self.region.clone())) - .build(); - - near_lake_framework::s3_fetchers::LakeS3Client::new(aws_sdk_s3::Client::from_conf( - s3_config, - )) - } -} diff --git a/epoch-indexer/src/main.rs b/epoch-indexer/src/main.rs index fd842f39..895043b6 100644 --- a/epoch-indexer/src/main.rs +++ b/epoch-indexer/src/main.rs @@ -1,11 +1,12 @@ use crate::config::{Opts, StartOptions}; use clap::Parser; use database::StateIndexerDbManager; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; mod config; +// Categories for logging +pub(crate) const INDEXER: &str = "epoch_indexer"; + async fn index_epochs( s3_client: &near_lake_framework::s3_fetchers::LakeS3Client, s3_bucket_name: &str, @@ -47,80 +48,48 @@ async fn index_epochs( } } -pub(crate) fn init_tracing() -> anyhow::Result<()> { - let mut env_filter = tracing_subscriber::EnvFilter::new("epoch_indexer=info"); - - if let Ok(rust_log) = std::env::var("RUST_LOG") { - if !rust_log.is_empty() { - for directive in rust_log.split(',').filter_map(|s| match s.parse() { - Ok(directive) => Some(directive), - Err(err) => { - eprintln!("Ignoring directive `{}`: {}", s, err); - None - } - }) { - env_filter = env_filter.add_directive(directive); - } - } - } - - opentelemetry::global::shutdown_tracer_provider(); - - opentelemetry::global::set_text_map_propagator( - opentelemetry::sdk::propagation::TraceContextPropagator::new(), - ); - - let subscriber = tracing_subscriber::Registry::default().with(env_filter); - subscriber - .with(tracing_subscriber::fmt::Layer::default().compact()) - .try_init()?; - - Ok(()) -} - #[tokio::main] async fn main() -> anyhow::Result<()> { - dotenv::dotenv().ok(); + configuration::init_tracing(INDEXER).await?; + let indexer_config = + configuration::read_configuration::().await?; let opts: Opts = Opts::parse(); - init_tracing()?; - #[cfg(feature = "scylla_db")] - let db_manager = - database::prepare_db_manager::( - &opts.database_url, - opts.database_user.as_deref(), - opts.database_password.as_deref(), - opts.to_additional_database_options().await, - ) - .await?; + let db_manager = database::prepare_db_manager::< + database::scylladb::state_indexer::ScyllaDBManager, + >(&indexer_config.database) + .await?; #[cfg(all(feature = "postgres_db", not(feature = "scylla_db")))] - let db_manager = - database::prepare_db_manager::( - &opts.database_url, - opts.database_user.as_deref(), - opts.database_password.as_deref(), - opts.to_additional_database_options().await, - ) - .await?; + let db_manager = database::prepare_db_manager::< + database::postgres::state_indexer::PostgresDBManager, + >(&indexer_config.database) + .await?; - let s3_client = opts.to_s3_client().await; - let rpc_client = near_jsonrpc_client::JsonRpcClient::connect(opts.rpc_url()); + let indexer_id = &indexer_config.general.indexer_id; + let s3_client = indexer_config.lake_config.lake_s3_client().await; + let rpc_client = + near_jsonrpc_client::JsonRpcClient::connect(&indexer_config.general.near_rpc_url); - let epoch = match opts.start_options() { + let epoch = match opts.start_options { StartOptions::FromGenesis => { - epoch_indexer::first_epoch(&s3_client, &opts.s3_bucket_name, &rpc_client).await? + epoch_indexer::first_epoch( + &s3_client, + &indexer_config.lake_config.aws_bucket_name, + &rpc_client, + ) + .await? } StartOptions::FromInterruption => { let block_height = db_manager - .get_last_processed_block_height(opts.indexer_id.as_str()) + .get_last_processed_block_height(indexer_id) .await?; epoch_indexer::get_epoch_info_by_block_height( block_height, &s3_client, - &opts.s3_bucket_name, + &indexer_config.lake_config.aws_bucket_name, &rpc_client, ) .await? @@ -130,10 +99,10 @@ async fn main() -> anyhow::Result<()> { index_epochs( &s3_client, - &opts.s3_bucket_name, + &indexer_config.lake_config.aws_bucket_name, db_manager, rpc_client, - &opts.indexer_id, + indexer_id, epoch, ) .await?; diff --git a/rpc-server/Cargo.toml b/rpc-server/Cargo.toml index 574ec5d6..c7134690 100644 --- a/rpc-server/Cargo.toml +++ b/rpc-server/Cargo.toml @@ -8,17 +8,11 @@ actix-web = "4.2.1" actix-cors = "0.6.5" anyhow = "1.0.70" assert-json-diff = { version = "2.0.2", optional = true } -aws-credential-types = "1.0.0" -aws-sdk-s3 = { version = "0.39.1", features = ["behavior-version-latest"] } -aws-types = "1.0.0" borsh = "0.10.2" -clap = { version = "3.2.22", features = ["color", "derive", "env"] } -dotenv = "0.15.0" erased-serde = "0.3.23" futures = "0.3.24" futures-locks = "0.7.1" hex = "0.4.3" -http = "0.2.8" jsonrpc-v2 = { git = "https://github.com/kobayurii/jsonrpc-v2", rev = "95e7b1d2567ae841163af212a3f25abb6862becb" } lazy_static = "1.4.0" lru = "0.11.1" @@ -31,20 +25,10 @@ tokio = { version = "1.28.2", features = ["full", "tracing"] } tokio-stream = { version = "0.1.12" } tracing = { version = "0.1.36", features = ["std"] } tracing-actix-web = "0.6.1" -tracing-subscriber = { version = "0.3.15", features = [ - "fmt", - "env-filter", - "std", - "json", -] } -opentelemetry = { version = "0.17", features = ["rt-tokio-current-thread"] } -opentelemetry-jaeger = { version = "0.16", features = [ - "rt-tokio-current-thread", -] } sysinfo = "0.29.10" -tracing-opentelemetry = { version = "0.17" } -tracing-stackdriver = "0.7.2" # GCP logs + +configuration = { path = "../configuration" } database = { path = "../database" } readnode-primitives = { path = "../readnode-primitives" } @@ -60,7 +44,7 @@ near-vm-runner = "1.36.0" [features] default = ["send_tx_methods", "scylla_db"] send_tx_methods = [] -tracing-instrumentation = [] +tracing-instrumentation = ["configuration/tracing-instrumentation"] postgres_db = ["database/postgres_db"] scylla_db = ["database/scylla_db"] scylla_db_tracing = ["database/scylla_db_tracing", "scylla_db"] diff --git a/rpc-server/Dockerfile b/rpc-server/Dockerfile index 561ac13e..c0285f58 100644 --- a/rpc-server/Dockerfile +++ b/rpc-server/Dockerfile @@ -6,6 +6,7 @@ COPY Cargo.lock ./ COPY Cargo.toml ./ RUN sed '/perf-testing/d; /state-indexer/d; /tx-indexer/d; /epoch-indexer/d' Cargo.toml > Cargo.toml.new && mv Cargo.toml.new Cargo.toml COPY rpc-server/Cargo.toml rpc-server/Cargo.toml +COPY configuration configuration COPY database database COPY readnode-primitives readnode-primitives RUN mkdir rpc-server/src && echo 'fn main() {}' > rpc-server/src/main.rs cargo build --release && rm -r rpc-server/src @@ -17,6 +18,7 @@ COPY rpc-server/src ./rpc-server/src RUN cargo build --release --no-default-features --features "$features" FROM ubuntu:22.04 -RUN apt update && apt install -yy openssl ca-certificates +RUN apt update && apt install -yy openssl ca-certificates libcurl4 +COPY config.toml . COPY --from=builder /tmp/target/release/read-rpc-server . ENTRYPOINT ["./read-rpc-server"] diff --git a/rpc-server/src/config.rs b/rpc-server/src/config.rs index f379f9bb..5544afe9 100644 --- a/rpc-server/src/config.rs +++ b/rpc-server/src/config.rs @@ -1,152 +1,6 @@ use crate::modules::blocks::{CacheBlock, FinalBlockInfo}; -use clap::Parser; use futures::executor::block_on; -#[derive(Parser)] -#[clap(author, version, about, long_about = None)] -pub struct Opts { - // near network rpc url - #[clap(long, env = "NEAR_RPC_URL")] - pub rpc_url: http::Uri, - - // near network archival rpc url - #[clap(long, env = "ARCHIVAL_NEAR_RPC_URL")] - pub archival_rpc_url: Option, - - // Referer header value - // We want to set a custom referer to let NEAR JSON RPC nodes know that we are a read-rpc instance - #[clap(long, env = "REFERER_HEADER_VALUE", default_value = "read-rpc")] - pub referer_header_value: String, - - // Indexer bucket name - #[clap(long, env = "AWS_BUCKET_NAME")] - pub s3_bucket_name: String, - - // DB url - #[clap(long, env)] - pub database_url: String, - - /// DB user(login) - #[clap(long, env)] - pub database_user: Option, - - ///DB password - #[clap(long, env)] - pub database_password: Option, - - // AWS access key id - #[clap(long, env = "AWS_ACCESS_KEY_ID")] - pub access_key_id: String, - - // AWS secret access key - #[clap(long, env = "AWS_SECRET_ACCESS_KEY")] - pub secret_access_key: String, - - // AWS default region - #[clap(long, env = "AWS_DEFAULT_REGION")] - pub region: String, - - // AWS default region - #[clap(long, env, default_value_t = 8000)] - pub server_port: u16, - - /// Max gas burnt for contract function call - /// Default value is 300_000_000_000_000 - #[clap(long, env, default_value_t = 300_000_000_000_000)] - pub max_gas_burnt: near_primitives::types::Gas, - - /// Max available memory for `block_cache` and `contract_code_cache` in gigabytes - /// By default we use all available memory - #[clap(long, env)] - pub limit_memory_cache: Option, - - /// Reserved memory for running the application in gigabytes - /// By default we use 0.25 gigabyte (256MB or 268_435_456 bytes) - #[clap(long, env, default_value_t = 0.25)] - pub reserved_memory: f64, - - /// Block cache size in gigabytes - /// By default we use 0.125 gigabyte (128MB or 134_217_728 bytes) - /// One cache_block size is ≈ 96 bytes - /// In 128MB we can put 1_398_101 cache_blocks - #[clap(long, env, default_value_t = 0.125)] - pub block_cache_size: f64, - - /// ScyllaDB preferred DataCenter - /// Accepts the DC name of the ScyllaDB to filter the connection to that DC only (preferrably). - /// If you connect to multi-DC cluter, you might experience big latencies while working with the DB. This is due to the fact that ScyllaDB driver tries to connect to any of the nodes in the cluster disregarding of the location of the DC. This option allows to filter the connection to the DC you need. Example: "DC1" where DC1 is located in the same region as the application. - #[cfg(feature = "scylla_db")] - #[clap(long, env)] - pub preferred_dc: Option, - - /// ScyllaDB keepalive interval - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = 60)] - pub keepalive_interval: u64, - - /// Max retry count for ScyllaDB if `strict_mode` is `false` - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = 2)] - pub max_retry: u8, - - /// Attempts to store data in the database should be infinite to ensure no data is missing. - /// Disable it to perform a limited write attempts (`max_retry`) - /// before giving up, and moving to the next piece of data - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = false)] - pub strict_mode: bool, - - /// Postgres database name - #[cfg(feature = "postgres_db")] - #[clap(long, env)] - pub database_name: Option, -} - -impl Opts { - pub async fn to_additional_database_options(&self) -> database::AdditionalDatabaseOptions { - database::AdditionalDatabaseOptions { - #[cfg(feature = "scylla_db")] - preferred_dc: self.preferred_dc.clone(), - #[cfg(feature = "scylla_db")] - keepalive_interval: Some(self.keepalive_interval), - #[cfg(feature = "scylla_db")] - max_retry: self.max_retry, - #[cfg(feature = "scylla_db")] - strict_mode: self.strict_mode, - #[cfg(feature = "postgres_db")] - database_name: self.database_name.clone(), - } - } - - pub async fn to_s3_config(&self) -> aws_sdk_s3::Config { - let credentials = aws_credential_types::Credentials::new( - &self.access_key_id, - &self.secret_access_key, - None, - None, - "", - ); - aws_sdk_s3::Config::builder() - .credentials_provider(credentials) - .region(aws_types::region::Region::new(self.region.clone())) - .build() - } - - pub async fn to_lake_config( - &self, - start_block_height: near_primitives::types::BlockHeight, - ) -> anyhow::Result { - let config_builder = near_lake_framework::LakeConfigBuilder::default(); - Ok(config_builder - .s3_config(self.to_s3_config().await) - .s3_region_name(&self.region) - .s3_bucket_name(&self.s3_bucket_name) - .start_block_height(start_block_height) - .build() - .expect("Failed to build LakeConfig")) - } -} - pub struct ServerContext { pub s3_client: near_lake_framework::s3_fetchers::LakeS3Client, pub db_manager: std::sync::Arc>, diff --git a/rpc-server/src/main.rs b/rpc-server/src/main.rs index 0ec0f829..dffb4c89 100644 --- a/rpc-server/src/main.rs +++ b/rpc-server/src/main.rs @@ -1,13 +1,6 @@ use crate::modules::blocks::FinalBlockInfo; -use crate::utils::{ - get_final_cache_block, gigabytes_to_bytes, update_final_block_height_regularly, -}; -use clap::Parser; -use config::{Opts, ServerContext}; -use dotenv::dotenv; +use crate::utils::{gigabytes_to_bytes, update_final_block_height_regularly}; use jsonrpc_v2::{Data, Server}; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; #[macro_use] extern crate lazy_static; @@ -19,79 +12,35 @@ mod metrics; mod modules; mod utils; -fn init_logging(use_tracer: bool) -> anyhow::Result<()> { - // Filter based on level - trace, debug, info, warn, error - // Tunable via `RUST_LOG` env variable - let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or(tracing_subscriber::EnvFilter::new("info")); - - // Combined them all together in a `tracing` subscriber - let subscriber = tracing_subscriber::Registry::default().with(env_filter); - - if use_tracer { - let app_name = "read_rpc_server"; - // Start a new Jaeger trace pipeline. - // Spans are exported in batch - recommended setup for a production application. - opentelemetry::global::set_text_map_propagator( - opentelemetry::sdk::propagation::TraceContextPropagator::new(), - ); - let tracer = opentelemetry_jaeger::new_pipeline() - .with_service_name(app_name) - .install_simple()?; - // Create a `tracing` layer using the Jaeger tracer - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - if std::env::var("ENABLE_JSON_LOGS").is_ok() { - subscriber - .with(telemetry) - .with(tracing_subscriber::fmt::Layer::default().json()) - .try_init()?; - } else { - subscriber - .with(telemetry) - .with(tracing_subscriber::fmt::Layer::default().compact()) - .try_init()?; - }; - } else if std::env::var("ENABLE_JSON_LOGS").is_ok() { - subscriber.with(tracing_stackdriver::layer()).try_init()?; - } else { - subscriber - .with(tracing_subscriber::fmt::Layer::default().compact()) - .try_init()?; - }; - - Ok(()) -} +// Categories for logging +pub(crate) const RPC_SERVER: &str = "read_rpc_server"; #[tokio::main] async fn main() -> anyhow::Result<()> { - dotenv().ok(); + configuration::init_tracing(RPC_SERVER).await?; + let rpc_server_config = + configuration::read_configuration::().await?; - let opts: Opts = Opts::parse(); - - #[cfg(feature = "tracing-instrumentation")] - init_logging(true)?; - - #[cfg(not(feature = "tracing-instrumentation"))] - init_logging(false)?; - - let near_rpc_client = - utils::JsonRpcClient::new(opts.rpc_url.clone(), opts.archival_rpc_url.clone()); + let near_rpc_client = utils::JsonRpcClient::new( + rpc_server_config.general.near_rpc_url.clone(), + rpc_server_config.general.near_archival_rpc_url.clone(), + ); // We want to set a custom referer to let NEAR JSON RPC nodes know that we are a read-rpc instance - let near_rpc_client = - near_rpc_client.header("Referer".to_string(), opts.referer_header_value.clone())?; - - let final_block = get_final_cache_block(&near_rpc_client) - .await - .expect("Error to get final block"); - - let limit_memory_cache_in_bytes = if let Some(limit_memory_cache) = opts.limit_memory_cache { - Some(gigabytes_to_bytes(limit_memory_cache).await) - } else { - None - }; - let reserved_memory_in_bytes = gigabytes_to_bytes(opts.reserved_memory).await; - let block_cache_size_in_bytes = gigabytes_to_bytes(opts.block_cache_size).await; + let near_rpc_client = near_rpc_client.header( + "Referer".to_string(), + rpc_server_config.general.referer_header_value.clone(), + )?; + + let limit_memory_cache_in_bytes = + if let Some(limit_memory_cache) = rpc_server_config.general.limit_memory_cache { + Some(gigabytes_to_bytes(limit_memory_cache).await) + } else { + None + }; + let reserved_memory_in_bytes = + gigabytes_to_bytes(rpc_server_config.general.reserved_memory).await; + let block_cache_size_in_bytes = + gigabytes_to_bytes(rpc_server_config.general.block_cache_size).await; let contract_code_cache_size = utils::calculate_contract_code_cache_sizes( reserved_memory_in_bytes, @@ -121,42 +70,42 @@ async fn main() -> anyhow::Result<()> { let genesis_config = near_rpc_client .call(near_jsonrpc_client::methods::EXPERIMENTAL_genesis_config::RpcGenesisConfigRequest) .await?; - let lake_config = opts.to_lake_config(final_block.block_height).await?; - let s3_config = opts.to_s3_config().await; + let lake_config = rpc_server_config + .lake_config + .lake_config( + finale_block_info + .read() + .await + .final_block_cache + .block_height, + ) + .await?; + let lake_s3_client = rpc_server_config.lake_config.lake_s3_client().await; #[cfg(feature = "scylla_db")] let db_manager = database::prepare_db_manager::( - opts.database_url.as_str(), - opts.database_user.as_deref(), - opts.database_password.as_deref(), - opts.to_additional_database_options().await, + &rpc_server_config.database, ) .await?; #[cfg(all(feature = "postgres_db", not(feature = "scylla_db")))] - let db_manager = - database::prepare_db_manager::( - opts.database_url.as_str(), - opts.database_user.as_deref(), - opts.database_password.as_deref(), - opts.to_additional_database_options().await, - ) - .await?; + let db_manager = database::prepare_db_manager::< + database::postgres::rpc_server::PostgresDBManager, + >(&rpc_server_config.database) + .await?; - let state = ServerContext::new( - near_lake_framework::s3_fetchers::LakeS3Client::new(aws_sdk_s3::Client::from_conf( - s3_config, - )), + let state = config::ServerContext::new( + lake_s3_client, db_manager, near_rpc_client.clone(), - opts.s3_bucket_name.clone(), + rpc_server_config.lake_config.aws_bucket_name.clone(), genesis_config, std::sync::Arc::clone(&blocks_cache), std::sync::Arc::clone(&finale_block_info), compiled_contract_code_cache, contract_code_cache, - opts.max_gas_burnt, + rpc_server_config.general.max_gas_burnt, ); tokio::spawn(async move { @@ -242,7 +191,10 @@ async fn main() -> anyhow::Result<()> { ) .service(metrics::get_metrics) }) - .bind(format!("0.0.0.0:{:0>5}", opts.server_port))? + .bind(format!( + "0.0.0.0:{:0>5}", + rpc_server_config.general.server_port + ))? .run() .await?; diff --git a/rpc-server/src/metrics.rs b/rpc-server/src/metrics.rs index 563320d5..9cfa8065 100644 --- a/rpc-server/src/metrics.rs +++ b/rpc-server/src/metrics.rs @@ -593,11 +593,8 @@ pub(crate) async fn get_metrics() -> impl Responder { tracing::error!("could not encode metrics: {}", e); }; - match String::from_utf8(buffer.clone()) { - Ok(v) => v, - Err(e) => { - tracing::error!("custom metrics could not be from_utf8'd: {}", e); - String::default() - } - } + String::from_utf8(buffer.clone()).unwrap_or_else(|err| { + tracing::error!("custom metrics could not be from_utf8'd: {}", err); + String::default() + }) } diff --git a/rpc-server/src/utils.rs b/rpc-server/src/utils.rs index 53cfd5ec..2a03e285 100644 --- a/rpc-server/src/utils.rs +++ b/rpc-server/src/utils.rs @@ -9,7 +9,7 @@ const DEFAULT_RETRY_COUNT: u8 = 3; /// JsonRpcClient represents a client capable of interacting with NEAR JSON-RPC endpoints, /// The client is capable of handling requests to both regular and archival nodes. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct JsonRpcClient { regular_client: near_jsonrpc_client::JsonRpcClient, archival_client: near_jsonrpc_client::JsonRpcClient, @@ -19,12 +19,10 @@ impl JsonRpcClient { /// Creates a new JsonRpcClient. /// The client is capable of handling requests to both regular and archival nodes. /// If the `archival_rpc_url` is not provided, the client will use the regular endpoint for both - pub fn new(rpc_url: http::Uri, archival_rpc_url: Option) -> Self { - let regular_client = near_jsonrpc_client::JsonRpcClient::connect(rpc_url.to_string()); + pub fn new(rpc_url: String, archival_rpc_url: Option) -> Self { + let regular_client = near_jsonrpc_client::JsonRpcClient::connect(rpc_url); let archival_client = match archival_rpc_url { - Some(archival_rpc_url) => { - near_jsonrpc_client::JsonRpcClient::connect(archival_rpc_url.to_string()) - } + Some(archival_rpc_url) => near_jsonrpc_client::JsonRpcClient::connect(archival_rpc_url), None => regular_client.clone(), }; Self { @@ -110,7 +108,10 @@ pub async fn get_final_cache_block(near_rpc_client: &JsonRpcClient) -> Option None, + Err(err) => { + tracing::warn!("Error to get final block: {:?}", err); + None + } } } diff --git a/state-indexer/Cargo.toml b/state-indexer/Cargo.toml index 73dcc35f..1a17800a 100644 --- a/state-indexer/Cargo.toml +++ b/state-indexer/Cargo.toml @@ -6,12 +6,9 @@ edition = "2021" [dependencies] actix-web = "4.2.1" -aws-sdk-s3 = { version = "0.39.1", features = ["behavior-version-latest"] } -aws-types = "1.0.0" anyhow = "1.0.70" borsh = "0.10.2" -clap = { version = "3.1.6", features = ["color", "derive", "env"] } -dotenv = "0.15.0" +clap = { version = "3.1.6", features = ["color", "derive"] } futures = "0.3.5" hex = "0.4.3" humantime = "2.1.0" @@ -26,21 +23,7 @@ tokio = { version = "1.28.2", features = [ ] } tokio-stream = { version = "0.1" } tracing = "0.1.34" -tracing-subscriber = { version = "0.3.15", features = [ - "fmt", - "env-filter", - "std", - "json", -] } -opentelemetry = { version = "0.19", features = ["rt-tokio-current-thread"] } -opentelemetry-jaeger = { version = "0.18", features = [ - "rt-tokio-current-thread", - "collector_client", - "isahc_collector_client", -] } -tracing-opentelemetry = { version = "0.19" } -tracing-stackdriver = "0.7.2" # GCP logs - +configuration = { path = "../configuration" } database = { path = "../database" } epoch-indexer = { path = "../epoch-indexer" } @@ -51,7 +34,7 @@ near-lake-framework = "0.7.6" [features] default = ["scylla_db"] -tracing-instrumentation = [] +tracing-instrumentation = ["configuration/tracing-instrumentation"] postgres_db = ["database/postgres_db"] scylla_db = ["database/scylla_db"] scylla_db_tracing = ["database/scylla_db_tracing", "scylla_db"] diff --git a/state-indexer/Dockerfile b/state-indexer/Dockerfile index 812e1cd6..29e45da4 100644 --- a/state-indexer/Dockerfile +++ b/state-indexer/Dockerfile @@ -6,6 +6,7 @@ COPY Cargo.lock ./ COPY Cargo.toml ./ RUN sed '/perf-testing/d; /rpc-server/d; /tx-indexer/d' Cargo.toml > Cargo.toml.new && mv Cargo.toml.new Cargo.toml COPY state-indexer/Cargo.toml state-indexer/Cargo.toml +COPY configuration configuration COPY database database COPY readnode-primitives readnode-primitives COPY epoch-indexer epoch-indexer @@ -18,6 +19,7 @@ COPY state-indexer/src ./state-indexer/src RUN cargo build --release --features "$features" FROM ubuntu:22.04 -RUN apt update && apt install -yy openssl ca-certificates +RUN apt update && apt install -yy openssl ca-certificates libcurl4 +COPY config.toml . COPY --from=builder /tmp/target/release/state-indexer . ENTRYPOINT ["./state-indexer"] diff --git a/state-indexer/README.md b/state-indexer/README.md index 7da56018..f3fed364 100644 --- a/state-indexer/README.md +++ b/state-indexer/README.md @@ -23,10 +23,8 @@ See the [documentation here](../database/README.md) ### Command to run ``` -cargo run --release -- +cargo run --release -- ``` - -- `chain_id` (\*) `testnet` or `mainnet` - `start_options`: - `from-latest` fetches the final block height from the RPC and starts indexing from that block - `from-interruption ` is used to retrieve the `last_processed_block_height` from the Scylla database. This value is used as the starting point for processing blocks. If a specific value `` is provided, it will be used as the fallback option. If `` is not provided or if the database does not have a record (for example, in the case of a fresh start with an empty storage), the fallback option will be `from-latest`. diff --git a/state-indexer/src/configs.rs b/state-indexer/src/configs.rs index dccf4a12..6fc720f5 100644 --- a/state-indexer/src/configs.rs +++ b/state-indexer/src/configs.rs @@ -2,9 +2,6 @@ pub use clap::{Parser, Subcommand}; use near_jsonrpc_client::{methods, JsonRpcClient}; use near_lake_framework::near_indexer_primitives::types::{BlockReference, Finality}; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; - /// NEAR Indexer for Explorer /// Watches for stream of blocks from the chain #[derive(Parser, Debug)] @@ -17,58 +14,8 @@ use tracing_subscriber::util::SubscriberInitExt; setting(clap::AppSettings::NextLineHelp) )] pub(crate) struct Opts { - /// Indexer ID to handle meta data about the instance - #[clap(long, env)] - pub indexer_id: String, - /// DB connection string - #[clap(long, default_value = "127.0.0.1:9042", env)] - pub database_url: String, - /// DB user(login) - #[clap(long, env)] - pub database_user: Option, - /// DB password - #[clap(long, env)] - pub database_password: Option, - /// Metrics HTTP server port - #[clap(long, default_value = "8080", env)] - pub port: u16, - #[clap(long, default_value = "1", env)] - pub concurrency: usize, - /// Chain ID: testnet or mainnet #[clap(subcommand)] - pub chain_id: ChainId, - - /// ScyllaDB preferred DataCenter - /// Accepts the DC name of the ScyllaDB to filter the connection to that DC only (preferrably). - /// If you connect to multi-DC cluter, you might experience big latencies while working with the DB. This is due to the fact that ScyllaDB driver tries to connect to any of the nodes in the cluster disregarding of the location of the DC. This option allows to filter the connection to the DC you need. Example: "DC1" where DC1 is located in the same region as the application. - #[cfg(feature = "scylla_db")] - #[clap(long, env)] - pub preferred_dc: Option, - - /// Max retry count for ScyllaDB if `strict_mode` is `false` - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = 5)] - pub max_retry: u8, - - /// Attempts to store data in the database should be infinite to ensure no data is missing. - /// Disable it to perform a limited write attempts (`max_retry`) - /// before giving up, and moving to the next piece of data - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = true)] - pub strict_mode: bool, - - /// Postgres database name - #[cfg(feature = "postgres_db")] - #[clap(long, env)] - pub database_name: Option, -} - -#[derive(Subcommand, Debug, Clone)] -pub enum ChainId { - #[clap(subcommand)] - Mainnet(StartOptions), - #[clap(subcommand)] - Testnet(StartOptions), + pub start_options: StartOptions, } #[allow(clippy::enum_variant_names)] @@ -84,141 +31,35 @@ pub enum StartOptions { FromLatest, } -impl Opts { - /// Returns [StartOptions] for current [Opts] - pub fn start_options(&self) -> &StartOptions { - match &self.chain_id { - ChainId::Mainnet(start_options) | ChainId::Testnet(start_options) => start_options, - } - } - - pub fn rpc_url(&self) -> &str { - match &self.chain_id { - ChainId::Mainnet(_) => "https://rpc.mainnet.near.org", - ChainId::Testnet(_) => "https://rpc.testnet.near.org", - } - } -} - -impl Opts { - pub async fn to_additional_database_options(&self) -> database::AdditionalDatabaseOptions { - database::AdditionalDatabaseOptions { - #[cfg(feature = "scylla_db")] - preferred_dc: self.preferred_dc.clone(), - #[cfg(feature = "scylla_db")] - keepalive_interval: None, - #[cfg(feature = "scylla_db")] - max_retry: self.max_retry, - #[cfg(feature = "scylla_db")] - strict_mode: self.strict_mode, - #[cfg(feature = "postgres_db")] - database_name: self.database_name.clone(), - } - } - - pub async fn to_lake_config( - &self, - db_manager: &(impl database::StateIndexerDbManager + Sync + Send + 'static), - ) -> anyhow::Result { - let config_builder = near_lake_framework::LakeConfigBuilder::default(); - - Ok(match &self.chain_id { - ChainId::Mainnet(_) => config_builder - .mainnet() - .start_block_height(get_start_block_height(self, db_manager).await?), - ChainId::Testnet(_) => config_builder - .testnet() - .start_block_height(get_start_block_height(self, db_manager).await?), - } - .build() - .expect("Failed to build LakeConfig")) - } -} - -async fn get_start_block_height( - opts: &Opts, +pub async fn get_start_block_height( + rpc_client: &JsonRpcClient, db_manager: &(impl database::StateIndexerDbManager + Sync + Send + 'static), + start_options: &StartOptions, + indexer_id: &str, ) -> anyhow::Result { - match opts.start_options() { + match start_options { StartOptions::FromBlock { height } => Ok(*height), StartOptions::FromInterruption { height } => { - if let Ok(block_height) = db_manager - .get_last_processed_block_height(opts.indexer_id.as_str()) - .await - { + if let Ok(block_height) = db_manager.get_last_processed_block_height(indexer_id).await { Ok(block_height) } else { if let Some(height) = height { return Ok(*height); } - Ok(final_block_height(opts.rpc_url()).await?) + Ok(final_block_height(rpc_client).await?) } } - StartOptions::FromLatest => Ok(final_block_height(opts.rpc_url()).await?), + StartOptions::FromLatest => Ok(final_block_height(rpc_client).await?), } } -pub(crate) async fn final_block_height(rpc_url: &str) -> anyhow::Result { +pub(crate) async fn final_block_height(rpc_client: &JsonRpcClient) -> anyhow::Result { tracing::debug!(target: crate::INDEXER, "Fetching final block from NEAR RPC",); - let client = JsonRpcClient::connect(rpc_url); let request = methods::block::RpcBlockRequest { block_reference: BlockReference::Finality(Finality::Final), }; - let latest_block = client.call(request).await?; + let latest_block = rpc_client.call(request).await?; Ok(latest_block.header.height) } - -pub(crate) fn init_tracing() -> anyhow::Result<()> { - let mut env_filter = tracing_subscriber::EnvFilter::new("state_indexer=info"); - - if let Ok(rust_log) = std::env::var("RUST_LOG") { - if !rust_log.is_empty() { - for directive in rust_log.split(',').filter_map(|s| match s.parse() { - Ok(directive) => Some(directive), - Err(err) => { - eprintln!("Ignoring directive `{}`: {}", s, err); - None - } - }) { - env_filter = env_filter.add_directive(directive); - } - } - } - - opentelemetry::global::shutdown_tracer_provider(); - - opentelemetry::global::set_text_map_propagator(opentelemetry::sdk::propagation::TraceContextPropagator::new()); - - #[cfg(feature = "tracing-instrumentation")] - let subscriber = { - let tracer = opentelemetry_jaeger::new_collector_pipeline() - .with_service_name("state_indexer") - .with_endpoint(std::env::var("OTEL_EXPORTER_JAEGER_ENDPOINT").unwrap_or_default()) - .with_isahc() - .with_batch_processor_config( - opentelemetry::sdk::trace::BatchConfig::default() - .with_max_queue_size(10_000) - .with_max_export_batch_size(10_000) - .with_max_concurrent_exports(100), - ) - .install_batch(opentelemetry::runtime::TokioCurrentThread)?; - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - tracing_subscriber::Registry::default().with(env_filter).with(telemetry) - }; - - #[cfg(not(feature = "tracing-instrumentation"))] - let subscriber = tracing_subscriber::Registry::default().with(env_filter); - - if std::env::var("ENABLE_JSON_LOGS").is_ok() { - subscriber.with(tracing_stackdriver::layer()).try_init()?; - } else { - subscriber - .with(tracing_subscriber::fmt::Layer::default().compact()) - .try_init()?; - } - - Ok(()) -} diff --git a/state-indexer/src/main.rs b/state-indexer/src/main.rs index 16b403cc..4b5050f0 100644 --- a/state-indexer/src/main.rs +++ b/state-indexer/src/main.rs @@ -18,13 +18,13 @@ pub(crate) const INDEXER: &str = "state_indexer"; #[cfg_attr( feature = "tracing-instrumentation", - tracing::instrument(skip(streamer_message, db_manager, indexer_id)) + tracing::instrument(skip(streamer_message, db_manager)) )] async fn handle_streamer_message( streamer_message: near_indexer_primitives::StreamerMessage, db_manager: &(impl database::StateIndexerDbManager + Sync + Send + 'static), rpc_client: &near_jsonrpc_client::JsonRpcClient, - indexer_id: &str, + indexer_config: configuration::StateIndexerConfig, stats: std::sync::Arc>, ) -> anyhow::Result<()> { let block_height = streamer_message.block.header.height; @@ -56,9 +56,10 @@ async fn handle_streamer_message( .collect(), db_manager, ); - let handle_state_change_future = handle_state_changes(streamer_message, db_manager, block_height, block_hash); + let handle_state_change_future = + handle_state_changes(streamer_message, db_manager, block_height, block_hash, &indexer_config); - let update_meta_future = db_manager.update_meta(indexer_id, block_height); + let update_meta_future = db_manager.update_meta(&indexer_config.general.indexer_id, block_height); futures::try_join!(handle_epoch_future, handle_block_future, handle_state_change_future, update_meta_future)?; @@ -145,6 +146,7 @@ async fn handle_state_changes( db_manager: &(impl database::StateIndexerDbManager + Sync + Send + 'static), block_height: u64, block_hash: CryptoHash, + indexer_config: &configuration::StateIndexerConfig, ) -> anyhow::Result> { let mut state_changes_to_store = std::collections::HashMap::::new(); @@ -157,6 +159,9 @@ async fn handle_state_changes( // Collecting a unique list of StateChangeWithCauseView for account_id + change kind + suffix // by overwriting the records in the HashMap for state_change in initial_state_changes { + if !indexer_config.state_should_be_indexed(&state_change.value) { + continue; + }; let key = match &state_change.value { StateChangeValueView::DataUpdate { account_id, key, .. } | StateChangeValueView::DataDeletion { account_id, key } => { @@ -283,41 +288,41 @@ async fn store_state_change( async fn main() -> anyhow::Result<()> { // We use it to automatically search the for root certificates to perform HTTPS calls // (sending telemetry and downloading genesis) - dotenv::dotenv().ok(); - openssl_probe::init_ssl_cert_env_vars(); - configs::init_tracing()?; - + configuration::init_tracing(INDEXER).await?; + let indexer_config = configuration::read_configuration::().await?; let opts: Opts = Opts::parse(); #[cfg(feature = "scylla_db")] - let db_manager = database::prepare_db_manager::( - &opts.database_url, - opts.database_user.as_deref(), - opts.database_password.as_deref(), - opts.to_additional_database_options().await, - ) - .await?; + let db_manager = + database::prepare_db_manager::(&indexer_config.database) + .await?; #[cfg(all(feature = "postgres_db", not(feature = "scylla_db")))] - let db_manager = database::prepare_db_manager::( - &opts.database_url, - opts.database_user.as_deref(), - opts.database_password.as_deref(), - opts.to_additional_database_options().await, + let db_manager = + database::prepare_db_manager::(&indexer_config.database) + .await?; + + let rpc_client = near_jsonrpc_client::JsonRpcClient::connect(&indexer_config.general.near_rpc_url); + let start_block_height = configs::get_start_block_height( + &rpc_client, + &db_manager, + &opts.start_options, + &indexer_config.general.indexer_id, ) .await?; - let config: near_lake_framework::LakeConfig = opts.to_lake_config(&db_manager).await?; - let (sender, stream) = near_lake_framework::streamer(config); + let lake_config = indexer_config.lake_config.lake_config(start_block_height).await?; + let (sender, stream) = near_lake_framework::streamer(lake_config); - let rpc_client = near_jsonrpc_client::JsonRpcClient::connect(opts.rpc_url()); // Initiate metrics http server - tokio::spawn(metrics::init_server(opts.port).expect("Failed to start metrics server")); + tokio::spawn( + metrics::init_server(indexer_config.general.metrics_server_port).expect("Failed to start metrics server"), + ); let stats = std::sync::Arc::new(tokio::sync::RwLock::new(metrics::Stats::new())); - tokio::spawn(metrics::state_logger(std::sync::Arc::clone(&stats), opts.rpc_url().to_string())); + tokio::spawn(metrics::state_logger(std::sync::Arc::clone(&stats), rpc_client.clone())); let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) .map(|streamer_message| { @@ -325,11 +330,11 @@ async fn main() -> anyhow::Result<()> { streamer_message, &db_manager, &rpc_client, - &opts.indexer_id, + indexer_config.clone(), std::sync::Arc::clone(&stats), ) }) - .buffer_unordered(opts.concurrency); + .buffer_unordered(indexer_config.general.concurrency); while let Some(_handle_message) = handlers.next().await { if let Err(err) = _handle_message { diff --git a/state-indexer/src/metrics.rs b/state-indexer/src/metrics.rs index 15535d1e..464917f8 100644 --- a/state-indexer/src/metrics.rs +++ b/state-indexer/src/metrics.rs @@ -1,4 +1,5 @@ use actix_web::{get, App, HttpServer, Responder}; +use near_jsonrpc_client::JsonRpcClient; use prometheus::{Encoder, IntCounter, IntGauge, Opts}; type Result = std::result::Result; @@ -78,7 +79,7 @@ impl Stats { } } -pub async fn state_logger(stats: std::sync::Arc>, rpc_url: String) { +pub async fn state_logger(stats: std::sync::Arc>, rpc_client: JsonRpcClient) { let interval_secs = 10; let mut prev_blocks_processed_count: u64 = 0; @@ -90,7 +91,7 @@ pub async fn state_logger(stats: std::sync::Arc>, rpc ((stats_lock.blocks_processed_count - prev_blocks_processed_count) as f64) / (interval_secs as f64); let time_to_catch_the_tip_duration = if block_processing_speed > 0.0 { - if let Ok(block_height) = crate::configs::final_block_height(&rpc_url).await { + if let Ok(block_height) = crate::configs::final_block_height(&rpc_client).await { Some(std::time::Duration::from_millis( (((block_height - stats_lock.last_processed_block_height) as f64 / block_processing_speed) * 1000f64) as u64, diff --git a/tx-indexer/Cargo.toml b/tx-indexer/Cargo.toml index 4e33b58a..849e923d 100644 --- a/tx-indexer/Cargo.toml +++ b/tx-indexer/Cargo.toml @@ -7,11 +7,7 @@ edition = "2021" actix-web = "4.2.1" anyhow = "1.0.70" async-trait = "0.1.66" -aws-credential-types = "1.0.0" -aws-sdk-s3 = { version = "0.39.1", features = ["behavior-version-latest"] } -aws-types = "1.0.0" clap = { version = "3.2.22", features = ["color", "derive", "env"] } -dotenv = "0.15.0" futures = "0.3.5" futures-locks = "0.7.1" humantime = "2.1.0" @@ -25,21 +21,8 @@ tokio = { version = "1.28.2", features = [ ] } tokio-stream = { version = "0.1.12" } tracing = "0.1.34" -tracing-subscriber = { version = "0.3.15", features = [ - "fmt", - "env-filter", - "std", - "json", -] } -opentelemetry = { version = "0.19", features = ["rt-tokio-current-thread"] } -opentelemetry-jaeger = { version = "0.18", features = [ - "rt-tokio-current-thread", - "collector_client", - "isahc_collector_client", -] } -tracing-opentelemetry = { version = "0.19" } -tracing-stackdriver = "0.7.2" # GCP logs +configuration = { path = "../configuration" } database = { path = "../database" } readnode-primitives = { path = "../readnode-primitives" } @@ -49,7 +32,7 @@ near-lake-framework = "0.7.6" [features] default = ["scylla_db"] -tracing-instrumentation = [] +tracing-instrumentation = ["configuration/tracing-instrumentation"] postgres_db = ["database/postgres_db"] scylla_db = ["database/scylla_db"] scylla_db_tracing = ["database/scylla_db_tracing", "scylla_db"] diff --git a/tx-indexer/Dockerfile b/tx-indexer/Dockerfile index 9c0a1083..4229e1f2 100644 --- a/tx-indexer/Dockerfile +++ b/tx-indexer/Dockerfile @@ -6,6 +6,7 @@ COPY Cargo.lock ./ COPY Cargo.toml ./ RUN sed '/perf-testing/d; /rpc-server/d; /state-indexer/d; /epoch-indexer/d' Cargo.toml > Cargo.toml.new && mv Cargo.toml.new Cargo.toml COPY tx-indexer/Cargo.toml tx-indexer/Cargo.toml +COPY configuration configuration COPY database database COPY readnode-primitives readnode-primitives RUN mkdir tx-indexer/src && echo 'fn main() {}' > tx-indexer/src/main.rs cargo build --release && rm -r tx-indexer/src @@ -18,5 +19,6 @@ RUN cargo build --release --features "$features" FROM ubuntu:22.04 RUN apt update && apt install -yy openssl ca-certificates libcurl4 +COPY config.toml . COPY --from=builder /tmp/target/release/tx-indexer . ENTRYPOINT ["./tx-indexer"] diff --git a/tx-indexer/README.md b/tx-indexer/README.md index fe4ed7ab..74dd4aec 100644 --- a/tx-indexer/README.md +++ b/tx-indexer/README.md @@ -53,10 +53,8 @@ See the [documentation here](../database/README.md) ### Command to run ``` -cargo run --release -- +cargo run --release -- ``` - -- `chain_id` (\*) `testnet` or `mainnet` - `start_options`: - `from-latest` fetches the final block height from the RPC and starts indexing from that block - `from-interruption ` is used to retrieve the `last_processed_block_height` from the Scylla database. This value is used as the starting point for processing blocks. If a specific value `` is provided, it will be used as the fallback option. If `` is not provided or if the database does not have a record (for example, in the case of a fresh start with an empty storage), the fallback option will be `from-latest`. diff --git a/tx-indexer/src/collector.rs b/tx-indexer/src/collector.rs index 3d811a9a..97914eb9 100644 --- a/tx-indexer/src/collector.rs +++ b/tx-indexer/src/collector.rs @@ -1,12 +1,10 @@ +use crate::storage::base::TxCollectingStorage; use futures::{ future::{join_all, try_join_all}, StreamExt, }; use near_indexer_primitives::IndexerTransactionWithOutcome; -use crate::config; -use crate::storage::base::TxCollectingStorage; - /// Blocks #47317863 and #47317864 with restored receipts. const PROBLEMATIC_BLOCKS: [near_indexer_primitives::CryptoHash; 2] = [ near_indexer_primitives::CryptoHash( @@ -21,14 +19,20 @@ const PROBLEMATIC_BLOCKS: [near_indexer_primitives::CryptoHash; 2] = [ #[cfg_attr(feature = "tracing-instrumentation", tracing::instrument(skip_all))] pub(crate) async fn index_transactions( - chain_id: config::ChainId, streamer_message: &near_indexer_primitives::StreamerMessage, db_manager: &std::sync::Arc>, tx_collecting_storage: &std::sync::Arc, + indexer_config: &configuration::TxIndexerConfig, ) -> anyhow::Result<()> { - extract_transactions_to_collect(streamer_message, db_manager, tx_collecting_storage).await?; + extract_transactions_to_collect( + streamer_message, + db_manager, + tx_collecting_storage, + indexer_config, + ) + .await?; collect_receipts_and_outcomes( - chain_id, + &indexer_config.general.chain_id, streamer_message, db_manager, tx_collecting_storage, @@ -58,6 +62,7 @@ async fn extract_transactions_to_collect( streamer_message: &near_indexer_primitives::StreamerMessage, db_manager: &std::sync::Arc>, tx_collecting_storage: &std::sync::Arc, + indexer_config: &configuration::TxIndexerConfig, ) -> anyhow::Result<()> { let block_height = streamer_message.block.header.height; @@ -74,6 +79,7 @@ async fn extract_transactions_to_collect( shard_id, db_manager, tx_collecting_storage, + indexer_config, ) }) }); @@ -90,7 +96,12 @@ async fn new_transaction_details_to_collecting_pool( shard_id: u64, db_manager: &std::sync::Arc>, tx_collecting_storage: &std::sync::Arc, + indexer_config: &configuration::TxIndexerConfig, ) -> anyhow::Result<()> { + if !indexer_config.tx_should_be_indexed(transaction) { + return Ok(()); + }; + let converted_into_receipt_id = transaction .outcome .execution_outcome @@ -133,7 +144,7 @@ async fn new_transaction_details_to_collecting_pool( #[cfg_attr(feature = "tracing-instrumentation", tracing::instrument(skip_all))] async fn collect_receipts_and_outcomes( - chain_id: config::ChainId, + chain_id: &configuration::ChainId, streamer_message: &near_indexer_primitives::StreamerMessage, db_manager: &std::sync::Arc>, tx_collecting_storage: &std::sync::Arc, @@ -143,7 +154,7 @@ async fn collect_receipts_and_outcomes( let shard_futures = streamer_message.shards.iter().map(|shard| { process_shard( - chain_id.clone(), + chain_id, db_manager, tx_collecting_storage, block_height, @@ -159,7 +170,7 @@ async fn collect_receipts_and_outcomes( #[cfg_attr(feature = "tracing-instrumentation", tracing::instrument(skip_all))] async fn process_shard( - chain_id: config::ChainId, + chain_id: &configuration::ChainId, db_manager: &std::sync::Arc>, tx_collecting_storage: &std::sync::Arc, block_height: u64, @@ -172,7 +183,7 @@ async fn process_shard( .iter() .map(|receipt_execution_outcome| { process_receipt_execution_outcome( - chain_id.clone(), + chain_id, db_manager, tx_collecting_storage, block_height, @@ -189,7 +200,7 @@ async fn process_shard( #[cfg_attr(feature = "tracing-instrumentation", tracing::instrument(skip_all))] async fn process_receipt_execution_outcome( - chain_id: config::ChainId, + chain_id: &configuration::ChainId, db_manager: &std::sync::Arc>, tx_collecting_storage: &std::sync::Arc, block_height: u64, @@ -198,7 +209,7 @@ async fn process_receipt_execution_outcome( receipt_execution_outcome: &near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, ) -> anyhow::Result<()> { if PROBLEMATIC_BLOCKS.contains(&block_hash) { - if let config::ChainId::Mainnet(_) = chain_id { + if let configuration::ChainId::Mainnet = chain_id { tx_collecting_storage .restore_transaction_by_receipt_id( &receipt_execution_outcome.receipt.receipt_id.to_string(), diff --git a/tx-indexer/src/config.rs b/tx-indexer/src/config.rs index 824acaf2..cdf13039 100644 --- a/tx-indexer/src/config.rs +++ b/tx-indexer/src/config.rs @@ -1,8 +1,6 @@ pub use clap::{Parser, Subcommand}; use near_indexer_primitives::types::{BlockReference, Finality}; use near_jsonrpc_client::{methods, JsonRpcClient}; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; /// NEAR Indexer for Explorer /// Watches for stream of blocks from the chain @@ -16,67 +14,8 @@ use tracing_subscriber::util::SubscriberInitExt; setting(clap::AppSettings::NextLineHelp) )] pub(crate) struct Opts { - /// Indexer ID to handle meta data about the instance - #[clap(long, env)] - pub indexer_id: String, - /// Port for metrics server - #[clap(long, default_value = "8080", env)] - pub port: u16, - /// DB connection string - #[clap(long, default_value = "127.0.0.1:9042", env)] - pub database_url: String, - /// DB user(login) - #[clap(long, env)] - pub database_user: Option, - /// DB password - #[clap(long, env)] - pub database_password: Option, - /// Chain ID: testnet or mainnet #[clap(subcommand)] - pub chain_id: ChainId, - /// To restore cache from db we use smart range blocks - /// Regular transaction takes some blocks to be finalized - /// We don't need to restore too old transactions for the indexer because we will probably never be able to reassemble them. - /// We use a range of 1000 blocks for our peace of mind. We also leave the option to increase or decrease this range - #[clap(long, default_value = "1000", env)] - pub cache_restore_blocks_range: u64, - /// Parallel queries = (nodes in cluster) ✕ (cores in node) ✕ 3 - /// Current we have 6 - nodes with 8 - cpus - /// 6 ✕ 8 ✕ 3 = 144 - #[clap(long, env, default_value = "144")] - pub max_db_parallel_queries: i64, - - /// ScyllaDB preferred DataCenter - /// Accepts the DC name of the ScyllaDB to filter the connection to that DC only (preferrably). - /// If you connect to multi-DC cluter, you might experience big latencies while working with the DB. This is due to the fact that ScyllaDB driver tries to connect to any of the nodes in the cluster disregarding of the location of the DC. This option allows to filter the connection to the DC you need. Example: "DC1" where DC1 is located in the same region as the application. - #[cfg(feature = "scylla_db")] - #[clap(long, env)] - pub preferred_dc: Option, - - /// Max retry count for ScyllaDB if `strict_mode` is `false` - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = 5)] - pub max_retry: u8, - - /// Attempts to store data in the database should be infinite to ensure no data is missing. - /// Disable it to perform a limited write attempts (`max_retry`) - /// before giving up, and moving to the next piece of data - #[cfg(feature = "scylla_db")] - #[clap(long, env, default_value_t = true)] - pub strict_mode: bool, - - /// Postgres database name - #[cfg(feature = "postgres_db")] - #[clap(long, env)] - pub database_name: Option, -} - -#[derive(Subcommand, Debug, Clone)] -pub enum ChainId { - #[clap(subcommand)] - Mainnet(StartOptions), - #[clap(subcommand)] - Testnet(StartOptions), + pub start_options: StartOptions, } #[allow(clippy::enum_variant_names)] @@ -92,144 +31,34 @@ pub enum StartOptions { FromLatest, } -impl Opts { - /// Returns [StartOptions] for current [Opts] - pub fn start_options(&self) -> &StartOptions { - match &self.chain_id { - ChainId::Mainnet(start_options) | ChainId::Testnet(start_options) => start_options, - } - } - - pub fn rpc_url(&self) -> &str { - match self.chain_id { - ChainId::Mainnet(_) => "https://rpc.mainnet.near.org", - ChainId::Testnet(_) => "https://rpc.testnet.near.org", - } - } -} - -impl Opts { - pub async fn to_additional_database_options(&self) -> database::AdditionalDatabaseOptions { - database::AdditionalDatabaseOptions { - #[cfg(feature = "scylla_db")] - preferred_dc: self.preferred_dc.clone(), - #[cfg(feature = "scylla_db")] - keepalive_interval: None, - #[cfg(feature = "scylla_db")] - max_retry: self.max_retry, - #[cfg(feature = "scylla_db")] - strict_mode: self.strict_mode, - #[cfg(feature = "postgres_db")] - database_name: self.database_name.clone(), - } - } - - pub async fn to_lake_config( - &self, - start_block_height: u64, - ) -> anyhow::Result { - let config_builder = near_lake_framework::LakeConfigBuilder::default(); - - Ok(match &self.chain_id { - ChainId::Mainnet(_) => config_builder - .mainnet() - .start_block_height(start_block_height), - ChainId::Testnet(_) => config_builder - .testnet() - .start_block_height(start_block_height), - } - .build() - .expect("Failed to build LakeConfig")) - } -} - pub(crate) async fn get_start_block_height( - opts: &Opts, + rpc_client: &JsonRpcClient, db_manager: &std::sync::Arc>, + start_options: &StartOptions, + indexer_id: &str, ) -> anyhow::Result { - match opts.start_options() { + match start_options { StartOptions::FromBlock { height } => Ok(*height), StartOptions::FromInterruption { height } => { - if let Ok(block_height) = db_manager - .get_last_processed_block_height(opts.indexer_id.as_str()) - .await - { + if let Ok(block_height) = db_manager.get_last_processed_block_height(indexer_id).await { Ok(block_height) } else { if let Some(height) = height { return Ok(*height); } - Ok(final_block_height(opts.rpc_url()).await?) + Ok(final_block_height(rpc_client).await?) } } - StartOptions::FromLatest => Ok(final_block_height(opts.rpc_url()).await?), + StartOptions::FromLatest => Ok(final_block_height(rpc_client).await?), } } -pub async fn final_block_height(rpc_url: &str) -> anyhow::Result { - let client = JsonRpcClient::connect(rpc_url.to_string()); +pub async fn final_block_height(rpc_client: &JsonRpcClient) -> anyhow::Result { let request = methods::block::RpcBlockRequest { block_reference: BlockReference::Finality(Finality::Final), }; - let latest_block = client.call(request).await?; + let latest_block = rpc_client.call(request).await?; Ok(latest_block.header.height) } - -pub fn init_tracing() -> anyhow::Result<()> { - let mut env_filter = tracing_subscriber::EnvFilter::new("tx_indexer=info"); - - if let Ok(rust_log) = std::env::var("RUST_LOG") { - if !rust_log.is_empty() { - for directive in rust_log.split(',').filter_map(|s| match s.parse() { - Ok(directive) => Some(directive), - Err(err) => { - eprintln!("Ignoring directive `{}`: {}", s, err); - None - } - }) { - env_filter = env_filter.add_directive(directive); - } - } - } - - opentelemetry::global::shutdown_tracer_provider(); - - opentelemetry::global::set_text_map_propagator( - opentelemetry::sdk::propagation::TraceContextPropagator::new(), - ); - - #[cfg(feature = "tracing-instrumentation")] - let subscriber = { - let tracer = opentelemetry_jaeger::new_collector_pipeline() - .with_service_name("tx_indexer") - .with_endpoint(std::env::var("OTEL_EXPORTER_JAEGER_ENDPOINT").unwrap_or_default()) - .with_isahc() - .with_batch_processor_config( - opentelemetry::sdk::trace::BatchConfig::default() - .with_max_queue_size(10_000) - .with_max_export_batch_size(10_000) - .with_max_concurrent_exports(100), - ) - .install_batch(opentelemetry::runtime::TokioCurrentThread)?; - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - tracing_subscriber::Registry::default() - .with(env_filter) - .with(telemetry) - }; - - #[cfg(not(feature = "tracing-instrumentation"))] - let subscriber = tracing_subscriber::Registry::default().with(env_filter); - - if std::env::var("ENABLE_JSON_LOGS").is_ok() { - subscriber.with(tracing_stackdriver::layer()).try_init()?; - } else { - subscriber - .with(tracing_subscriber::fmt::Layer::default().compact()) - .try_init()?; - } - - Ok(()) -} diff --git a/tx-indexer/src/main.rs b/tx-indexer/src/main.rs index 6c970e57..9ce6734b 100644 --- a/tx-indexer/src/main.rs +++ b/tx-indexer/src/main.rs @@ -1,4 +1,3 @@ -use crate::config::{init_tracing, Opts}; use clap::Parser; use futures::StreamExt; mod collector; @@ -13,11 +12,11 @@ pub(crate) const INDEXER: &str = "tx_indexer"; #[tokio::main] async fn main() -> anyhow::Result<()> { - dotenv::dotenv().ok(); + configuration::init_tracing(INDEXER).await?; + let indexer_config = + configuration::read_configuration::().await?; - init_tracing()?; - - let opts: Opts = Opts::parse(); + let opts = config::Opts::parse(); tracing::info!(target: INDEXER, "Connecting to db..."); #[cfg(feature = "scylla_db")] @@ -25,10 +24,7 @@ async fn main() -> anyhow::Result<()> { Box, > = std::sync::Arc::new(Box::new( database::prepare_db_manager::( - &opts.database_url, - opts.database_user.as_deref(), - opts.database_password.as_deref(), - opts.to_additional_database_options().await, + &indexer_config.database, ) .await?, )); @@ -37,50 +33,61 @@ async fn main() -> anyhow::Result<()> { Box, > = std::sync::Arc::new(Box::new( database::prepare_db_manager::( - &opts.database_url, - opts.database_user.as_deref(), - opts.database_password.as_deref(), - opts.to_additional_database_options().await, + &indexer_config.database, ) .await?, )); - let start_block_height = config::get_start_block_height(&opts, &db_manager).await?; + let rpc_client = + near_jsonrpc_client::JsonRpcClient::connect(&indexer_config.general.near_rpc_url); + let start_block_height = config::get_start_block_height( + &rpc_client, + &db_manager, + &opts.start_options, + &indexer_config.general.indexer_id, + ) + .await?; + tracing::info!(target: INDEXER, "Generating LakeConfig..."); - let config: near_lake_framework::LakeConfig = opts.to_lake_config(start_block_height).await?; + let lake_config = indexer_config + .lake_config + .lake_config(start_block_height) + .await?; tracing::info!(target: INDEXER, "Creating hash storage..."); let tx_collecting_storage = std::sync::Arc::new( storage::database::HashStorageWithDB::init_with_restore( db_manager.clone(), start_block_height, - opts.cache_restore_blocks_range, - opts.max_db_parallel_queries, + indexer_config.general.cache_restore_blocks_range, + indexer_config.database.max_db_parallel_queries, ) .await?, ); tracing::info!(target: INDEXER, "Instantiating the stream...",); - let (sender, stream) = near_lake_framework::streamer(config); + let (sender, stream) = near_lake_framework::streamer(lake_config); // Initiate metrics http server - tokio::spawn(metrics::init_server(opts.port).expect("Failed to start metrics server")); + tokio::spawn( + metrics::init_server(indexer_config.general.metrics_server_port) + .expect("Failed to start metrics server"), + ); let stats = std::sync::Arc::new(tokio::sync::RwLock::new(metrics::Stats::new())); tokio::spawn(metrics::state_logger( std::sync::Arc::clone(&stats), - opts.rpc_url().to_string(), + rpc_client.clone(), )); tracing::info!(target: INDEXER, "Starting tx indexer...",); let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) .map(|streamer_message| { handle_streamer_message( - opts.chain_id.clone(), streamer_message, &db_manager, &tx_collecting_storage, - &opts.indexer_id, + indexer_config.clone(), std::sync::Arc::clone(&stats), ) }) @@ -103,11 +110,10 @@ async fn main() -> anyhow::Result<()> { #[cfg_attr(feature = "tracing-instrumentation", tracing::instrument(skip_all))] async fn handle_streamer_message( - chain_id: config::ChainId, streamer_message: near_indexer_primitives::StreamerMessage, db_manager: &std::sync::Arc>, tx_collecting_storage: &std::sync::Arc, - indexer_id: &str, + indexer_config: configuration::TxIndexerConfig, stats: std::sync::Arc>, ) -> anyhow::Result { let block_height = streamer_message.block.header.height; @@ -120,14 +126,16 @@ async fn handle_streamer_message( .insert(block_height); let tx_future = collector::index_transactions( - chain_id, &streamer_message, db_manager, tx_collecting_storage, + &indexer_config, ); - let update_meta_future = - db_manager.update_meta(indexer_id, streamer_message.block.header.height); + let update_meta_future = db_manager.update_meta( + &indexer_config.general.indexer_id, + streamer_message.block.header.height, + ); match futures::try_join!(tx_future, update_meta_future) { Ok(_) => tracing::debug!( diff --git a/tx-indexer/src/metrics.rs b/tx-indexer/src/metrics.rs index 1fc11830..6d71f768 100644 --- a/tx-indexer/src/metrics.rs +++ b/tx-indexer/src/metrics.rs @@ -1,4 +1,5 @@ use actix_web::{get, App, HttpServer, Responder}; +use near_jsonrpc_client::JsonRpcClient; use prometheus::{Encoder, IntCounter, IntGauge, Opts}; type Result = std::result::Result; @@ -81,7 +82,10 @@ impl Stats { } } -pub async fn state_logger(stats: std::sync::Arc>, rpc_url: String) { +pub async fn state_logger( + stats: std::sync::Arc>, + rpc_client: JsonRpcClient, +) { let interval_secs = 10; let mut prev_blocks_processed_count: u64 = 0; @@ -94,7 +98,7 @@ pub async fn state_logger(stats: std::sync::Arc>, rpc / (interval_secs as f64); let time_to_catch_the_tip_duration = if block_processing_speed > 0.0 { - if let Ok(block_height) = crate::config::final_block_height(&rpc_url).await { + if let Ok(block_height) = crate::config::final_block_height(&rpc_client).await { Some(std::time::Duration::from_millis( (((block_height - stats_lock.last_processed_block_height) as f64 / block_processing_speed) diff --git a/tx-indexer/src/storage/database.rs b/tx-indexer/src/storage/database.rs index 91aed07a..4a7ddf59 100644 --- a/tx-indexer/src/storage/database.rs +++ b/tx-indexer/src/storage/database.rs @@ -188,15 +188,17 @@ impl HashStorageWithDB { #[async_trait::async_trait] impl TxCollectingStorage for HashStorageWithDB { async fn restore_transaction_by_receipt_id(&self, receipt_id: &str) -> anyhow::Result<()> { - let transaction_details = self + if let Ok(transaction_details) = self .db_manager .get_transaction_by_receipt_id(receipt_id) + .await + { + self.restore_transaction_with_receipts( + &transaction_details.transaction_key(), + &transaction_details, + ) .await?; - self.restore_transaction_with_receipts( - &transaction_details.transaction_key(), - &transaction_details, - ) - .await?; + } Ok(()) }