diff --git a/Cargo.lock b/Cargo.lock index b5368a3a06..b00e6de4ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -616,9 +616,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-lc-rs" -version = "1.12.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ea835662a0af02443aa1396d39be523bbf8f11ee6fad20329607c480bea48c3" +checksum = "4c2b7ddaa2c56a367ad27a094ad8ef4faacf8a617c2575acb2ba88949df999ca" dependencies = [ "aws-lc-sys", "paste", @@ -1037,7 +1037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", - "regex-automata 0.4.7", + "regex-automata 0.4.9", "serde", ] @@ -1275,9 +1275,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.52" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" +checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" dependencies = [ "cc", ] @@ -2422,6 +2422,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.30" @@ -2529,7 +2535,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util 0.7.11", @@ -2548,7 +2554,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.2.0", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util 0.7.11", @@ -3386,9 +3392,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -3799,7 +3805,7 @@ dependencies = [ "hyper 1.5.2", "hyper-rustls 0.27.5", "hyper-util", - "indexmap 2.7.0", + "indexmap 2.7.1", "ipnet", "metrics 0.24.1", "metrics-util 0.19.0", @@ -3830,7 +3836,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1ada651cd6bdffe01e5f35067df53491f1fe853d2b154008ca2bd30b3d3fcf6" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "lockfree-object-pool", "metrics 0.24.1", @@ -3851,7 +3857,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.15.2", - "indexmap 2.7.0", + "indexmap 2.7.1", "metrics 0.24.1", "ordered-float", "quanta", @@ -4569,7 +4575,7 @@ dependencies = [ "rand_chacha", "rand_core", "regex", - "reqwest 0.12.12", + "reqwest 0.12.9", "rocksdb", "rustls 0.23.21", "serde", @@ -4968,7 +4974,7 @@ dependencies = [ "rand", "rand_core", "regex", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_json", "sha2 0.10.8", @@ -5513,7 +5519,7 @@ dependencies = [ "rand", "rand_core", "regex", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_json", "sha2 0.10.8", @@ -6039,7 +6045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.7.0", + "indexmap 2.7.1", ] [[package]] @@ -6079,10 +6085,12 @@ name = "pindexer" version = "1.0.0" dependencies = [ "anyhow", + "assert_cmd", "chrono", "clap", "cometindex", "ethnum", + "http 1.2.0", "num-bigint", "penumbra-sdk-app", "penumbra-sdk-asset", @@ -6097,11 +6105,16 @@ dependencies = [ "penumbra-sdk-sct", "penumbra-sdk-shielded-pool", "penumbra-sdk-stake", + "predicates 2.1.5", "prost 0.13.4", + "prost-reflect", + "reqwest 0.12.9", + "rstest", "serde_json", "sqlx", "tokio", "tracing", + "url", ] [[package]] @@ -6371,11 +6384,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.21.1", + "toml_edit 0.22.22", ] [[package]] @@ -6489,7 +6502,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "rusty-fork", "tempfile", "unarray", @@ -6574,9 +6587,9 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.14.4" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9647f03b808b79abca8408b1609be9887ba90453c940d00332a60eeb6f5748" +checksum = "e92b959d24e05a3e2da1d0beb55b48bc8a97059b8336ea617780bd6addbbfb5a" dependencies = [ "once_cell", "prost 0.13.4", @@ -6624,9 +6637,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2 1.0.93", ] @@ -6812,14 +6825,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] @@ -6833,13 +6846,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -6850,9 +6863,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" @@ -6897,9 +6916,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.12" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", @@ -6932,7 +6951,6 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-util 0.7.11", - "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -7032,6 +7050,36 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rstest" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e905296805ab93e13c1ec3a03f4b6c4f35e9498a3d5fa96dc626d22c03cd89" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef0053bbffce09062bee4bcc499b0fbe7a57b879f1efe088d6d8d4c7adcdef9b" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2 1.0.93", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.96", + "unicode-ident", +] + [[package]] name = "rtoolbox" version = "0.0.2" @@ -7076,9 +7124,9 @@ checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] @@ -7424,7 +7472,7 @@ version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "ryu", "serde", @@ -7453,9 +7501,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -7501,7 +7549,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_derive", "serde_json", @@ -7752,7 +7800,7 @@ dependencies = [ "hashbrown 0.14.5", "hashlink", "hex", - "indexmap 2.7.0", + "indexmap 2.7.1", "log", "memchr", "once_cell", @@ -8584,7 +8632,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", @@ -8600,14 +8648,14 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.16", + "toml_edit 0.22.22", ] [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -8618,7 +8666,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", @@ -8627,26 +8675,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.7.0", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.16" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.24", + "winnow 0.6.25", ] [[package]] @@ -8967,9 +9004,9 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-normalization" @@ -9520,9 +9557,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.24" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +checksum = "ad699df48212c6cc6eb4435f35500ac6fd3b9913324f938aea302022ce19d310" dependencies = [ "memchr", ] diff --git a/crates/bin/pindexer/Cargo.toml b/crates/bin/pindexer/Cargo.toml index 8fe7fd59f7..124cab351c 100644 --- a/crates/bin/pindexer/Cargo.toml +++ b/crates/bin/pindexer/Cargo.toml @@ -12,6 +12,10 @@ publish = false [package.metadata.dist] dist = true +[features] +default = [] +network-integration = [] + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] @@ -39,3 +43,13 @@ tracing = {workspace = true} tokio = {workspace = true, features = ["full"]} serde_json = {workspace = true} sqlx = { workspace = true, features = ["chrono", "postgres"] } + +[dev-dependencies] +url = {workspace = true} +assert_cmd = { workspace = true } +predicates = "2.1" +prost-reflect = "0.14.3" +# TODO: move reqwest to workspace dependency +reqwest= { version = "0.12.9", features = ["json", "stream"] } +http = { workspace = true } +rstest = "0.24.0" diff --git a/crates/bin/pindexer/tests/network_integration.rs b/crates/bin/pindexer/tests/network_integration.rs new file mode 100644 index 0000000000..ed98b41cdd --- /dev/null +++ b/crates/bin/pindexer/tests/network_integration.rs @@ -0,0 +1,166 @@ +#![cfg(feature = "network-integration")] +//! Basic integration testing for pindexer. +//! +//! Performs queries against a local db to confirm core functionality, +//! such as all blocks are indexed. The setup and creation of the local db is not +//! managed by the Rust code. Use the nix env to ensure all relevant software +//! is installed. The `just smoke` target will run this logic under the proper settings. + +use anyhow::Context; +use rstest::rstest; +use sqlx::postgres::PgPoolOptions; +use sqlx::{PgPool, Row}; +use tokio::time::{sleep, Duration}; + +/// Hardcoded URL to the local PostgreSQL database for CometBFT ABCI events. +const COMETBFT_DATABASE_URL: &str = + "postgresql://penumbra:penumbra@localhost:5432/penumbra_cometbft?sslmode=disable"; + +/// Hardcoded URL to the local PostgreSQL database for pindexer. +const PINDEXER_DATABASE_URL: &str = + "postgresql://penumbra:penumbra@localhost:5432/penumbra_pindexer?sslmode=disable"; + +/// Reusable fn to connect to target postgres db, based on DATABASE_URL connection string. +async fn get_db_handle(database_url: &str) -> anyhow::Result { + Ok(PgPoolOptions::new() + .max_connections(5) + .connect(database_url) + .await?) +} + +/// Poll the CometBFT instance for its notion of current block height. +async fn get_current_height() -> anyhow::Result { + let client = reqwest::Client::new(); + let cmt_url = + std::env::var("PENUMBRA_NODE_CMT_URL").unwrap_or("http://localhost:26657".to_string()); + let r = client.get(format!("{}/status", cmt_url)).send().await?; + + assert_eq!(r.status(), reqwest::StatusCode::OK); + + let current_height: u64 = r + .json::() + .await? + .get_mut("result") + .and_then(|v| v.get_mut("sync_info")) + .and_then(|v| v.get_mut("latest_block_height")) + .ok_or_else(|| anyhow::anyhow!("could not parse block height from cometbft json"))? + .take() + .as_str() + .expect("could not find height in cometbft response") + .parse()?; + Ok(current_height) +} + +/// Query the CometBFT PostgreSQL database, and report its highest known block. +async fn get_highest_indexed_block_from_cometbft_db() -> anyhow::Result { + let conn = get_db_handle(COMETBFT_DATABASE_URL).await?; + + // Execute query and parse result as u64 + let row = sqlx::query("SELECT height FROM blocks ORDER BY height DESC LIMIT 1") + .fetch_one(&conn) + .await + .context("failed to get results from cometbft db; is postgres indexing configured?")?; + + let result: i64 = row.get(0); + let height: u64 = result.try_into()?; + + Ok(height) +} + +/// Query the pindexer PostgreSQL database, and report its highest known block. +async fn get_highest_indexed_block_from_pindexer_db() -> anyhow::Result { + let conn = get_db_handle(PINDEXER_DATABASE_URL).await?; + + // Execute query and parse result as u64 + let row = sqlx::query("SELECT height FROM block_details ORDER BY height DESC LIMIT 1") + .fetch_one(&conn) + .await + .context("failed to get results from pindexer db; did pindexer crash?")?; + + let result: i64 = row.get(0); + let height: u64 = result.try_into()?; + + Ok(height) +} + +#[tokio::test] +/// Confirm that the devnet chain is creating new blocks. Sanity-check, +/// so that our assumptions about the events in the database are grounded. +async fn chain_is_progressing() -> anyhow::Result<()> { + let height_1 = get_current_height().await?; + assert!(height_1 > 0, "initial height is not greater than 0!"); + // The default block time is ~5s, so we'll wait a bit longer than that, to ensure the chain + // has progressed by at least 1 block. + sleep(Duration::from_secs(7)).await; + + let height_2 = get_current_height().await?; + assert!( + height_2 > height_1, + "second height is not greater than initial height" + ); + Ok(()) +} + +#[tokio::test] +/// Confirm that the highest height reported by the CometBFT RPC +/// matches what's reported by the database. +async fn cometbft_indexing_is_working() -> anyhow::Result<()> { + let height_rpc = get_current_height().await?; + let height_db: u64 = get_highest_indexed_block_from_cometbft_db().await?; + assert!( + vec![height_rpc, height_rpc - 1].contains(&height_db), + "cometbft database is behind chain; is indexing broken?" + ); + Ok(()) +} + +#[rstest] +/// Database queries for the CometBFT event database, checking for null values. +/// All of these queries should return zero records in a well-formed database. +#[case("SELECT COUNT(*) FROM block_events WHERE key IS NULL;")] +#[case("SELECT COUNT(*) FROM block_events WHERE value IS NULL;")] +#[case("SELECT COUNT(*) FROM block_events WHERE composite_key IS NULL;")] +#[case("SELECT COUNT(*) FROM event_attributes WHERE key IS NULL;")] +#[case("SELECT COUNT(*) FROM event_attributes WHERE value IS NULL;")] +#[case("SELECT COUNT(*) FROM event_attributes WHERE composite_key IS NULL;")] +#[tokio::test] +/// Assert structure of cometbft database. Even if we assume cometbft will +/// "do the right thing" with ABCI event data it receives from pd, we cannot +/// be sure that pd is in fact emitting properly constructed ABCI Events +/// unless we check what's in the db. +async fn cometbft_events_are_not_null(#[case] query: &str) -> anyhow::Result<()> { + let conn = get_db_handle(COMETBFT_DATABASE_URL).await?; + + // Execute query and parse result as u64 + let row = sqlx::query(query) + .fetch_one(&conn) + .await + .context("failed to get results from cometbft db")?; + + let result: i64 = row.get(0); + let count: u64 = result.try_into()?; + assert_eq!( + count, 0, + "found {} null keys in cometbft event db, via query: '{}'", + count, query + ); + Ok(()) +} + +#[tokio::test] +/// Confirm that the highest height reported by the pindexer db +/// matches what's in the cometbft db. +async fn pindexer_is_working() -> anyhow::Result<()> { + let height_rpc = get_current_height().await?; + let height_db: u64 = get_highest_indexed_block_from_pindexer_db().await?; + // Check that pindexer's height is within tolerance. We allow lagging by ~2 blocks because + // local devnets and integration test suites run with faster block times, ~1s vs the default + // 5s, so being a bit behind is quite possible. + assert!( + vec![height_rpc, height_rpc - 1, height_rpc - 2].contains(&height_db), + "pindexer database is behind chain ({} vs {}); is indexing broken?", + height_db, + height_rpc + ); + Ok(()) +} diff --git a/deployments/compose/process-compose-postgres.yml b/deployments/compose/process-compose-postgres.yml index b106a832c1..d97054d9cd 100644 --- a/deployments/compose/process-compose-postgres.yml +++ b/deployments/compose/process-compose-postgres.yml @@ -74,3 +74,8 @@ processes: depends_on: postgresql-init: condition: process_completed_successfully + # Make sure fullnode is running, otherwise database may be empty + pd: + condition: process_healthy + cometbft: + condition: process_healthy