feat: impl quick indexing (#23)

* chore: rename type utils to utils

* feat: impl quick indexer poc

* feat: initial impl of the db and repo

* feat: add batch handling for the quick indexer

* fix: sql string not generated correctly

- fix issue where attempting to return the bounded sql statement fails
  as it loses the bind values. Done via passing tx context. Does make it
  less testable without sqlx.
- add debug config for vscode

* chore: fix clippy formatting

* chore: relax tracing for now to allow easier debug

* chore: fix clippy lints

* feat: impl batch indexing (#25)

* feat: impl batch indexing

* chore: fix linting and formatting

* fix: propagate rpc error encountered during index

* chore: fix formatting

* chore: remove redundant utils

* chore: fix formatting

.vscode/launch.json

@@ -0,0 +1,101 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug unit tests in library 'fossil_headers_db'",
+            "cargo": {
+                "args": [
+                    "test",
+                    "--no-run",
+                    "--lib",
+                    "--package=fossil_headers_db"
+                ],
+                "filter": {
+                    "name": "fossil_headers_db",
+                    "kind": "lib"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug executable 'fossil_headers_db'",
+            "cargo": {
+                "args": [
+                    "build",
+                    "--bin=fossil_headers_db",
+                    "--package=fossil_headers_db"
+                ],
+                "filter": {
+                    "name": "fossil_headers_db",
+                    "kind": "bin"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug unit tests in executable 'fossil_headers_db'",
+            "cargo": {
+                "args": [
+                    "test",
+                    "--no-run",
+                    "--bin=fossil_headers_db",
+                    "--package=fossil_headers_db"
+                ],
+                "filter": {
+                    "name": "fossil_headers_db",
+                    "kind": "bin"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug executable 'fossil_indexer'",
+            "cargo": {
+                "args": [
+                    "build",
+                    "--bin=fossil_indexer",
+                    "--package=fossil_headers_db"
+                ],
+                "filter": {
+                    "name": "fossil_indexer",
+                    "kind": "bin"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug unit tests in executable 'fossil_indexer'",
+            "cargo": {
+                "args": [
+                    "test",
+                    "--no-run",
+                    "--bin=fossil_indexer",
+                    "--package=fossil_headers_db"
+                ],
+                "filter": {
+                    "name": "fossil_indexer",
+                    "kind": "bin"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}

Cargo.toml

@@ -2,6 +2,7 @@
 name = "fossil_headers_db"
 version = "0.1.0"
 edition = "2021"
+default-run = "fossil_headers_db"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -22,9 +23,19 @@ sqlx = { version = "0.8.2", features = [
     "tls-rustls",
     "macros",
 ] }
+chrono = { version = "0.4.38", features = ["serde"] }
 tokio = { version = "1.38.0", features = ["rt", "rt-multi-thread", "macros"] }
 tracing = "0.1.40"
 eyre = "0.6.10"
 once_cell = "1.1.0"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 serde_json = "1.0.105"
+
+
+[[bin]]
+name = "fossil_headers_db"
+path = "src/main.rs"
+
+[[bin]]
+name = "fossil_indexer"
+path = "src/indexer/main.rs"

migrations/20250107071623_index_metadata.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+DROP TABLE IF EXISTS index_metadata;

migrations/20250107071623_index_metadata.up.sql

@@ -0,0 +1,9 @@
+-- Add up migration script here
+CREATE TABLE
+    IF NOT EXISTS index_metadata (
+        id INT8 GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+        current_latest_block_number BIGINT NOT NULL,
+        indexing_starting_block_number BIGINT NOT NULL,
+        is_backfilling BOOLEAN DEFAULT TRUE,
+        updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
+    )

migrations/20250108182341_index_metadata_new_field.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE index_metadata DROP COLUMN backfilling_block_number;

migrations/20250108182341_index_metadata_new_field.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE index_metadata ADD COLUMN backfilling_block_number BIGINT;

src/db/mod.rs

@@ -1,5 +1,5 @@
-use crate::types::type_utils::convert_hex_string_to_i64;
-use crate::types::BlockHeaderWithFullTransaction;
+use crate::rpc::BlockHeaderWithFullTransaction;
+use crate::utils::convert_hex_string_to_i64;
 use eyre::{Context, Error, Result};
 use futures::FutureExt;
 use sqlx::postgres::PgConnectOptions;
@@ -13,12 +13,12 @@ use tokio::sync::OnceCell;
 use tokio::time::sleep;
 use tracing::{error, info, warn};
 
-#[cfg(test)]
 mod db_test;
 
 static DB_POOL: OnceCell<Arc<Pool<Postgres>>> = OnceCell::const_new();
 pub const DB_MAX_CONNECTIONS: u32 = 50;
 
+// TODO: Not use a oncecell but instead use some sort of DI for easier testing.
 pub async fn get_db_pool() -> Result<Arc<Pool<Postgres>>> {
     if let Some(pool) = DB_POOL.get() {
         Ok(pool.clone())
@@ -304,6 +304,7 @@ where
     }
 }
 
+#[allow(clippy::all)]
 fn is_transient_error(e: &Error) -> bool {
     // Check for database connection errors
     if let Some(db_err) = e.downcast_ref::<sqlx::Error>() {
@@ -322,3 +323,34 @@ fn is_transient_error(e: &Error) -> bool {
         false
     }
 }
+
+#[derive(Debug)]
+pub struct DbConnection {
+    pub pool: Pool<Postgres>,
+}
+
+impl DbConnection {
+    // TODO: allow dead code for now. Adding tests in future PRs should allow us to remove this.
+    #[allow(dead_code)]
+    pub async fn new(db_conn_string: Option<String>) -> Result<Arc<Self>> {
+        let mut conn_options: PgConnectOptions = match db_conn_string {
+            Some(conn_string) => conn_string.parse()?,
+            None => dotenvy::var("DB_CONNECTION_STRING")
+                .context("DB_CONNECTION_STRING must be set")?
+                .parse()?,
+        };
+
+        conn_options = conn_options
+            .log_slow_statements(tracing::log::LevelFilter::Debug, Duration::new(120, 0));
+
+        let pool = PgPoolOptions::new()
+            .max_connections(DB_MAX_CONNECTIONS)
+            .connect_with(conn_options)
+            .await?;
+
+        Ok(Arc::new(Self { pool }))
+    }
+}
+
+#[cfg(test)]
+mod tests {}