Neardata streamer

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "inindexer"
-version = "1.2.0"
+version = "2.0.0"
 edition = "2021"
 readme = "README.md"
 repository = "https://github.com/INTEARnear/inindexer"
@@ -11,46 +11,31 @@ description = "A framework for building NEAR Protocol indexers"
 [dependencies]
 async-trait = "0.1.79"
 log = "0.4.21"
-near-lake-framework = { version = "=0.7.9", optional = true }
-near-indexer-primitives = "0.23.0"
+near-indexer-primitives = "0.28.0"
 serde_json = "1.0.115"
 tokio-stream = "0.1.15"
-tokio = { version = "1.37.0", features = [ "signal", "fs", "rt", "sync" ] }
-reqwest = { version = "0.12.4", optional = true }
+tokio = { version = "1.37.0", features = ["signal", "fs", "rt", "sync"] }
+reqwest = "0.12.4"
 serde = { version = "1.0.198", features = ["derive"] }
 semver = "1.0.22"
 futures = "0.3"
 tokio-util = "0.7.11"
+fastnear-neardata-fetcher = { git = "https://github.com/fastnear/libs", branch = "flat-state" }
+fastnear-primitives = { git = "https://github.com/fastnear/libs", branch = "flat-state" }
 
 [dev-dependencies]
 tokio = { version = "1.37.0", features = ["macros", "rt-multi-thread", "time"] }
 simple_logger = "5.0.0"
 temp-file = "0.1.8"
 
-[features]
-default = []
-neardata = [ "reqwest" ]
-lake = [ "near-lake-framework" ]
-
 [[example]]
 name = "watch-the-slime"
 path = "examples/watch_the_slime.rs"
-required-features = [ "neardata" ]
-
-[[example]]
-name = "watch-the-slime-lake"
-path = "examples/watch_the_slime_lake.rs"
-required-features = [ "lake" ]
 
 [[example]]
 name = "ft-transfers"
 path = "examples/ft_transfers.rs"
-required-features = [ "neardata" ]
 
 [[example]]
 name = "download"
 path = "examples/download.rs"
-required-features = [ "neardata" ]
-
-[package.metadata.docs.rs]
-all-features = true

README.md

@@ -6,9 +6,8 @@ InIndexer is a NEAR indexer framework.
 ## Features
 
 - Different sources of near data: [neardata](https://github.com/fastnear/neardata-server) (implemented),
-  [AWS Lake](https://docs.near.org/concepts/advanced/near-lake-framework) (only consecutive ascending ranges
-  are supported), local file storage for backfilling (planned), you can add your own sources by implementing
-  `MessageStreamer` or `message_provider::MessageProvider` trait.
+  nearcore indexer framework, you can add your own sources by implementing `MessageStreamer` or
+  `message_provider::MessageProvider` trait.
 - Simple indexer interface: you only need to implement `Indexer` trait and handle receipts, blocks,
   transactions, or transactions with all receipts included, at a cost of some preprocessing overhead (around 1-2ms
   in release mode with 80-100 TPS on Slime's PC, this can be disabled in `IndexerOptions::preprocess_transactions`).
@@ -19,11 +18,6 @@ InIndexer is a NEAR indexer framework.
 - Some helper functions and types for working with logs, balances, and other commonly used functionality in
   `near_utils`.
 
-## Feature flags
-
-- `neardata`: Neardata data source
-- `lake`: NEAR Lake data source
-
 This crate only works with tokio runtime.
 
 If you want to see some examples, check minimal examples in [examples/](examples/) or real indexers used in Intear infrastructure ([nft-indexer](https://github.com/INTEARnear/nft-indexer), [potlock-indexer](https://github.com/INTEARnear/potlock-indexer), [trade-indexer](https://github.com/INTEARnear/trade-indexer), [new-token-indexer](https://github.com/INTEARnear/new-token-indexer), [intear-oracle indexer](https://github.com/INTEARnear/oracle/tree/main/crates/indexer)). By the way, some of these repositories are libraries, so if you want the same functionality but with a different event handler, you can use them in your code by specifying them as git dependencies.

examples/ft_transfers.rs

@@ -72,7 +72,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         IndexerOptions {
             range: BlockIterator::AutoContinue(AutoContinue {
                 save_location: Box::new(PathBuf::from("example_ft_trasnfers_last_block.txt")),
-                start_height_if_does_not_exist: 114_625_946,
+                start_height_if_does_not_exist: 139770436,
                 end: inindexer::AutoContinueEnd::Infinite,
             }),
             stop_on_error: false,

examples/watch_the_slime.rs

@@ -2,7 +2,7 @@
 
 use async_trait::async_trait;
 use inindexer::{
-    neardata::NeardataProvider, run_indexer, BlockIterator, CompleteTransaction, Indexer,
+    neardata_old::OldNeardataProvider, run_indexer, BlockIterator, CompleteTransaction, Indexer,
     IndexerOptions,
 };
 use near_indexer_primitives::{types::AccountId, StreamerMessage};
@@ -44,7 +44,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     run_indexer(
         &mut indexer,
-        NeardataProvider::mainnet(),
+        OldNeardataProvider::mainnet(),
         IndexerOptions {
             range: BlockIterator::iterator(112_037_807..=112_037_810),
             ..Default::default()

src/indexer_tests.rs

@@ -1,16 +1,12 @@
-#[cfg(not(feature = "neardata"))]
-compile_error!("Use `cargo test --all-features` to run tests. If you want to skip AWS Lake test, run with `cargo test --features neardata`");
-
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
 use std::{collections::HashMap, ops::Range, path::PathBuf};
 
-#[cfg(feature = "lake")]
-use crate::lake::LakeStreamer;
+use crate::neardata::NeardataProvider;
 use crate::{
     message_provider::ParallelProviderStreamer, near_utils::MAINNET_GENESIS_BLOCK_HEIGHT,
-    neardata::NeardataProvider, AutoContinue, AutoContinueEnd, BlockIterator, CompleteTransaction,
-    IndexerOptions, PreprocessTransactionsSettings,
+    neardata_old::OldNeardataProvider, AutoContinue, AutoContinueEnd, BlockIterator,
+    CompleteTransaction, IndexerOptions, PreprocessTransactionsSettings,
 };
 use async_trait::async_trait;
 use near_indexer_primitives::{
@@ -22,7 +18,7 @@ use crate::multiindexer::{ChainIndexers, ParallelJoinIndexers};
 use crate::{run_indexer, Indexer};
 
 #[tokio::test]
-async fn neardata_provider() {
+async fn neardata_old_provider() {
     const RANGE: Range<BlockHeight> =
         MAINNET_GENESIS_BLOCK_HEIGHT..(MAINNET_GENESIS_BLOCK_HEIGHT + 10);
 
@@ -46,7 +42,7 @@ async fn neardata_provider() {
 
     run_indexer(
         &mut indexer,
-        NeardataProvider::mainnet(),
+        OldNeardataProvider::mainnet(),
         IndexerOptions {
             range: BlockIterator::iterator(RANGE),
             preprocess_transactions: None,
@@ -60,7 +56,7 @@ async fn neardata_provider() {
 }
 
 #[tokio::test]
-async fn neardata_optimistic_provider() {
+async fn neardata_provider() {
     const RANGE: Range<BlockHeight> =
         MAINNET_GENESIS_BLOCK_HEIGHT..(MAINNET_GENESIS_BLOCK_HEIGHT + 10);
 
@@ -84,7 +80,7 @@ async fn neardata_optimistic_provider() {
 
     run_indexer(
         &mut indexer,
-        NeardataProvider::mainnet().optimistic(),
+        NeardataProvider::mainnet(),
         IndexerOptions {
             range: BlockIterator::iterator(RANGE),
             preprocess_transactions: None,
@@ -94,12 +90,11 @@ async fn neardata_optimistic_provider() {
     .await
     .unwrap();
 
-    assert_eq!(indexer.blocks_processed, 4) // I guess old optimistic blocks are purged, so idk how to test
+    assert_eq!(indexer.blocks_processed, 4)
 }
 
-#[cfg(feature = "lake")]
 #[tokio::test]
-async fn lake_provider() {
+async fn neardata_optimistic_provider() {
     const RANGE: Range<BlockHeight> =
         MAINNET_GENESIS_BLOCK_HEIGHT..(MAINNET_GENESIS_BLOCK_HEIGHT + 10);
 
@@ -123,7 +118,7 @@ async fn lake_provider() {
 
     run_indexer(
         &mut indexer,
-        LakeStreamer::mainnet(),
+        OldNeardataProvider::mainnet().optimistic(),
         IndexerOptions {
             range: BlockIterator::iterator(RANGE),
             preprocess_transactions: None,
@@ -133,7 +128,7 @@ async fn lake_provider() {
     .await
     .unwrap();
 
-    assert_eq!(indexer.blocks_processed, 4)
+    assert_eq!(indexer.blocks_processed, 4) // I guess old optimistic blocks are purged, so idk how to test
 }
 
 #[tokio::test]
@@ -165,7 +160,7 @@ async fn parallel_provider_with_correct_order() {
 
     run_indexer(
         &mut indexer,
-        ParallelProviderStreamer::new(NeardataProvider::mainnet(), 3),
+        ParallelProviderStreamer::new(OldNeardataProvider::mainnet(), 3),
         IndexerOptions {
             range: BlockIterator::iterator(RANGE),
             preprocess_transactions: None,
@@ -202,7 +197,7 @@ async fn auto_continue() {
 
     run_indexer(
         &mut indexer,
-        NeardataProvider::mainnet(),
+        OldNeardataProvider::mainnet(),
         IndexerOptions {
             range: BlockIterator::AutoContinue(AutoContinue {
                 save_location: Box::new(PathBuf::from(save_path)),
@@ -222,7 +217,7 @@ async fn auto_continue() {
 
     run_indexer(
         &mut indexer,
-        NeardataProvider::mainnet(),
+        OldNeardataProvider::mainnet(),
         IndexerOptions {
             range: BlockIterator::AutoContinue(AutoContinue {
                 save_location: Box::new(PathBuf::from(save_path)),
@@ -327,7 +322,7 @@ async fn prefetch_and_postfetch_dont_process_blocks() {
 
     run_indexer(
         &mut indexer,
-        NeardataProvider::mainnet(),
+        OldNeardataProvider::mainnet(),
         IndexerOptions {
             range: BlockIterator::iterator(RANGE),
             preprocess_transactions: Some(PreprocessTransactionsSettings {
@@ -374,7 +369,7 @@ async fn preprocessing_should_supply_completed_transaction() {
 
     run_indexer(
         &mut indexer,
-        NeardataProvider::mainnet(),
+        OldNeardataProvider::mainnet(),
         IndexerOptions {
             range: BlockIterator::iterator(116_917_957..=116_917_962),
             preprocess_transactions: Some(PreprocessTransactionsSettings {