diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b00a5614c..df2aa7203 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,7 @@ jobs: - name: Run all checks run: pdm run all env: - ALCHEMY_KEY: ${{ secrets.ALCHEMY_KEY }} + NODE_API_KEY: ${{ secrets.ALCHEMY_KEY }} - name: Publish stable image uses: docker/build-push-action@v5 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68f01d251..05106cf92 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,4 +49,4 @@ jobs: - name: Run tests run: pdm run test env: - ALCHEMY_KEY: ${{ secrets.ALCHEMY_KEY }} + NODE_API_KEY: ${{ secrets.ALCHEMY_KEY }} diff --git a/Makefile b/Makefile index e5b8fd914..8c5e0aaa3 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ ruff: ## Lint with ruff ruff check --fix ${SOURCE} mypy: ## Lint with mypy - mypy --no-incremental --exclude ${PACKAGE} ${SOURCE} + mypy --no-incremental ${SOURCE} ## diff --git a/docs/0.quickstart-evm.md b/docs/0.quickstart-evm.md index 1d82790ea..c54986567 100644 --- a/docs/0.quickstart-evm.md +++ b/docs/0.quickstart-evm.md @@ -15,7 +15,7 @@ Let's create an indexer for the [USDt token contract](https://etherscan.io/addre A modern Linux/macOS distribution with Python 3.11 installed is required to run DipDup. -The easiest way to install DipDup as a CLI application [pipx](https://pypa.github.io/pipx/). We have a convenient wrapper script that installs DipDup for the current user. Run the following command in your terminal: +The easiest way to install DipDup as a CLI application [pipx](https://pipx.pypa.io/stable/). We have a convenient wrapper script that installs DipDup for the current user. Run the following command in your terminal: ```shell [Terminal] curl -Lsf https://dipdup.io/install.py | python3 diff --git a/docs/0.quickstart-tezos.md b/docs/0.quickstart-tezos.md index 7257de381..7e5b776db 100644 --- a/docs/0.quickstart-tezos.md +++ b/docs/0.quickstart-tezos.md @@ -15,7 +15,7 @@ Let's create an indexer for the [tzBTC FA1.2 token contract](https://tzkt.io/KT1 A modern Linux/macOS distribution with Python 3.11 installed is required to run DipDup. -The easiest way to install DipDup as a CLI application [pipx](https://pypa.github.io/pipx/). We have a convenient wrapper script that installs DipDup for the current user. Run the following command in your terminal: +The easiest way to install DipDup as a CLI application [pipx](https://pipx.pypa.io/stable/). We have a convenient wrapper script that installs DipDup for the current user. Run the following command in your terminal: ```shell [Terminal] curl -Lsf https://dipdup.io/install.py | python3 diff --git a/docs/1.getting-started/6.datasources.md b/docs/1.getting-started/6.datasources.md index c400db10e..6f37f9d39 100644 --- a/docs/1.getting-started/6.datasources.md +++ b/docs/1.getting-started/6.datasources.md @@ -9,16 +9,16 @@ Datasources are DipDup connectors to various APIs. They are defined in config an Index datasources, ones that can be attached to a specific index, are prefixed with blockchain name, e.g. `tezos.tzkt` or `evm.subsquid`. -| kind | blockchain | description | -| ---------------------------------------------------------- | -------------- | ------------------------------- | -| [abi.etherscan](../3.datasources/1.abi_etherscan.md) | EVM-compatible | Provides ABIs for EVM contracts | -| [coinbase](../3.datasources/2.coinbase.md) | any | Coinbase price feed | -| [evm.node](../3.datasources/3.evm_node.md) | EVM-compatible | Ethereum node | -| [evm.subsquid](../3.datasources/4.evm_subsquid.md) | EVM-compatible | Subsquid Archives node | -| [http](../3.datasources/5.http.md) | any | Generic HTTP API | -| [ipfs](../3.datasources/6.ipfs.md) | any | IPFS gateway | -| [tezos.tzkt](../3.datasources/7.tezos_tzkt.md) | Tezos | TzKT API | -| [tezos.tzip_metadata](../3.datasources/8.tzip_metadata.md) | Tezos | TZIP-16 metadata | +| kind | blockchain | description | +| ---------------------------------------------------------- | ---------------- | ------------------------------- | +| [abi.etherscan](../3.datasources/1.abi_etherscan.md) | ⟠ EVM-compatible | Provides ABIs for EVM contracts | +| [coinbase](../3.datasources/2.coinbase.md) | any | Coinbase price feed | +| [evm.node](../3.datasources/3.evm_node.md) | ⟠ EVM-compatible | Ethereum node | +| [evm.subsquid](../3.datasources/4.evm_subsquid.md) | ⟠ EVM-compatible | Subsquid Network node | +| [http](../3.datasources/5.http.md) | any | Generic HTTP API | +| [ipfs](../3.datasources/6.ipfs.md) | any | IPFS gateway | +| [tezos.tzkt](../3.datasources/7.tezos_tzkt.md) | ꜩ Tezos | TzKT API | +| [tezos.tzip_metadata](../3.datasources/8.tzip_metadata.md) | ꜩ Tezos | TZIP-16 metadata | ## Connection settings diff --git a/docs/15.thanks.md b/docs/15.thanks.md index da493d91b..0787d7eda 100644 --- a/docs/15.thanks.md +++ b/docs/15.thanks.md @@ -24,11 +24,11 @@ We are grateful to all the people who helped us with the project. - [852Kerfunkle](https://github.com/852Kerfunkle) - [Anshit Bhardwaj](https://github.com/Anshit01) - [arrijabba](https://github.com/arrijabba) +- [Do Quang Huy](https://github.com/huydo2105) - [Fitblip](https://github.com/Fitblip) -- [Florian PAUTOT](https://github.com/xflpt) +- [Florian PAUTOT](https://github.com/0x666c6f) - [gdsoumya](https://github.com/gdsoumya) - [Göran Sandström](https://github.com/veqtor) -- [herohthd](https://github.com/herohthd) - [Javier Graciá Carpio](https://github.com/jagracar) - [Karan Dua](https://github.com/Karantezsure) - [Nick Kalomoiris](https://github.com/nikos-kalomoiris) diff --git a/docs/2.indexes/1.evm_subsquid_events.md b/docs/2.indexes/1.evm_subsquid_events.md index cba398099..1fb835ad7 100644 --- a/docs/2.indexes/1.evm_subsquid_events.md +++ b/docs/2.indexes/1.evm_subsquid_events.md @@ -8,7 +8,7 @@ network: "ethereum" This index allows indexing events emitted by Ethereum smart contracts. You can define a handler for each contract/tag pair. Only necessary events are processed. -RPC node is optional for this index. If not specified, the index will lack real-time data and will be able to process only historical data provided by Subsquid Archives API. +RPC node is optional for this index. If not specified, the index will lack real-time data and will be able to process only historical data provided by Subsquid Network API. This is a basic config for USDt token contract. You can use this demo as a template for your project choosing `demo_evm_events` in `dipdup new` command. diff --git a/docs/3.datasources/3.evm_node.md b/docs/3.datasources/3.evm_node.md index 34a5339d0..43809992e 100644 --- a/docs/3.datasources/3.evm_node.md +++ b/docs/3.datasources/3.evm_node.md @@ -1,12 +1,12 @@ --- title: "EVM Node" -description: "DipDup can connect to any EVM-compatible node via JSON-RPC. It can be used as a last mile datasource for EVM indexes (data that is not in Subsquid Archives yet) or as a standalone datasource for handlers and hooks." +description: "DipDup can connect to any EVM-compatible node via JSON-RPC. It can be used as a last mile datasource for EVM indexes (data that is not in Subsquid Network yet) or as a standalone datasource for handlers and hooks." network: "ethereum" --- # EVM node -DipDup can connect to any EVM-compatible node via JSON-RPC. It can be used as a "last mile" datasource for EVM indexes (data that is not in Subsquid Archives yet) or as a standalone datasource for handlers and hooks. +DipDup can connect to any EVM-compatible node via JSON-RPC. It can be used as a "last mile" datasource for EVM indexes (data that is not in Subsquid Network yet) or as a standalone datasource for handlers and hooks. Examples below show how to connect to Infura and Alchemy nodes for Ethereum mainnet indexes. You can also use your own node, but make sure it has all the necessary data (e.g. archive node). @@ -14,18 +14,18 @@ Examples below show how to connect to Infura and Alchemy nodes for Ethereum main datasources: mainnet_alchemy_node: kind: evm.node - url: https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY} - ws_url: wss://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY} + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} ``` EVM node datasources could linked to specific Subsquid ones instead of being used directly in indexes. ```yaml [dipdup.yaml] datasources: - mainnet_subsquid: + subsquid: kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} - node: mainnet_node + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: evm_node ``` ## web3 client @@ -39,7 +39,7 @@ Don't initialize web3 clients manually! It will break the connection pooling, lo To access the client, use `web3` property of the datasource. The underlying web3 client is asynchronous, so you should use `await` keyword to call its methods. ```python -web3: AsyncWeb3 = ctx.get_evm_node_datasource('mainnet_node').web3 +web3: AsyncWeb3 = ctx.get_evm_node_datasource('evm_node').web3 contract = self.web3.eth.contract(...) symbol = await contract.functions.symbol().call() ``` diff --git a/docs/3.datasources/4.evm_subsquid.md b/docs/3.datasources/4.evm_subsquid.md index ceb177531..6b3317198 100644 --- a/docs/3.datasources/4.evm_subsquid.md +++ b/docs/3.datasources/4.evm_subsquid.md @@ -1,31 +1,31 @@ --- -title: "Subsquid Archives" -description: "DipDup uses Subsquid Archives as a source of historical data for EVM-compatible blockchains." +title: "Subsquid Network" +description: "DipDup uses Subsquid Network as a source of historical data for EVM-compatible blockchains." network: "ethereum" --- -# Subsquid Archives +# Subsquid Network -DipDup uses [Subsquid Archives](https://docs.subsquid.io/archives/) as a source of historical data for EVM-compatible blockchains. Configure the datasource in your project config: +DipDup uses [Subsquid Network](https://docs.subsquid.io/subsquid-network/reference/evm-api/) as a source of historical data for EVM-compatible blockchains. Configure the datasource in your project config: ```yaml [dipdup.yaml] datasources: - mainnet_subsquid: + subsquid: kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} ``` You can also link the datasource to a specific node to allow realtime indexing: ```yaml [dipdup.yaml] datasources: - mainnet_node: + evm_node: kind: evm.node - url: https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} - ws_url: wss://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} - mainnet_subsquid: + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + subsquid: kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} ``` Now, use this datasource in EVM index definitions: @@ -34,6 +34,6 @@ Now, use this datasource in EVM index definitions: indexes: uniswap_v3: kind: evm.subsquid.events - datasource: mainnet_subsquid + datasource: subsquid ... ``` diff --git a/docs/4.graphql/4.genql.md b/docs/4.graphql/4.genql.md index 846c7b0fb..1063655f7 100644 --- a/docs/4.graphql/4.genql.md +++ b/docs/4.graphql/4.genql.md @@ -52,7 +52,7 @@ yarn yarn build ``` -Read more about CLI [options](https://genql.vercel.app/docs/cli/generate) available. +Read more about CLI [options](https://genql.dev/docs/cli-reference) available. ### Demo diff --git a/docs/5.advanced/6.evm-compatible.md b/docs/5.advanced/6.evm-compatible.md index 1ef176532..de9140776 100644 --- a/docs/5.advanced/6.evm-compatible.md +++ b/docs/5.advanced/6.evm-compatible.md @@ -1,235 +1,218 @@ --- title: "EVM networks" -description: "This page contents list of supported EVM-compatible networks" +description: "This page contains the list of supported EVM-compatible networks" network: "ethereum" --- - + # EVM networks -This page contents list of supported EVM-compatible networks +Most of the examples in this documentation are for Ethereum. But DipDup can index any EVM-compatible network as long as there's enough historical data. This page contains a list of supported networks and instructions on how to configure your indexer for them. -## How to configure datasources for EVM networks +## Configuring datasources -A typical EVM indexer uses three types of datasources for different purposes, detailed in their respective articles: [abi.etherscan](../3.datasources/1.abi_etherscan.md), [evm.subsquid](../3.datasources/4.evm_subsquid.md), [evm.node](../3.datasources/3.evm_node.md). Here is a brief guide on how to set them up for EVM networks. -Examples have been taken from demo_evm_events from which you can start building your EVM network indexer using: - -```shell [Terminal] -dipdup new -``` - -### [abi.etherscan](../3.datasources/1.abi_etherscan.md) - -Many explorers have etherscan-like API and it could be used to retrieve ABIs for futher types generating using abi.etherscan -You can get url for abi.etherscan datasource in table for your network, for some etherscan-like explorers you will also need API key. +EVM indexes can use datasources of three kinds. If you've created a new project using `dipdup new` command and used one of the EVM templates, `datasources` section in your config file should look like this: ```yaml [dipdup.yaml] - ethscan: +datasources: + subsquid: + kind: evm.subsquid + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: evm_node + etherscan: kind: abi.etherscan - url: ${ETHERSCAN_API_URL:-https://api.etherscan.io/api} - api_key: ${ETHERSCAN_API_KEY:-YourApiKeyToken} + url: ${ETHERSCAN_URL:-https://api.etherscan.io/api} + api_key: ${ETHERSCAN_API_KEY:-''} + evm_node: + kind: evm.node + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} ``` -#### If abi.etherscan not supported for network - -Put json with abi to abi/{contract typename}/abi.json and run dipdup init, it will generate all necessary types, then you can run your indexer. +Default values here are for Ethereum mainnet and Alchemy node provider. To configure datasources for other networks, you need to change URLs and API keys. You can do it in the config file directly, but it's better to use environment variables. Check the `deploy/.env.default` file in your project directory; it contains all the variables used in config. ### [evm.subsquid](../3.datasources/4.evm_subsquid.md) -To configure evm.subsquid datasource paste evm.subsquid url from tables below or set ARCHIVE_URL variable in .env file. +Subsquid Network is where DipDup gets historical data from. It's the only datasource required to operate. If your favorite blockchain is on this page, it means that Subsquid has historical data for it. -```yaml [demo_evm_events/dipdup.yaml] -datasources: - ... - mainnet_subsquid: - kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} - node: mainnet_node - ... -``` +### [abi.etherscan](../3.datasources/1.abi_etherscan.md) -```yaml [arbitrum/dipdup.yaml] - ... - url: https://v2.archive.subsquid.io/network/arbitrum-one - ... -``` +Etherscan is a source of contract ABIs, which are used to generate types for the indexer. Many explorers have Etherscan-like API which could be used to retrieve ABIs. You can get a URL for it in the table below. Please note, that some Etherscan-like explorers require an API key, which you can get on their website. + +"🔴 no API" in the table below means we couldn't find an explorer with an Etherscan-like API for this network. You need to obtain contract ABI JSON somewhere and put it to the `abi//abi.json` path. Don't forget to run `dipdup init` after that to generate all necessary types. ### [evm.node](../3.datasources/3.evm_node.md) -Node RPC API vary a lot across EVM networks, yet most of them supported by dipdup indexer to retrieve historical data. -Public node providers usualy require API key, so demo_evm_events template allow you to pass https url, websocket(wss) url, and api key into NODE_RPC_URL, NODE_WS_URL, and NODE_API_KEY using .env file to use node providers like Alchemy and Infura from the box. +EVM node datasource can be used to fetch recent data not yet in Subsquid Network. API methods could vary a lot across different networks, but DipDup only uses a few of them, so most of the nodes will work. -```yaml [dipdup.yaml] - mainnet_node: - kind: evm.node - url: ${NODE_RPC_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} - ws_url: ${NODE_WEBSOCKET_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} -``` +WebSocket URL can be specified to get real-time updates. This option can save you some requests to the node, but otherwise, it's not required. If DipDup doesn't work with the EVM node in WebSocket mode, it's marked as "🟡 HTTP only" in the table below. + +## Supported networks -Yellow support status for evm.node for the network means only historical data could be retrieved from the node. Green support status means also real-time subscriptions supported. Red support status means no node providers were available for test - you can try to use own's node RPC API. +_Updated 2024-01-26._ -## Networks list 🟢🟡🔴 +Something here is missing or wrong? Open an issue or pull request in [DipDup repository](https://github.com/dipdup-io/dipdup). ### Arbitrum -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/arbitrum-one | -| **abi.etherscan** | 🟢 | https://api.arbiscan.io/api | -| **evm.node** | 🟡 | https://arb-mainnet.g.alchemy.com/v2
wss://arb-mainnet.g.alchemy.com/v2 | +| datasource | supported | URLs | +| -----------------:|:------------ | ----------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/arbitrum-one` | +| **abi.etherscan** | 🟢 yes | `https://api.arbiscan.io/api` | +| **evm.node** | 🟡 HTTP only | `https://arb-mainnet.g.alchemy.com/v2` | ### Astar -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/astar-mainnet | -| **abi.etherscan** | 🔴 | N/A | -| **evm.node** | 🟡 | https://astar-mainnet.g.alchemy.com/v2
wss://astar-mainnet.g.alchemy.com/v2 | +| datasource | supported | URLs | +| -----------------:|:------------ | ------------------------------------------------------ | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/astar-mainnet` | +| **abi.etherscan** | 🔴 no API | N/A | +| **evm.node** | 🟡 HTTP only | `https://astar-mainnet.g.alchemy.com/v2` | ### Avalanche -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/avalanche-mainnet | -| **abi.etherscan** | 🟢 | https://api.routescan.io/v2/network/mainnet/evm/43114/etherscan/api | -| **evm.node** | 🟡 | https://avalanche-mainnet.infura.io/v3
wss://avalanche-mainnet.infura.io/v3 | +| datasource | supported | URLs | +| -----------------:|:------------ | --------------------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/avalanche-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api.routescan.io/v2/network/mainnet/evm/43114/etherscan/api` | +| **evm.node** | 🟡 HTTP only | `https://avalanche-mainnet.infura.io/v3` | ### Base -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/base-mainnet | -| **abi.etherscan** | 🟢 | https://api.basescan.org/api | -| **evm.node** | 🟡 | https://base-mainnet.g.alchemy.com/v2
wss://base-mainnet.g.alchemy.com/v2 | +| datasource | supported | URLs | +| -----------------:|:------------ | ----------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/base-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api.basescan.org/api` | +| **evm.node** | 🟡 HTTP only | `https://base-mainnet.g.alchemy.com/v2` | -### Binance +### Binance Smart Chain -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/binance-mainnet | -| **abi.etherscan** | 🟢 | https://api.bscscan.com/api | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | -------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/binance-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api.bscscan.com/api` | +| **evm.node** | 🔴 not tested | N/A | ### Exosama -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/exosama | -| **abi.etherscan** | 🔴 | N/A | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | ------------------------------------------------ | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/exosama` | +| **abi.etherscan** | 🔴 no API | N/A | +| **evm.node** | 🔴 not tested | N/A | ### Fantom -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/fantom-mainnet | -| **abi.etherscan** | 🟢 | https://api.ftmscan.com/api | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | ------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/fantom-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api.ftmscan.com/api` | +| **evm.node** | 🔴 not tested | N/A | ### Gnosis -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/gnosis-mainnet | -| **abi.etherscan** | 🟢 | https://api.gnosisscan.io/api | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | ------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/gnosis-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api.gnosisscan.io/api` | +| **evm.node** | 🔴 not tested | N/A | ### Linea -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/linea-mainnet | -| **abi.etherscan** | 🟢 | https://api.lineascan.build/api | -| **evm.node** | 🟢 | https://linea-mainnet.infura.io/v3
wss://mainnet.infura.io/ws/v3 | +| datasource | supported | URLs | +| -----------------:|:--------- | ------------------------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/linea-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api.lineascan.build/api` | +| **evm.node** | 🟢 yes | `https://linea-mainnet.infura.io/v3`
`wss://mainnet.infura.io/ws/v3` | ### Moonbeam -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/moonbeam-mainnet | -| **abi.etherscan** | 🟢 | https://api-moonbeam.moonscan.io/api | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | --------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/moonbeam-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api-moonbeam.moonscan.io/api` | +| **evm.node** | 🔴 not tested | N/A | ### opBNB -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/opbnb-mainnet | -| **abi.etherscan** | 🔴 | N/A | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | ------------------------------------------------------ | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/opbnb-mainnet` | +| **abi.etherscan** | 🔴 no API | N/A | +| **evm.node** | 🔴 not tested | N/A | ### Optimism -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/optimism-mainnet | -| **abi.etherscan** | 🟢 | https://api-optimistic.etherscan.io/api | -| **evm.node** | 🟢 | https://opt-mainnet.g.alchemy.com/v2
wss://opt-mainnet.g.alchemy.com/v2 | +| datasource | supported | URLs | +| -----------------:|:--------- | -------------------------------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/optimism-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api-optimistic.etherscan.io/api` | +| **evm.node** | 🟢 yes | `https://opt-mainnet.g.alchemy.com/v2`
`wss://opt-mainnet.g.alchemy.com/v2` | ### Polygon PoS -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/polygon-mainnet | -| **abi.etherscan** | 🟢 | https://api.polygonscan.com/api | -| **evm.node** | 🟡 | https://polygon-mainnet.g.alchemy.com/v2
wss://polygon-mainnet.g.alchemy.com/v2 | +| datasource | supported | URLs | +| -----------------:|:------------ | -------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/polygon-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api.polygonscan.com/api` | +| **evm.node** | 🟡 HTTP only | `https://polygon-mainnet.g.alchemy.com/v2` | ### Polygon zkEVM -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/polygon-zkevm-mainnet | -| **abi.etherscan** | 🟢 | https://api-zkevm.polygonscan.com/api | -| **evm.node** | 🟡 | https://polygonzkevm-mainnet.g.alchemy.com/v2
wss://polygonzkevm-mainnet.g.alchemy.com/v2 | +| datasource | supported | URLs | +| -----------------:|:------------ | -------------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/polygon-zkevm-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api-zkevm.polygonscan.com/api` | +| **evm.node** | 🟡 HTTP only | `https://polygonzkevm-mainnet.g.alchemy.com/v2` | ### Shiden -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/shiden-mainnet | -| **abi.etherscan** | 🔴 | N/A | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | ------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/shiden-mainnet` | +| **abi.etherscan** | 🔴 no API | N/A | +| **evm.node** | 🔴 not tested | N/A | ### Tanssi -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/tanssi | -| **abi.etherscan** | 🔴 | N/A | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | ----------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/tanssi` | +| **abi.etherscan** | 🔴 no API | N/A | +| **evm.node** | 🔴 not tested | N/A | ### zkSync -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/zksync-mainnet | -| **abi.etherscan** | 🔴 | N/A | -| **evm.node** | 🔴 | N/A | +| datasource | supported | URLs | +| -----------------:|:------------- | ------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/zksync-mainnet` | +| **abi.etherscan** | 🔴 no API | N/A | +| **evm.node** | 🔴 not tested | N/A | ### Zora -| datasource | status | URL | -|---------------------|--------|------------------------| -| **evm.subsquid** | 🟢 | https://v2.archive.subsquid.io/network/zora-mainnet | -| **abi.etherscan** | 🟢 | https://api.routescan.io/v2/network/mainnet/evm/7777777/etherscan/api | -| **evm.node** | 🔴 | N/A | - -## Explorers list - -Below is a list of explorers with etherscan-like API available to use in `abi.etherscan` datasource, retrieving ABIs. - -| Network | Explorer link | API endpoints(url) | -|---------------|-------------------------------------|-------------------------------------------| -| arbitrum | [arbiscan.io](https://arbiscan.io/) | https://api.arbiscan.io/api | -| avalanche | [snowtrace.io](https://snowtrace.io/) | https://api.routescan.io/v2/network/mainnet/evm/43114/etherscan/api | -| base | [basescan.org](https://basescan.org/) | https://api.basescan.org/api | -| binance | [bscscan.com](https://bscscan.com/) | https://api.bscscan.com/api | -| fantom | [ftmscan.com](https://ftmscan.com/) | https://api.ftmscan.com/api | -| gnois | [gnosisscan.io](https://gnosisscan.io/) | https://api.gnosisscan.io/api | -| linea | [lineascan.build](https://lineascan.build/) | https://api.lineascan.build/api | -| moonbeam | [moonscan.io](https://moonscan.io/) | https://api-moonbeam.moonscan.io/api | -| optimism | [optimistic.etherscan.io](https://optimistic.etherscan.io/) | https://api-optimistic.etherscan.io/api | -| polygon pos | [polygonscan.com](https://polygonscan.com) | https://api.polygonscan.com/api | -| polygon zkevm | [zkevm.polygonscan.com](https://zkevm.polygonscan.com/) | https://api-zkevm.polygonscan.com/api | -| zora | [zora.superscan.network](https://zora.superscan.network/) | https://api.routescan.io/v2/network/mainnet/evm/7777777/etherscan/api | +| datasource | supported | URLs | +| -----------------:|:------------- | ----------------------------------------------------------------------- | +| **evm.subsquid** | 🟢 yes | `https://v2.archive.subsquid.io/network/zora-mainnet` | +| **abi.etherscan** | 🟢 yes | `https://api.routescan.io/v2/network/mainnet/evm/7777777/etherscan/api` | +| **evm.node** | 🔴 not tested | N/A | + +## Blockchain explorers + +Below is a list of explorers with Etherscan-like API available to use in `abi.etherscan` datasource, retrieving ABIs. + +| network | explorer | API | +| ------------------- | ----------------------------------------------------------- | ----------------------------------------------------------------------- | +| Arbitrum | [arbiscan.io](https://arbiscan.io/) | `https://api.arbiscan.io/api` | +| Avalanche | [snowtrace.io](https://snowtrace.io/) | `https://api.routescan.io/v2/network/mainnet/evm/43114/etherscan/api` | +| Base | [basescan.org](https://basescan.org/) | `https://api.basescan.org/api` | +| Binance Smart Chain | [bscscan.com](https://bscscan.com/) | `https://api.bscscan.com/api` | +| Fantom | [ftmscan.com](https://ftmscan.com/) | `https://api.ftmscan.com/api` | +| Gnois | [gnosisscan.io](https://gnosisscan.io/) | `https://api.gnosisscan.io/api` | +| Linea | [lineascan.build](https://lineascan.build/) | `https://api.lineascan.build/api` | +| Moonbeam | [moonscan.io](https://moonscan.io/) | `https://api-moonbeam.moonscan.io/api` | +| Optimism | [optimistic.etherscan.io](https://optimistic.etherscan.io/) | `https://api-optimistic.etherscan.io/api` | +| Polygon PoS | [polygonscan.com](https://polygonscan.com) | `https://api.polygonscan.com/api` | +| Polygon zkEVM | [zkevm.polygonscan.com](https://zkevm.polygonscan.com/) | `https://api-zkevm.polygonscan.com/api` | +| Zora | [zora.superscan.network](https://zora.superscan.network/) | `https://api.routescan.io/v2/network/mainnet/evm/7777777/etherscan/api` | diff --git a/docs/6.deployment/1.database.md b/docs/6.deployment/1.database.md index 44c444d02..5f72180bf 100644 --- a/docs/6.deployment/1.database.md +++ b/docs/6.deployment/1.database.md @@ -7,13 +7,13 @@ description: "DipDup officially supports SQLite, PostgreSQL and TimescaleDB as a DipDup officially supports SQLite, PostgreSQL and TimescaleDB as a database engine. This table will help you choose a database engine that mostly suits your needs. -| | SQLite | PostgreSQL | TimescaleDB | -|:-------------------- |:-----------------:|:-------------:|:-----------------------:| -| Recommended versions | latest | 14, 15 | 14, 15 | +| | SQLite | PostgreSQL | TimescaleDB | +|:-------------------- |:----------------- |:------------- |:----------------------- | +| Recommended versions | latest | 14, 15 | 14, 15 | | Best application | early development | general usage | working with timeseries | -| SQL scripts | ✅ | ✅ | ✅ | -| Immune tables | ⚠ | ✅ | ✅ | -| Hasura integration | ❌ | ✅ | ✅ | +| SQL scripts | 🟢 | 🟢 | 🟢 | +| Immune tables | ⚠️ | 🟢 | 🟢 | +| Hasura integration | 🔴 | 🟢 | 🟢 | By default, DipDup uses an in-memory SQLite database that is destroyed after the process exits. @@ -62,4 +62,4 @@ Now you can create a continuous aggregate on top of the hypertable: For more information visit the official TimescaleDB documentation: - [Hypertables](https://docs.timescale.com/use-timescale/latest/hypertables/) -- [Continuous aggregates](https://docs.timescale.com/use-timescale/continuous-aggregates/) +- [Continuous aggregates](https://docs.timescale.com/use-timescale/latest/continuous-aggregates/) diff --git a/docs/6.deployment/4.prometheus.md b/docs/6.deployment/4.prometheus.md index 679a23777..382d3a25a 100644 --- a/docs/6.deployment/4.prometheus.md +++ b/docs/6.deployment/4.prometheus.md @@ -27,10 +27,10 @@ The following metrics are exposed under `dipdup` namespace: Also, DipDup exposes the following metrics for compatibility with Subsquid Cloud: -| metric name | description | -| ------------------------------------------ | ---------------------------------------------------------- | -| `sqd_processor_last_block` | Level of the last processed block from Subsquid Archives | -| `sqd_processor_chain_height` | Current chain height as reported by Subsquid Archives | -| `sqd_processor_archive_http_errors_in_row` | Number of consecutive failed requests to Subsquid Archives | +| metric name | description | +| ------------------------------------------ | --------------------------------------------------------- | +| `sqd_processor_last_block` | Level of the last processed block from Subsquid Network | +| `sqd_processor_chain_height` | Current chain height as reported by Subsquid Network | +| `sqd_processor_archive_http_errors_in_row` | Number of consecutive failed requests to Subsquid Network | If you need more complex metrics, consider querying [Internal tables](../5.advanced/3.sql.md#internal-tables). diff --git a/docs/7.references/2.config.md b/docs/7.references/2.config.md index b7a04913c..0121dd893 100644 --- a/docs/7.references/2.config.md +++ b/docs/7.references/2.config.md @@ -146,14 +146,14 @@ description: "Config file reference"
-class dipdup.config.evm_node.EvmNodeDatasourceConfig(kind, url, ws_url, http=None, rollback_depth=32)
+class dipdup.config.evm_node.EvmNodeDatasourceConfig(kind, url, ws_url=None, http=None, rollback_depth=32)

Subsquid datasource config

Parameters:

  • kind (Literal['evm.node']) – Always ‘evm.node’

  • url (str) – Ethereum node URL

    • -

  • ws_url (str) – Ethereum node WebSocket URL

    • +

  • ws_url (str | None) – Ethereum node WebSocket URL

  • http (HttpConfig | None) – HTTP client configuration

  • rollback_depth (int) – A number of blocks to store in database for rollback

@@ -187,7 +187,7 @@ description: "Config file reference"

  • datasource (str | SubsquidDatasourceConfig) – Subsquid datasource

  • handlers (tuple[SubsquidEventsHandlerConfig, ...]) – Event handlers

  • abi (AbiDatasourceConfig | tuple[AbiDatasourceConfig, ...] | None) – One or more evm.abi datasource(s) for the same network

    • -

  • node_only (bool) – Don’t use Subsquid Archives API (dev only)

    • +

  • node_only (bool) – Don’t use Subsquid Network API (dev only)

  • first_level (int) – Level to start indexing from

  • last_level (int) – Level to stop indexing and disable this index

    • @@ -203,7 +203,7 @@ description: "Config file reference"
    Parameters:

    • kind (Literal['evm.subsquid']) – always ‘evm.subsquid’

      • -

    • url (str) – URL of Subsquid Archives API

      • +

    • url (str) – URL of Subsquid Network API

    • node (str | tuple[str, ...] | EvmNodeDatasourceConfig | tuple[EvmNodeDatasourceConfig, ...] | None) – One or more evm.node datasource(s) for the same network

    • http (HttpConfig | None) – HTTP client configuration

    diff --git a/docs/8.examples/2.in-production.md b/docs/8.examples/2.in-production.md index 040cc5a04..4b1878c7d 100644 --- a/docs/8.examples/2.in-production.md +++ b/docs/8.examples/2.in-production.md @@ -7,7 +7,7 @@ description: "This page is a brief overview of projects which use DipDup as an i This page is a brief overview of projects which use DipDup as an indexing solution. -Want to see your project on this page? Create an issue on [GitHub](https://github.com/dipdup-io/dipdup-docs/issues)! +Want to see your project on this page? Create an issue on [GitHub](https://github.com/dipdup-io/dipdup/issues)! ## Rarible / Tezos marketplace indexer diff --git a/docs/9.release-notes/2.v7.2.md b/docs/9.release-notes/2.v7.2.md index 6c79b5aa3..b395c1c8a 100644 --- a/docs/9.release-notes/2.v7.2.md +++ b/docs/9.release-notes/2.v7.2.md @@ -89,7 +89,7 @@ See the [API reference](../7.references/4.api.md) for more details. `make` is back! After gathering feedback from the community we decided to return `Makefile` to the base template, but _make_ it package manager-agnostic. Instead, when creating a new project DipDup suggests choosing a package manager from PDM/Poetry/None (sets `package_manager` field in `replay.yaml` config). No action is required for existing projects, but you can run `dipdup init --base [--force]` to update the project base. -Indexing EVM events with node RPC (`evm.node` datasource) has become faster. Node RPC while inherently slow, is used only for the "last mile" indexing (fetching a few hundred latest blocks not yet in Subsquid Archives API) and realtime subscriptions. +Indexing EVM events with node RPC (`evm.node` datasource) has become faster. Node RPC while inherently slow, is used only for the "last mile" indexing (fetching a few hundred latest blocks not yet in Subsquid Network API) and realtime subscriptions. Finally, we have sped up the generation of Pydantic typeclasses with `datamodel-codegen`. Also, resulting classes use `X | Y` union syntax when possible. diff --git a/docs/9.release-notes/4.v7.0.md b/docs/9.release-notes/4.v7.0.md index 983859263..e11ccc1e5 100644 --- a/docs/9.release-notes/4.v7.0.md +++ b/docs/9.release-notes/4.v7.0.md @@ -21,7 +21,7 @@ Join our socials to discuss this release and ask any questions! ## EVM support -Now DipDup supports EVM-compatible blockchains in addition to Tezos. The new index allows you to process contract events from Ethereum and other EVM-compatible networks. DipDup uses historical data from [Subsquid Archives](https://www.subsquid.io/), real-time data from RPC nodes, and ABIs from [Etherscan](https://etherscan.io/). All you need is to define an index in your config and implement handlers for each event. +Now DipDup supports EVM-compatible blockchains in addition to Tezos. The new index allows you to process contract events from Ethereum and other EVM-compatible networks. DipDup uses historical data from [Subsquid Network](https://www.subsquid.io/), real-time data from RPC nodes, and ABIs from [Etherscan](https://etherscan.io/). All you need is to define an index in your config and implement handlers for each event. We have two demo projects for EVM: a very basic USDt price indexer and a more complex one for Uniswap v3 protocol. Run the `dipdup new` command, choose "EVM" on the first page, then a template to use. diff --git a/pdm.lock b/pdm.lock index 7b38b60c0..ea71553fd 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "docs"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:bf0ac3be9b802fecd1f23fda05b975aafc1302029e06e390f347a7ec1ed19c7e" +content_hash = "sha256:cb22f060be22021ad453feb7a64138981271902eca95a4eca834afd1348d38ac" [[package]] name = "aiohttp" diff --git a/pyproject.toml b/pyproject.toml index e78f09e54..67745c4da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,7 +90,7 @@ dev = [ "docker", "mypy", "pprofile", - "pytest", + "pytest~=7.4", "pytest-aiohttp", "pytest-asyncio", "pytest-cov", diff --git a/requirements.dev.txt b/requirements.dev.txt index 76cd4ebb8..0b33d8e98 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -5,39 +5,40 @@ aiohttp==3.9.1 aiolimiter==1.1.0 aiosignal==1.3.1 aiosqlite==0.17.0 -alabaster==0.7.13 +alabaster==0.7.16 anyio==4.2.0 -APScheduler==3.10.4 -argcomplete==3.1.6 +apscheduler==3.10.4 +argcomplete==3.2.2 async-lru==2.0.4 async-timeout==4.0.3; python_version < "3.12.0" asyncclick==8.1.7.1 asyncpg==0.29.0 -attrs==23.1.0 -babel==2.13.1 -bitarray==2.8.3 -black==24.1a1 +attrs==23.2.0 +babel==2.14.0 +bitarray==2.9.2 +black==24.1.0 certifi==2023.11.17 charset-normalizer==3.3.2 click==8.1.7 colorama==0.4.6; platform_system == "Windows" or sys_platform == "win32" -coverage==7.3.2 -cytoolz==0.12.2; implementation_name == "cpython" +coverage==7.4.0 +cytoolz==0.12.3; implementation_name == "cpython" datamodel-code-generator==0.25.2 -dnspython==2.4.2; python_version ~= "3.11" +dc-schema @ git+https://github.com/dipdup-io/dc_schema.git@6ed9a0cac3e1c43a2cc702e920d501493f9827d3 +dnspython==2.5.0; python_version ~= "3.11" docker==7.0.0 docutils==0.20.1 email-validator==2.1.0.post1; python_version ~= "3.11" eth-abi==4.2.1 eth-account==0.10.0 -eth-hash==0.5.2 -eth-keyfile==0.6.1 -eth-keys==0.4.0 -eth-rlp==0.3.0 -eth-typing==3.5.2 -eth-utils==2.3.1 +eth-hash==0.6.0 +eth-keyfile==0.7.0 +eth-keys==0.5.0 +eth-rlp==1.0.1 +eth-typing==4.0.0 +eth-utils==3.0.0 execnet==2.0.2 -frozenlist==1.4.0 +frozenlist==1.4.1 genson==1.2.2 hexbytes==0.3.1 idna==3.6 @@ -45,30 +46,30 @@ imagesize==1.4.1 inflect==5.6.2 iniconfig==2.0.0 iso8601==1.1.0 -isort==5.12.0 -jinja2==3.1.2 -jsonschema==4.20.0 -jsonschema-specifications==2023.11.1 +isort==5.13.2 +jinja2==3.1.3 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 lru-dict==1.3.0 -MarkupSafe==2.1.3 +markupsafe==2.1.4 msgpack==1.0.7 multidict==6.0.4 mypy==1.8.0 mypy-extensions==1.0.0 -numpy==1.26.2 +numpy==1.26.3 orjson==3.9.12 packaging==23.2 parsimonious==0.9.0 -pathspec==0.11.2 -platformdirs==4.0.0 -pluggy==1.3.0 +pathspec==0.12.1 +platformdirs==4.1.0 +pluggy==1.4.0 pprofile==2.1.0 prometheus-client==0.19.0 -protobuf==4.25.1 +protobuf==4.25.2 pyarrow==14.0.2 pycryptodome==3.20.0 pydantic==1.10.14 -Pygments==2.17.2 +pygments==2.17.2 pyhumps==3.8.0 pypika-tortoise==0.1.6 pysignalr==0.2.0 @@ -82,11 +83,11 @@ pytz==2023.3.post1 pyunormalize==15.1.0 pywin32==306; platform_system == "Windows" or sys_platform == "win32" pyyaml==6.0.1 -referencing==0.31.1 -regex==2023.10.3 +referencing==0.32.1 +regex==2023.12.25 requests==2.31.0 -rlp==3.0.0 -rpds-py==0.13.2 +rlp==4.0.0 +rpds-py==0.17.1 ruamel-yaml==0.18.5 ruamel-yaml-clib==0.2.8; platform_python_implementation == "CPython" and python_version < "3.13" ruff==0.1.14 @@ -95,28 +96,28 @@ setuptools==69.0.3 six==1.16.0 sniffio==1.3.0 snowballstemmer==2.2.0 -Sphinx==7.2.6 +sphinx==7.2.6 sphinx-click==5.1.0 sphinx-markdown-builder==0.6.6 -sphinxcontrib-applehelp==1.0.7 -sphinxcontrib-devhelp==1.0.5 -sphinxcontrib-htmlhelp==2.0.4 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.6 -sphinxcontrib-serializinghtml==1.1.9 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 sqlparse==0.4.4 strict-rfc3339==0.7 survey==4.5.4 tabulate==0.9.0 -toolz==0.12.0; implementation_name == "pypy" or implementation_name == "cpython" +toolz==0.12.1; implementation_name == "pypy" or implementation_name == "cpython" tortoise-orm==0.19.3 types-pytz==2023.3.1.1 types-tabulate==0.9.0.20240106 -typing-extensions==4.8.0 -tzdata==2023.3; platform_system == "Windows" +typing-extensions==4.9.0 +tzdata==2023.4; platform_system == "Windows" tzlocal==5.2 urllib3==2.1.0 watchdog==3.0.0 web3==6.11.2 websockets==10.4 -yarl==1.9.3 +yarl==1.9.4 diff --git a/requirements.txt b/requirements.txt index 43b8e3243..4f4995977 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,54 +6,54 @@ aiolimiter==1.1.0 aiosignal==1.3.1 aiosqlite==0.17.0 anyio==4.2.0 -APScheduler==3.10.4 -argcomplete==3.1.6 +apscheduler==3.10.4 +argcomplete==3.2.2 async-lru==2.0.4 async-timeout==4.0.3; python_version < "3.12.0" asyncclick==8.1.7.1 asyncpg==0.29.0 -attrs==23.1.0 -bitarray==2.8.3 -black==24.1a1 +attrs==23.2.0 +bitarray==2.9.2 +black==24.1.0 certifi==2023.11.17 charset-normalizer==3.3.2 click==8.1.7 -colorama==0.4.6; platform_system == "Windows" -cytoolz==0.12.2; implementation_name == "cpython" +colorama==0.4.6; platform_system == "Windows" or sys_platform == "win32" +cytoolz==0.12.3; implementation_name == "cpython" datamodel-code-generator==0.25.2 -dnspython==2.4.2; python_version ~= "3.11" +dnspython==2.5.0; python_version ~= "3.11" email-validator==2.1.0.post1; python_version ~= "3.11" eth-abi==4.2.1 eth-account==0.10.0 -eth-hash==0.5.2 -eth-keyfile==0.6.1 -eth-keys==0.4.0 -eth-rlp==0.3.0 -eth-typing==3.5.2 -eth-utils==2.3.1 -frozenlist==1.4.0 +eth-hash==0.6.0 +eth-keyfile==0.7.0 +eth-keys==0.5.0 +eth-rlp==1.0.1 +eth-typing==4.0.0 +eth-utils==3.0.0 +frozenlist==1.4.1 genson==1.2.2 hexbytes==0.3.1 idna==3.6 inflect==5.6.2 iso8601==1.1.0 -isort==5.12.0 -jinja2==3.1.2 -jsonschema==4.20.0 -jsonschema-specifications==2023.11.1 +isort==5.13.2 +jinja2==3.1.3 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 lru-dict==1.3.0 -MarkupSafe==2.1.3 +markupsafe==2.1.4 msgpack==1.0.7 multidict==6.0.4 mypy-extensions==1.0.0 -numpy==1.26.2 +numpy==1.26.3 orjson==3.9.12 packaging==23.2 parsimonious==0.9.0 -pathspec==0.11.2 -platformdirs==4.0.0 +pathspec==0.12.1 +platformdirs==4.1.0 prometheus-client==0.19.0 -protobuf==4.25.1 +protobuf==4.25.2 pyarrow==14.0.2 pycryptodome==3.20.0 pydantic==1.10.14 @@ -63,13 +63,13 @@ pysignalr==0.2.0 python-dotenv==1.0.1 pytz==2023.3.post1 pyunormalize==15.1.0 -pywin32==306; platform_system == "Windows" +pywin32==306; platform_system == "Windows" or sys_platform == "win32" pyyaml==6.0.1 -referencing==0.31.1 -regex==2023.10.3 +referencing==0.32.1 +regex==2023.12.25 requests==2.31.0 -rlp==3.0.0 -rpds-py==0.13.2 +rlp==4.0.0 +rpds-py==0.17.1 ruamel-yaml==0.18.5 ruamel-yaml-clib==0.2.8; platform_python_implementation == "CPython" and python_version < "3.13" sentry-sdk==1.39.2 @@ -80,12 +80,12 @@ sqlparse==0.4.4 strict-rfc3339==0.7 survey==4.5.4 tabulate==0.9.0 -toolz==0.12.0; implementation_name == "pypy" or implementation_name == "cpython" +toolz==0.12.1; implementation_name == "pypy" or implementation_name == "cpython" tortoise-orm==0.19.3 -typing-extensions==4.8.0 -tzdata==2023.3; platform_system == "Windows" +typing-extensions==4.9.0 +tzdata==2023.4; platform_system == "Windows" tzlocal==5.2 urllib3==2.1.0 web3==6.11.2 websockets==10.4 -yarl==1.9.3 +yarl==1.9.4 diff --git a/schema.json b/schema.json index 33e804e79..3e807b455 100644 --- a/schema.json +++ b/schema.json @@ -692,7 +692,15 @@ "type": "string" }, "ws_url": { - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null }, "http": { "anyOf": [ @@ -716,8 +724,7 @@ }, "required": [ "kind", - "url", - "ws_url" + "url" ] }, "SubsquidDatasourceConfig": { diff --git a/scripts/docs.py b/scripts/docs.py index 5f0372f58..6577a6c5e 100755 --- a/scripts/docs.py +++ b/scripts/docs.py @@ -299,21 +299,24 @@ def build(source: Path, destination: Path, watch: bool, serve: bool) -> None: '--source', type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True, path_type=Path), help='docs/ directory path to check.', + default='docs', ) -def check_links(source: Path) -> None: +@click.option('--http', is_flag=True, help='Check HTTP links too.') +def check_links(source: Path, http: bool) -> None: green_echo('=> Checking relative links') - files, links, http_links, bad_links, bad_anchors = 0, 0, 0, 0, 0 + files, links, bad_paths, bad_anchors, bad_http = 0, 0, 0, 0, 0 + http_links: set[str] = set() for path in source.rglob('*.md'): - logging.info('checking file `%s`', path) + _logger.info('checking file `%s`', path) files += 1 data = path.read_text() for match in re.finditer(MD_LINK_REGEX, data): links += 1 link = match.group(1) - # TODO: Check them too? + if link.startswith('http'): - http_links += 1 + http_links.add(link) continue link, anchor = link.split('#') if '#' in link else (link, None) @@ -321,7 +324,7 @@ def check_links(source: Path) -> None: full_path = path.parent.joinpath(link) if not full_path.exists(): logging.error('broken link: `%s`', full_path) - bad_links += 1 + bad_paths += 1 continue if anchor: @@ -336,10 +339,30 @@ def check_links(source: Path) -> None: bad_anchors += 1 continue - logging.info('_' * 80) - logging.info('checked %d files and %d links:', files, links) - logging.info('%d URLs, %d bad links, %d bad anchors', http_links, bad_links, bad_anchors) - if bad_links or bad_anchors: + if http: + green_echo('=> Checking HTTP links') + + for link in http_links: + green_echo(f'checking link `{link}`') + try: + res = subprocess.run( + ('curl', '-s', '-L', '-o', '/dev/null', '-w', '%{http_code}', link), + check=True, + capture_output=True, + ) + status_code = int(res.stdout.decode().strip()) + if status_code != 200: + raise subprocess.CalledProcessError(status_code, 'curl') + except subprocess.CalledProcessError: + red_echo(f'broken http link: `{status_code}`') + bad_http += 1 + + _logger.info('_' * 80) + _logger.info('checked %d files and %d links:', files, links) + _logger.info('paths: %d bad links, %d bad anchors', bad_paths, bad_anchors) + _logger.info('http: %d bad links', bad_http) + + if bad_paths or bad_anchors or bad_http: red_echo('=> Fix broken links and try again') exit(1) @@ -365,6 +388,7 @@ def dump_jsonschema() -> None: schema_path = Path(__file__).parent.parent / 'schema.json' schema_path.write_bytes(orjson.dumps(schema_dict, option=orjson.OPT_INDENT_2)) + @main.command('dump-references', help='Dump Sphinx references to ugly Markdown files') def dump_references() -> None: green_echo('=> Dumping Sphinx references') diff --git a/src/demo_evm_events/deploy/.env.default b/src/demo_evm_events/deploy/.env.default index 6d33753fb..94d783e16 100644 --- a/src/demo_evm_events/deploy/.env.default +++ b/src/demo_evm_events/deploy/.env.default @@ -1,13 +1,12 @@ # This env file was generated automatically by DipDup. Do not edit it! # Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. # -ARCHIVE_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet -ETHERSCAN_API_KEY=YourApiKeyToken -ETHERSCAN_API_URL=https://api.etherscan.io/api +ETHERSCAN_API_KEY='' +ETHERSCAN_URL=https://api.etherscan.io/api HASURA_HOST=hasura HASURA_SECRET= NODE_API_KEY='' -NODE_RPC_URL=https://eth-mainnet.g.alchemy.com/v2 +NODE_URL=https://eth-mainnet.g.alchemy.com/v2 NODE_WS_URL=wss://eth-mainnet.g.alchemy.com/v2 POSTGRES_DB=dipdup POSTGRES_HOST=db @@ -15,3 +14,4 @@ POSTGRES_PASSWORD= POSTGRES_USER=dipdup SENTRY_DSN="" SENTRY_ENVIRONMENT="" +SUBSQUID_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet diff --git a/src/demo_evm_events/deploy/sqlite.env.default b/src/demo_evm_events/deploy/sqlite.env.default index e7700151f..a8d8f87e0 100644 --- a/src/demo_evm_events/deploy/sqlite.env.default +++ b/src/demo_evm_events/deploy/sqlite.env.default @@ -1,10 +1,10 @@ # This env file was generated automatically by DipDup. Do not edit it! # Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. # -ARCHIVE_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet -ETHERSCAN_API_KEY=YourApiKeyToken -ETHERSCAN_API_URL=https://api.etherscan.io/api +ETHERSCAN_API_KEY='' +ETHERSCAN_URL=https://api.etherscan.io/api NODE_API_KEY='' -NODE_RPC_URL=https://eth-mainnet.g.alchemy.com/v2 +NODE_URL=https://eth-mainnet.g.alchemy.com/v2 NODE_WS_URL=wss://eth-mainnet.g.alchemy.com/v2 SQLITE_PATH=/tmp/demo_evm_events.sqlite +SUBSQUID_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet diff --git a/src/demo_evm_events/deploy/swarm.env.default b/src/demo_evm_events/deploy/swarm.env.default index 474aa6b18..ca87e34ee 100644 --- a/src/demo_evm_events/deploy/swarm.env.default +++ b/src/demo_evm_events/deploy/swarm.env.default @@ -1,13 +1,12 @@ # This env file was generated automatically by DipDup. Do not edit it! # Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. # -ARCHIVE_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet -ETHERSCAN_API_KEY=YourApiKeyToken -ETHERSCAN_API_URL=https://api.etherscan.io/api +ETHERSCAN_API_KEY='' +ETHERSCAN_URL=https://api.etherscan.io/api HASURA_HOST=demo_evm_events_hasura HASURA_SECRET= NODE_API_KEY='' -NODE_RPC_URL=https://eth-mainnet.g.alchemy.com/v2 +NODE_URL=https://eth-mainnet.g.alchemy.com/v2 NODE_WS_URL=wss://eth-mainnet.g.alchemy.com/v2 POSTGRES_DB=dipdup POSTGRES_HOST=demo_evm_events_db @@ -15,3 +14,4 @@ POSTGRES_PASSWORD= POSTGRES_USER=dipdup SENTRY_DSN="" SENTRY_ENVIRONMENT="" +SUBSQUID_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet diff --git a/src/demo_evm_events/dipdup.yaml b/src/demo_evm_events/dipdup.yaml index d6f42a850..d5dbd6cec 100644 --- a/src/demo_evm_events/dipdup.yaml +++ b/src/demo_evm_events/dipdup.yaml @@ -2,21 +2,19 @@ spec_version: 2.0 package: demo_evm_events datasources: - ethscan: + subsquid: + kind: evm.subsquid + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: evm_node + etherscan: kind: abi.etherscan - url: ${ETHERSCAN_API_URL:-https://api.etherscan.io/api} - api_key: ${ETHERSCAN_API_KEY:-YourApiKeyToken} - - mainnet_node: + url: ${ETHERSCAN_URL:-https://api.etherscan.io/api} + api_key: ${ETHERSCAN_API_KEY:-''} + evm_node: kind: evm.node - url: ${NODE_RPC_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} - mainnet_subsquid: - kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} - node: mainnet_node - contracts: eth_usdt: kind: evm @@ -26,7 +24,7 @@ contracts: indexes: eth_usdt_events: kind: evm.subsquid.events - datasource: mainnet_subsquid + datasource: subsquid handlers: - callback: on_transfer contract: eth_usdt diff --git a/src/demo_uniswap/deploy/.env.default b/src/demo_uniswap/deploy/.env.default index b27f07c1d..94d783e16 100644 --- a/src/demo_uniswap/deploy/.env.default +++ b/src/demo_uniswap/deploy/.env.default @@ -1,13 +1,17 @@ # This env file was generated automatically by DipDup. Do not edit it! # Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. # -ALCHEMY_KEY='' -ARCHIVE_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet +ETHERSCAN_API_KEY='' +ETHERSCAN_URL=https://api.etherscan.io/api HASURA_HOST=hasura HASURA_SECRET= +NODE_API_KEY='' +NODE_URL=https://eth-mainnet.g.alchemy.com/v2 +NODE_WS_URL=wss://eth-mainnet.g.alchemy.com/v2 POSTGRES_DB=dipdup POSTGRES_HOST=db POSTGRES_PASSWORD= POSTGRES_USER=dipdup SENTRY_DSN="" SENTRY_ENVIRONMENT="" +SUBSQUID_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet diff --git a/src/demo_uniswap/deploy/sqlite.env.default b/src/demo_uniswap/deploy/sqlite.env.default index 8045bd5dc..ce2d7d32b 100644 --- a/src/demo_uniswap/deploy/sqlite.env.default +++ b/src/demo_uniswap/deploy/sqlite.env.default @@ -1,6 +1,10 @@ # This env file was generated automatically by DipDup. Do not edit it! # Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. # -ALCHEMY_KEY='' -ARCHIVE_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet +ETHERSCAN_API_KEY='' +ETHERSCAN_URL=https://api.etherscan.io/api +NODE_API_KEY='' +NODE_URL=https://eth-mainnet.g.alchemy.com/v2 +NODE_WS_URL=wss://eth-mainnet.g.alchemy.com/v2 SQLITE_PATH=/tmp/demo_uniswap.sqlite +SUBSQUID_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet diff --git a/src/demo_uniswap/deploy/swarm.env.default b/src/demo_uniswap/deploy/swarm.env.default index 8a61a0f8c..f66a68e37 100644 --- a/src/demo_uniswap/deploy/swarm.env.default +++ b/src/demo_uniswap/deploy/swarm.env.default @@ -1,13 +1,17 @@ # This env file was generated automatically by DipDup. Do not edit it! # Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. # -ALCHEMY_KEY='' -ARCHIVE_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet +ETHERSCAN_API_KEY='' +ETHERSCAN_URL=https://api.etherscan.io/api HASURA_HOST=demo_uniswap_hasura HASURA_SECRET= +NODE_API_KEY='' +NODE_URL=https://eth-mainnet.g.alchemy.com/v2 +NODE_WS_URL=wss://eth-mainnet.g.alchemy.com/v2 POSTGRES_DB=dipdup POSTGRES_HOST=demo_uniswap_db POSTGRES_PASSWORD= POSTGRES_USER=dipdup SENTRY_DSN="" SENTRY_ENVIRONMENT="" +SUBSQUID_URL=https://v2.archive.subsquid.io/network/ethereum-mainnet diff --git a/src/demo_uniswap/dipdup.yaml b/src/demo_uniswap/dipdup.yaml index 4b999c988..68a01b216 100644 --- a/src/demo_uniswap/dipdup.yaml +++ b/src/demo_uniswap/dipdup.yaml @@ -2,19 +2,18 @@ spec_version: 2.0 package: demo_uniswap datasources: - ethscan: + subsquid: + kind: evm.subsquid + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: evm_node + etherscan: kind: abi.etherscan - - mainnet_node: + url: ${ETHERSCAN_URL:-https://api.etherscan.io/api} + api_key: ${ETHERSCAN_API_KEY:-''} + evm_node: kind: evm.node - url: https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} - ws_url: wss://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} - - mainnet_subsquid: - kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} - node: - - mainnet_node + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} contracts: factory: @@ -71,7 +70,7 @@ indexes: uniswap_mainnet: template: uniswap_v3_factory values: - datasource: mainnet_subsquid + datasource: subsquid first_level: 12369521 advanced: diff --git a/src/demo_uniswap/handlers/factory/pool_created.py b/src/demo_uniswap/handlers/factory/pool_created.py index cee638133..9e385d70e 100644 --- a/src/demo_uniswap/handlers/factory/pool_created.py +++ b/src/demo_uniswap/handlers/factory/pool_created.py @@ -19,7 +19,7 @@ async def create_token(ctx: HandlerContext, address: str, pool_id: str) -> None: await models.Token.cached_get(address) return - web3 = ctx.get_evm_node_datasource('mainnet_subsquid').web3 + web3 = ctx.get_evm_node_datasource('subsquid').web3 erc20_iface = ERC20Token.from_address(web3, address) token = models.Token( id=address, diff --git a/src/demo_uniswap/models/position.py b/src/demo_uniswap/models/position.py index 0b9da3085..9df6aee70 100644 --- a/src/demo_uniswap/models/position.py +++ b/src/demo_uniswap/models/position.py @@ -9,7 +9,7 @@ async def position_validate( ctx: HandlerContext, contract_address: str, position_id: int, position: models.Position, ) -> None: - web3 = ctx.get_evm_node_datasource('mainnet_subsquid').web3 + web3 = ctx.get_evm_node_datasource('subsquid').web3 manager = web3.eth.contract( address=to_checksum_address(contract_address), abi=get_abi('position_manager.abi'), diff --git a/src/dipdup/config/evm_node.py b/src/dipdup/config/evm_node.py index c67966f04..15534eeb0 100644 --- a/src/dipdup/config/evm_node.py +++ b/src/dipdup/config/evm_node.py @@ -23,7 +23,7 @@ class EvmNodeDatasourceConfig(IndexDatasourceConfig): kind: Literal['evm.node'] url: str - ws_url: str + ws_url: str | None = None http: HttpConfig | None = None rollback_depth: int = 32 @@ -38,7 +38,7 @@ def _valid_url(cls, v: str) -> str: return v @validator('ws_url') - def _valid_ws_url(cls, v: str) -> str: - if not v.startswith(('ws://', 'wss://')): + def _valid_ws_url(cls, v: str | None) -> str | None: + if v and not v.startswith(('ws://', 'wss://')): raise ConfigurationError('Ethereum node WebSocket URL must start with ws(s)://') return v diff --git a/src/dipdup/config/evm_subsquid.py b/src/dipdup/config/evm_subsquid.py index 3c3fc73e7..664875be8 100644 --- a/src/dipdup/config/evm_subsquid.py +++ b/src/dipdup/config/evm_subsquid.py @@ -17,7 +17,7 @@ class SubsquidDatasourceConfig(IndexDatasourceConfig): """Subsquid datasource config :param kind: always 'evm.subsquid' - :param url: URL of Subsquid Archives API + :param url: URL of Subsquid Network API :param node: One or more `evm.node` datasource(s) for the same network :param http: HTTP client configuration """ diff --git a/src/dipdup/config/evm_subsquid_events.py b/src/dipdup/config/evm_subsquid_events.py index 55f2ac254..cda85a321 100644 --- a/src/dipdup/config/evm_subsquid_events.py +++ b/src/dipdup/config/evm_subsquid_events.py @@ -58,7 +58,7 @@ class SubsquidEventsIndexConfig(IndexConfig): :param datasource: Subsquid datasource :param handlers: Event handlers :param abi: One or more `evm.abi` datasource(s) for the same network - :param node_only: Don't use Subsquid Archives API (dev only) + :param node_only: Don't use Subsquid Network API (dev only) :param first_level: Level to start indexing from :param last_level: Level to stop indexing and disable this index """ diff --git a/src/dipdup/datasources/evm_node.py b/src/dipdup/datasources/evm_node.py index 2ab69c83f..ee7f71a2e 100644 --- a/src/dipdup/datasources/evm_node.py +++ b/src/dipdup/datasources/evm_node.py @@ -38,6 +38,7 @@ from dipdup.utils import Watchdog WEB3_CACHE_SIZE = 256 +POLL_INTERVAL = 5 EmptyCallback = Callable[[], Awaitable[None]] @@ -62,7 +63,6 @@ def __init__(self, config: EvmNodeDatasourceConfig, merge_subscriptions: bool = super().__init__(config, merge_subscriptions) self._web3_client: AsyncWeb3 | None = None self._ws_client: WebsocketTransport | None = None - self._use_realtime: asyncio.Event = asyncio.Event() self._requests: dict[str, tuple[asyncio.Event, Any]] = {} self._subscription_ids: dict[str, EvmNodeSubscription] = {} self._logs_queue: Queue[dict[str, Any]] = Queue() @@ -107,12 +107,17 @@ async def make_request(_, method: str, params: list[Any]) -> Any: self.set_sync_level(None, level) async def run(self) -> None: - await self._use_realtime.wait() - await asyncio.gather( - self._ws_loop(), - self._log_processor_loop(), - self._watchdog.run(), - ) + if self.realtime: + await asyncio.gather( + self._ws_loop(), + self._log_processor_loop(), + self._watchdog.run(), + ) + else: + while True: + level = await self.get_head_level() + self.set_sync_level(None, level) + await asyncio.sleep(POLL_INTERVAL) async def _log_processor_loop(self) -> None: while True: @@ -149,11 +154,12 @@ async def _ws_loop(self) -> None: raise DatasourceError('Websocket connection failed', self.name) - def use_realtime(self) -> None: - self._use_realtime.set() + @property + def realtime(self) -> bool: + return self._config.ws_url is not None async def subscribe(self) -> None: - if not self._use_realtime.is_set(): + if not self.realtime: return missing_subscriptions = self._subscriptions.missing_subscriptions @@ -358,6 +364,8 @@ def _get_ws_client(self) -> WebsocketTransport: self._logger.debug('Creating Websocket client') url = self._config.ws_url + if not url: + raise FrameworkException('Spawning node datasource, but `ws_url` is not set') self._ws_client = WebsocketTransport( url=url, protocol=WebsocketProtocol(), diff --git a/src/dipdup/datasources/evm_subsquid.py b/src/dipdup/datasources/evm_subsquid.py index 2d8e6f6a9..b0b147489 100644 --- a/src/dipdup/datasources/evm_subsquid.py +++ b/src/dipdup/datasources/evm_subsquid.py @@ -125,7 +125,7 @@ async def initialize(self) -> None: level = await self.get_head_level() if not level: - raise DatasourceError('Archive is not ready yet', self.name) + raise DatasourceError('Subsquid is not ready yet', self.name) self.set_sync_level(None, level) diff --git a/src/dipdup/exceptions.py b/src/dipdup/exceptions.py index 14eb03aef..a89de9994 100644 --- a/src/dipdup/exceptions.py +++ b/src/dipdup/exceptions.py @@ -54,11 +54,13 @@ def help(self) -> str: @classmethod def default_help(cls) -> str: - return format_help(""" + return format_help( + """ An unexpected error has occurred! Most likely it's a framework bug. Please, tell us about it: https://github.com/dipdup-io/dipdup/issues - """) + """ + ) @abstractmethod def _help(self) -> str: ... diff --git a/src/dipdup/hasura.py b/src/dipdup/hasura.py index a9d594391..ef3a0cdac 100644 --- a/src/dipdup/hasura.py +++ b/src/dipdup/hasura.py @@ -81,7 +81,11 @@ } } } -""".replace('\n', ' ').replace(' ', '') +""".replace( + '\n', ' ' +).replace( + ' ', '' +) @dataclass diff --git a/src/dipdup/indexes/evm_subsquid_events/index.py b/src/dipdup/indexes/evm_subsquid_events/index.py index 6a62a0dfd..2cffd1025 100644 --- a/src/dipdup/indexes/evm_subsquid_events/index.py +++ b/src/dipdup/indexes/evm_subsquid_events/index.py @@ -42,7 +42,6 @@ def __init__( ) -> None: super().__init__(ctx, config, datasource) self._node_datasources: tuple[EvmNodeDatasource, ...] | None = None - self._realtime_node: EvmNodeDatasource | None = None self._topics: dict[str, dict[str, str]] | None = None @property @@ -66,13 +65,6 @@ def random_node(self) -> EvmNodeDatasource: raise FrameworkException('A node datasource requested, but none attached to this index') return random.choice(self.node_datasources) - @property - def realtime_node(self) -> EvmNodeDatasource: - if self._realtime_node is None: - self._realtime_node = self.random_node - self._realtime_node.use_realtime() - return self._realtime_node - @property def topics(self) -> dict[str, dict[str, str]]: if self._topics is None: @@ -141,7 +133,7 @@ async def _synchronize(self, sync_level: int) -> None: use_node = False if self.node_datasources: - node_sync_level = await self.realtime_node.get_head_level() + node_sync_level = await self.random_node.get_head_level() subsquid_lag = abs(node_sync_level - subsquid_sync_level) subsquid_available = subsquid_sync_level - index_level self._logger.info('Subsquid is %s levels behind; %s available', subsquid_lag, subsquid_available) diff --git a/src/dipdup/projects/demo_evm_events/dipdup.yaml.j2 b/src/dipdup/projects/demo_evm_events/dipdup.yaml.j2 index 09d7bf71f..e5e01aad4 100644 --- a/src/dipdup/projects/demo_evm_events/dipdup.yaml.j2 +++ b/src/dipdup/projects/demo_evm_events/dipdup.yaml.j2 @@ -2,21 +2,19 @@ spec_version: 2.0 package: {{ project.package }} datasources: - ethscan: + subsquid: + kind: evm.subsquid + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: evm_node + etherscan: kind: abi.etherscan - url: ${ETHERSCAN_API_URL:-https://api.etherscan.io/api} - api_key: ${ETHERSCAN_API_KEY:-YourApiKeyToken} - - mainnet_node: + url: ${ETHERSCAN_URL:-https://api.etherscan.io/api} + api_key: ${ETHERSCAN_API_KEY:-''} + evm_node: kind: evm.node - url: ${NODE_RPC_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} - mainnet_subsquid: - kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} - node: mainnet_node - contracts: eth_usdt: kind: evm @@ -26,7 +24,7 @@ contracts: indexes: eth_usdt_events: kind: evm.subsquid.events - datasource: mainnet_subsquid + datasource: subsquid handlers: - callback: on_transfer contract: eth_usdt diff --git a/src/dipdup/projects/demo_uniswap/dipdup.yaml.j2 b/src/dipdup/projects/demo_uniswap/dipdup.yaml.j2 index da63acdc5..788a5cd4d 100644 --- a/src/dipdup/projects/demo_uniswap/dipdup.yaml.j2 +++ b/src/dipdup/projects/demo_uniswap/dipdup.yaml.j2 @@ -2,19 +2,18 @@ spec_version: 2.0 package: {{ project.package }} datasources: - ethscan: + subsquid: + kind: evm.subsquid + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: evm_node + etherscan: kind: abi.etherscan - - mainnet_node: + url: ${ETHERSCAN_URL:-https://api.etherscan.io/api} + api_key: ${ETHERSCAN_API_KEY:-''} + evm_node: kind: evm.node - url: https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} - ws_url: wss://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} - - mainnet_subsquid: - kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} - node: - - mainnet_node + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} contracts: factory: @@ -71,7 +70,7 @@ indexes: uniswap_mainnet: template: uniswap_v3_factory values: - datasource: mainnet_subsquid + datasource: subsquid first_level: 12369521 advanced: diff --git a/src/dipdup/projects/demo_uniswap/handlers/factory/pool_created.py.j2 b/src/dipdup/projects/demo_uniswap/handlers/factory/pool_created.py.j2 index 614ca2c8f..8dc5e7d37 100644 --- a/src/dipdup/projects/demo_uniswap/handlers/factory/pool_created.py.j2 +++ b/src/dipdup/projects/demo_uniswap/handlers/factory/pool_created.py.j2 @@ -20,7 +20,7 @@ async def create_token(ctx: HandlerContext, address: str, pool_id: str) -> None: await models.Token.cached_get(address) return None - web3 = ctx.get_evm_node_datasource('mainnet_subsquid').web3 + web3 = ctx.get_evm_node_datasource('subsquid').web3 erc20_iface = ERC20Token.from_address(web3, address) token = models.Token( id=address, diff --git a/src/dipdup/projects/demo_uniswap/models/position.py.j2 b/src/dipdup/projects/demo_uniswap/models/position.py.j2 index 5e0dce09f..c305e1a9f 100644 --- a/src/dipdup/projects/demo_uniswap/models/position.py.j2 +++ b/src/dipdup/projects/demo_uniswap/models/position.py.j2 @@ -9,7 +9,7 @@ from dipdup.context import HandlerContext async def position_validate( ctx: HandlerContext, contract_address: str, position_id: int, position: models.Position, ) -> None: - web3 = ctx.get_evm_node_datasource('mainnet_subsquid').web3 + web3 = ctx.get_evm_node_datasource('subsquid').web3 manager = web3.eth.contract( address=to_checksum_address(contract_address), abi=get_abi('position_manager.abi'), diff --git a/src/dipdup/prometheus.py b/src/dipdup/prometheus.py index 7a807a7c7..5fb418c14 100644 --- a/src/dipdup/prometheus.py +++ b/src/dipdup/prometheus.py @@ -60,15 +60,15 @@ _sqd_processor_last_block = Gauge( 'sqd_processor_last_block', - 'Level of the last processed block from Subsquid Archives', + 'Level of the last processed block from Subsquid Network', ) _sqd_processor_chain_height = Gauge( 'sqd_processor_chain_height', - 'Current chain height as reported by Subsquid Archives', + 'Current chain height as reported by Subsquid Network', ) _sqd_processor_archive_http_errors_in_row = Histogram( 'sqd_processor_archive_http_errors_in_row', - 'Number of consecutive failed requests to Subsquid Archives', + 'Number of consecutive failed requests to Subsquid Network', ) diff --git a/tests/configs/demo_evm_events.yml b/tests/configs/demo_evm_events.yml index 677aefc86..3b9d036df 100644 --- a/tests/configs/demo_evm_events.yml +++ b/tests/configs/demo_evm_events.yml @@ -2,18 +2,18 @@ spec_version: 2.0 package: demo_evm_events datasources: - ethscan: + etherscan: kind: abi.etherscan - mainnet_node: + evm_node: kind: evm.node - url: https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} - ws_url: wss://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} - mainnet_subsquid: + subsquid: kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} - node: mainnet_node + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: evm_node http: replay_path: ${DIPDUP_REPLAY_PATH:-} @@ -26,7 +26,7 @@ contracts: indexes: eth_usdt_events: kind: evm.subsquid.events - datasource: mainnet_subsquid + datasource: subsquid handlers: - callback: on_transfer contract: eth_usdt diff --git a/tests/configs/demo_evm_events_node.yml b/tests/configs/demo_evm_events_node.yml index 51b83c648..eef2c6afe 100644 --- a/tests/configs/demo_evm_events_node.yml +++ b/tests/configs/demo_evm_events_node.yml @@ -2,18 +2,18 @@ spec_version: 2.0 package: demo_evm_events datasources: - ethscan: + etherscan: kind: abi.etherscan - mainnet_node: + evm_node: kind: evm.node - url: https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} - ws_url: wss://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} + ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} - mainnet_subsquid: + subsquid: kind: evm.subsquid - url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} - node: mainnet_node + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: evm_node http: replay_path: ${DIPDUP_REPLAY_PATH:-} @@ -26,7 +26,7 @@ contracts: indexes: eth_usdt_events: kind: evm.subsquid.events - datasource: mainnet_subsquid + datasource: subsquid handlers: - callback: on_transfer contract: eth_usdt diff --git a/tests/test_config/test_custom_config.py b/tests/test_config/test_custom_config.py index 45bdfab38..ad7007990 100644 --- a/tests/test_config/test_custom_config.py +++ b/tests/test_config/test_custom_config.py @@ -24,14 +24,18 @@ def appended_config_path(dummy_config_path: str, tmp_path_factory: TempPathFacto @pytest.fixture( scope='session', - params=([""" + params=( + [ + """ custom: foo: bar spam: - eggs - rice -"""]), +""" + ] + ), ) def config_with_custom_section_path( self, dummy_config_path: str, tmp_path_factory: TempPathFactory, request: Any