feat: migrate indexer (#951)

* import the indexer to rpc

* update dockerfiles and docker composes

* fix

* fix dockerfiles and docker composes

.vscode/settings.json

@@ -4,5 +4,12 @@
     "**/.git": false
   },
   "rust-analyzer.check.command": "clippy",
-  "rust-analyzer.cargo.features": ["testing", "hive"]
+  "rust-analyzer.cargo.features": ["testing", "hive"],
+  "deno.enable": true,
+  "deno.suggest.imports.autoDiscover": true,
+  "[typescript]": {
+    "editor.defaultFormatter": "trunk.io"
+  },
+  "deno.lint": true,
+  "deno.unstable": true
 }

docker-compose.prod.yaml

@@ -80,6 +80,9 @@ services:
       - UNINITIALIZED_ACCOUNT_CLASS_HASH=0x1d8b8047e26b484d3f6262d1967217d980d0f2dfc69afa5661492bd5bfe2954
       - ACCOUNT_CONTRACT_CLASS_HASH=0x56d311021950bf65ee500426e007b9e3ced0db97f9c1e0d29a9e03d79a9bf6c
     restart: on-failure
+    volumes:
+      # Mount the indexer code
+      - indexer_code:/usr/src/indexer
     depends_on:
       starknet:
         condition: service_started
@@ -110,16 +113,11 @@ services:
       MONGO_INITDB_ROOT_USERNAME: mongo
       MONGO_INITDB_ROOT_PASSWORD: mongo
 
-  clone-repo:
-    extends:
-      file: docker-compose.yaml
-      service: clone-repo
-
   indexer:
     image: quay.io/apibara/sink-mongo:0.7.0
     command:
       - run
-      - /code/kakarot-indexer/src/main.ts
+      - /indexer/src/main.ts
     environment:
       # Whitelist environment variables
       - ALLOW_ENV_FROM_ENV=DEBUG,APIBARA_AUTH_TOKEN,STARTING_BLOCK,STREAM_URL,SINK_TYPE,MONGO_CONNECTION_STRING,MONGO_DATABASE_NAME,STARKNET_NETWORK,KAKAROT_ADDRESS,ALLOW_NET,MONGO_REPLACE_DATA_INSIDE_TRANSACTION
@@ -136,12 +134,10 @@ services:
       - KAKAROT_ADDRESS=0x612fb5de32723a19b073b3aba348e48a3d9f51448426d8931694e51db5795e6
     restart: on-failure
     volumes:
-      - indexer_code:/code
+      - indexer_code:/indexer
     networks:
       - internal
     depends_on:
-      clone-repo:
-        condition: service_completed_successfully
       starknet:
         condition: service_started
 

docker-compose.staging.yaml

@@ -74,6 +74,9 @@ services:
       - UNINITIALIZED_ACCOUNT_CLASS_HASH=0x1d8b8047e26b484d3f6262d1967217d980d0f2dfc69afa5661492bd5bfe2954
       - ACCOUNT_CONTRACT_CLASS_HASH=0x56d311021950bf65ee500426e007b9e3ced0db97f9c1e0d29a9e03d79a9bf6c
     restart: on-failure
+    volumes:
+      # Mount the indexer code
+      - indexer_code:/usr/src/indexer
     depends_on:
       starknet:
         condition: service_started
@@ -104,16 +107,11 @@ services:
       MONGO_INITDB_ROOT_USERNAME: mongo
       MONGO_INITDB_ROOT_PASSWORD: mongo
 
-  clone-repo:
-    extends:
-      file: docker-compose.yaml
-      service: clone-repo
-
   indexer:
     image: quay.io/apibara/sink-mongo:0.7.0
     command:
       - run
-      - /code/kakarot-indexer/src/main.ts
+      - /indexer/src/main.ts
     environment:
       # Whitelist environment variables
       - ALLOW_ENV_FROM_ENV=DEBUG,APIBARA_AUTH_TOKEN,STARTING_BLOCK,STREAM_URL,SINK_TYPE,MONGO_CONNECTION_STRING,MONGO_DATABASE_NAME,STARKNET_NETWORK,KAKAROT_ADDRESS,ALLOW_NET,MONGO_REPLACE_DATA_INSIDE_TRANSACTION
@@ -130,12 +128,10 @@ services:
       - KAKAROT_ADDRESS=0x464f7e37179d2f93ea208795bdb2d0912e8257f6fb5f67ae2559251523aee19
     restart: on-failure
     volumes:
-      - indexer_code:/code
+      - indexer_code:/indexer
     networks:
       - internal
     depends_on:
-      clone-repo:
-        condition: service_completed_successfully
       starknet:
         condition: service_started
 

docker-compose.yaml

@@ -79,6 +79,8 @@ services:
     volumes:
       # Mount the volume on workdir and use .env stored in root of the volume
       - deployments:/usr/src/app
+      # Mount the indexer code
+      - indexer_code:/usr/src/indexer
     depends_on:
       deployments-parser:
         condition: service_completed_successfully
@@ -115,22 +117,11 @@ services:
       MONGO_INITDB_ROOT_USERNAME: mongo
       MONGO_INITDB_ROOT_PASSWORD: mongo
 
-  clone-repo:
-    image: docker.io/alpine/git:latest
-    entrypoint: /bin/sh
-    command:
-      - -c
-      # clone the repository in `/code`, removing any old copy.
-      - cd /code && rm -rf kakarot-indexer && git clone -v https://github.com/kkrt-labs/kakarot-indexer.git
-    volumes:
-      - indexer_code:/code
-    restart: on-failure
-
   indexer:
     image: quay.io/apibara/sink-mongo:0.7.0
     command:
       - run
-      - /code/kakarot-indexer/src/main.ts
+      - /indexer/src/main.ts
     environment:
       # Whitelist environment variables
       - ALLOW_ENV_FROM_ENV=DEBUG,APIBARA_AUTH_TOKEN,STARTING_BLOCK,STREAM_URL,SINK_TYPE,MONGO_CONNECTION_STRING,MONGO_DATABASE_NAME,STARKNET_NETWORK,KAKAROT_ADDRESS,ALLOW_NET,MONGO_REPLACE_DATA_INSIDE_TRANSACTION
@@ -147,13 +138,11 @@ services:
       - ALLOW_ENV=/deployments/.env
     restart: on-failure
     volumes:
-      - indexer_code:/code
+      - indexer_code:/indexer
       - deployments:/deployments
     networks:
       - internal
     depends_on:
-      clone-repo:
-        condition: service_completed_successfully
       starknet:
         condition: service_started
       deployments-parser:

docker/hive/Dockerfile

@@ -42,12 +42,6 @@ RUN case $BUILDPLATFORM in \
     ;; \
     esac
 
-### Indexer transform plugin
-#### First, clone the indexer repository
-FROM docker.io/alpine/git:latest as indexer-cloner
-WORKDIR /code
-RUN git clone "https://github.com/kkrt-labs/kakarot-indexer.git"
-
 #### MongoDB
 FROM mongo:6.0.8 as mongo
 
@@ -165,7 +159,7 @@ COPY --from=katana /usr/local/bin/katana /usr/local/bin
 COPY --from=apibara-build /usr/local/bin/starknet /usr/local/bin/starknet
 
 #### We need the indexer typescript code and the binary that knows how to run it
-COPY --from=indexer-cloner /code /usr/src/app/code
+COPY ./indexer /usr/src/app/code/indexer
 COPY --from=apibara-build /usr/local/bin/sink-mongo /usr/local/bin/sink-mongo
 
 #### We need the mongo binary

docker/hive/start.sh

@@ -50,7 +50,7 @@ echo "Launching DNA..."
 starknet start --rpc=http://localhost:5050 --wait-for-rpc --head-refresh-interval-ms=500 --data=/data &
 # ## Indexer
 echo "Launching indexer..."
-sink-mongo run /usr/src/app/code/kakarot-indexer/src/main.ts &
+sink-mongo run /usr/src/app/code/indexer/src/main.ts &
 
 ### 3.5. Await the Indexer to be healthy
 echo "Waiting for the indexer to start..."

docker/rpc/Dockerfile

@@ -41,6 +41,9 @@ RUN apt-get update && \
     apt-get install -y libssl-dev ca-certificates tini curl && \ 
     rm -rf /var/lib/apt/lists/*
 
+# Copy the indexer code into the RPC
+COPY ./indexer /usr/src/indexer
+
 # Seen in https://github.com/eqlabs/pathfinder/blob/4ab915a830953ed6f02af907937b46cb447d9a92/Dockerfile#L120 - 
 # Allows for passing args down to the underlying binary easily
 ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/kakarot-rpc"]

docker/rpc/Dockerfile.cross

@@ -32,6 +32,9 @@ RUN apt-get update && \
     apt-get install -y libssl-dev ca-certificates tini curl && \ 
     rm -rf /var/lib/apt/lists/*
 
+# Copy the indexer code into the RPC
+COPY ./indexer /usr/src/indexer
+
 # Seen in https://github.com/eqlabs/pathfinder/blob/4ab915a830953ed6f02af907937b46cb447d9a92/Dockerfile#L120 - 
 # Allows for passing args down to the underlying binary easily
 ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/kakarot-rpc"]

indexer/.env.example

@@ -0,0 +1,16 @@
+APIBARA_AUTH_TOKEN=
+DEBUG=
+
+# e.g. http://localhost:7171 or https://goerli.starknet.a5a.ch
+STREAM_URL=http://localhost:7171
+STARTING_BLOCK=0
+
+# one of: console, mongo
+SINK_TYPE=console
+
+# if SINK_TYPE=mongo, then this is required, e.g. mongodb://mongo:mongo@mongo:27017
+MONGO_CONNECTION_STRING=
+MONGO_DATABASE_NAME=
+STARKNET_NETWORK=
+KAKAROT_ADDRESS=
+ALLOW_NET=

indexer/README.md

@@ -0,0 +1,69 @@
+# Kakarot Indexer
+
+Kakarot Indexer fits in the three-part architecture of the Kakarot zkEVM rollup
+(Kakarot EVM Cairo Programs, Kakarot RPC, Kakarot Indexer). It indexes the
+underlying CairoVM network on which Kakarot core is deployed in an Ethereum
+compatible format.
+
+<p align="center" style="margin-top: 25px; margin-bottom: 25px;">
+    <img src="./docs/kakarot_indexer.png" alt="Kakarot Indexer" width="80%"
+    style="border-radius: 10px;">
+</p>
+
+## Table of Contents
+
+- [Motivation and FAQ](#motivation-and-faq)
+- [Installation](#installation)
+- [Usage](#usage)
+- [License](#license)
+
+## Motivation and FAQ
+
+_Why is an indexer needed?_
+
+- Because CairoVM chains (also called StarknetOS chains) have different data
+  types, it is necessary to reformat the data in an Ethereum compatible way. We
+  decided to perform this reformatting outside of the CairoVM chain, resulting
+  in duplication of data, but more modularity. Note that this reformatting
+  could've been done at runtime, at a high computational cost. Lastly, a third
+  option would consist in forking a CairoVM client (e.g. Starknet) and change
+  the data types directly within the storage of the full node.
+
+_Why not directly modify a CairoVM client?_
+
+- CairoVM chains are very novel and evolve quite fast. We are thus waiting for
+  more stability to directly fork/modify a client. New engines, storage
+  technologies and other innovations are being developed by talented teams (e.g.
+  [Starknet](https://starkware.co/starknet/),
+  [Katana](https://book.dojoengine.org/toolchain/katana/overview.html),
+  [Madara](https://madara-docs.vercel.app/)).
+
+<details>
+<summary>For reference, click to see how the above mentioned monolithic architecture
+would look like (note that it is not the current architecture of Kakarot zkEVM):
+</summary>
+
+<p align="center">
+<img src="./docs/Monolith_Kakarot_zkEVM_not_live.png"
+alt="Kakarot Monolithic Architecture" width="50%" style="border-radius: 10px;">
+</p>
+
+</details>
+
+## Installation
+
+The Kakarot Indexer is based on the [Apibara](https://www.apibara.com/docs)
+third-party service, an indexing product for StarknetOS chains. Apibara relies
+on [Deno](https://deno.com/), a specific Javascript runtime. Follow the Deno
+installation requirements to be able to interact with the codebase.
+
+## Usage
+
+Although this codebase is open-source, it relies on third-party software, the
+[DNA protocol](https://www.apibara.com/docs/advanced/streaming-protocol). One
+may read the code, run it locally, or self-host the indexer using Apibara's
+documentation and API keys.
+
+## License
+
+[MIT License](./docs/LICENSE)

indexer/deno.json

@@ -0,0 +1,5 @@
+{
+  "tasks": {
+    "dev": "deno run --watch main.ts"
+  }
+}