feat(task db): implement a task DB (#208)

* task-manager: dump fight vs sqlite

* task-manager: SQL tables+views+triggers success - but arguments passed in execute are 'NULL'

* task-manager: passing enqueue_task tests

* task-manager: cleanup - ease copy pasting to SQL script, Registered status, persistent views, remove debug print

* task-manager: add DB size query

* task-manager: id_proof is unneeded + prettify queries

* task-manager: change DB schema - allow multiple provers and status for same task in case of failures-retry

* task-manager: allow task updates

* task-manager: retrieve cached proofs from DB

* task-manager: add status check

* task-manager: add progress reports

* chore(task_manager): Run cargo fmt

* feat: address small lints

* feat(task-manager): use result type with thiserror

* chore(task-db): fix typos

* refactor(task-manager): clean up tests

* fix(docker): unignore task manager

* [WIP](task_manager): write initial task handler stubs

* chore(task_manager): run cargo fmt

* [WIP](task_manager): write status and proof get handlers

* refactor(host): use merge instead of nest

* chore(format): format workflow files

* chore(deps): use consistent dependency style

* chore(host): rename tx to task_channel

* [WIP](task_manager): add initial submit logic

* chore(clippy): remove unused parameter

* chore(clippy): remove unused imports

* refactor(core): add copy trait to proof types

* feat(task_manager): simplify db and adapt tests

* fix(clippy): fix dereference issue

* [WIP]: handle proof request by worker and update task status

* [WIP]: add block fetching and initial blockhash getting for submit

* [WIP]: handle task creation, status and proof retrieval

* fix(host): fix route sub-path

* feat(raiko): abstract task manager and impl a mem db for easy integration (#296)

* impl a mem db for easy integration

Signed-off-by: smtmfft <[email protected]>

* fix clippy and unit test

Signed-off-by: smtmfft <[email protected]>

* fix fmt

Signed-off-by: smtmfft <[email protected]>

---------

Signed-off-by: smtmfft <[email protected]>

* fix: throw error instead of panicing on runtime checks

* fix(core,task_manager): add custom ensure and require fns

* feat(task_db): sqlite and in memory abstraction (#301)

* enable sqlite db by feature

Signed-off-by: smtmfft <[email protected]>

* debug lifetime

Signed-off-by: smtmfft <[email protected]>

* resolve lifetime issue and make all tests pass

Signed-off-by: smtmfft <[email protected]>

* refactor(task_db): simplify structure for sqlite and use cached statements

* feat(task_db): abstract task db implementation into wrapper

* fix(task_db): add await to test call

* fix(task_db): fix import declaration

* fix(task_db): add async and mutable variables

* fix(host): fix task manager usage

* fix(task_db): fix test for async

* Update Cargo.toml

use in-mem as default.

---------

Signed-off-by: smtmfft <[email protected]>
Co-authored-by: smtmfft <[email protected]>
Co-authored-by: smtmfft <[email protected]>

* feat(task_manager): return empty list on key not found

* feat(host,task_manager): add tracing and handle workers

* feat(host): fix response structure

* chore(clippy): remove unused imports

* fix(ci): remove git merge added lines

* fix(task_manager): add blob proof type field

---------

Signed-off-by: smtmfft <[email protected]>
Co-authored-by: Petar Vujović <[email protected]>
Co-authored-by: smtmfft <[email protected]>
Co-authored-by: smtmfft <[email protected]>

.dockerignore

@@ -19,3 +19,4 @@
 !/provers/sgx/setup
 !/kzg_settings_raw.bin
 !/core
+!/task_manager

.github/workflows/ci-native.yml

@@ -2,11 +2,10 @@ name: CI - Native
 
 on: workflow_call
 
-
 jobs:
-    build-test-native:
-      name: Build and test native
-      uses: ./.github/workflows/ci-build-test-reusable.yml
-      with:
-        version_name: "native"
-        version_toolchain: "nightly-2024-04-17"
+  build-test-native:
+    name: Build and test native
+    uses: ./.github/workflows/ci-build-test-reusable.yml
+    with:
+      version_name: "native"
+      version_toolchain: "nightly-2024-04-17"

.github/workflows/ci-risc0.yml

@@ -12,9 +12,9 @@ on:
   merge_group:
 
 jobs:
-    build-test-risc0:
-      name: Build and test risc0
-      uses: ./.github/workflows/ci-build-test-reusable.yml
-      with:
-        version_name: "risc0"
-        version_toolchain: "stable"
+  build-test-risc0:
+    name: Build and test risc0
+    uses: ./.github/workflows/ci-build-test-reusable.yml
+    with:
+      version_name: "risc0"
+      version_toolchain: "stable"

.github/workflows/ci-sgx-hardware.yml

@@ -11,7 +11,7 @@ jobs:
       TARGET: sgx
       CI: 1
       EDMM: 0
-  
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -21,15 +21,15 @@ jobs:
         with:
           toolchain: stable
           profile: minimal
-  
+
       - name: Install cargo-binstall
         uses: cargo-bins/[email protected]
-  
+
       - name: Install sgx
         run: make install
-  
+
       - name: Build sgx prover
         run: make build
-  
+
       - name: Test sgx prover
         run: make test

.github/workflows/ci-sp1.yml

@@ -12,9 +12,9 @@ on:
   merge_group:
 
 jobs:
-    build-test-sgx:
-      name: Build and test sp1
-      uses: ./.github/workflows/ci-build-test-reusable.yml
-      with:
-        version_name: "sp1"
-        version_toolchain: "nightly-2024-04-18"
+  build-test-sgx:
+    name: Build and test sp1
+    uses: ./.github/workflows/ci-build-test-reusable.yml
+    with:
+      version_name: "sp1"
+      version_toolchain: "nightly-2024-04-18"

.github/workflows/openapi-deploy.yml

@@ -44,7 +44,7 @@ jobs:
         if: github.ref == 'refs/heads/main'
         uses: actions/upload-pages-artifact@v2
         with:
-          path: './openapi'
+          path: "./openapi"
 
       - name: Deploy to GitHub Pages
         if: github.ref == 'refs/heads/main'

.gitignore

@@ -38,6 +38,13 @@ target/
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
 
+# SQLite
+# -----------------------------------------------------------------------------------------
+*.sqlite
+*.sqlite-shm
+*.sqlite-wal
+*.sqlite-journal
+
 # Temp files, swap, debug, log, perf, cache
 # -----------------------------------------------------------------------------------------
 *.swp

Cargo.toml

@@ -14,6 +14,7 @@ members = [
     "provers/sgx/setup",
     "pipeline",
     "core",
+    "task_manager",
 ]
 
 # Always optimize; building and running the guest takes much longer without optimization.
@@ -35,6 +36,7 @@ opt-level = 3
 # raiko
 raiko-lib = { path = "./lib", features = ["std"] }
 raiko-core = { path = "./core" }
+raiko-task-manager = { path = "./task_manager" }
 
 # reth
 reth-primitives = { git = "https://github.com/taikoxyz/taiko-reth.git", branch = "v1.0.0-rc.2-taiko", default-features = false, features = ["alloy-compat", "taiko"] }
@@ -109,6 +111,7 @@ base64-serde = "0.7.0"
 base64 = "0.21.7"
 libflate = { version = "2.0.0" }
 typetag = { version = "0.2.15" }
+num_enum = "0.7.2"
 
 # tracing, logging
 tracing = "0.1"
@@ -135,6 +138,7 @@ tokio = { version = "^1.23", features = ["full"] }
 hyper = { version = "0.14.27", features = ["server"] }
 reqwest = { version = "0.11.22", features = ["json"] }
 url = "2.5.0"
+async-trait = "0.1.80"
 
 # crypto
 kzg = { package = "rust-kzg-zkcrypto", git = "https://github.com/brechtpd/rust-kzg.git", branch = "sp1-patch", default-features = false }
@@ -156,6 +160,9 @@ anyhow = "1.0"
 thiserror = "1.0"
 thiserror-no-std = "2.0.2"
 
+# SQLite
+rusqlite = { version = "0.31.0", features = ["bundled"] }
+
 # misc
 hashbrown = { version = "0.14", features = ["inline-more"] }
 tempfile = "3.8"
@@ -180,4 +187,4 @@ revm = { git = "https://github.com/taikoxyz/revm.git", branch = "v36-taiko" }
 revm-primitives = { git = "https://github.com/taikoxyz/revm.git", branch = "v36-taiko" }
 revm-precompile = { git = "https://github.com/taikoxyz/revm.git", branch = "v36-taiko" }
 secp256k1 = { git = "https://github.com/CeciliaZ030/rust-secp256k1", branch = "sp1-patch" }
-blst = { git = "https://github.com/CeciliaZ030/blst.git", branch = "v0.3.12-serialize" }
+blst = { git = "https://github.com/CeciliaZ030/blst.git", branch = "v0.3.12-serialize" }