New project structure + various changes

docker-compose.yml

@@ -1,9 +1,9 @@
 services:
   subql-ai:
-    image: subquerynetwork/subql-ai-app
-    # build:
-    #   context: .
-    #   dockerfile: ./Dockerfile
+    # image: subquerynetwork/subql-ai-app
+    build:
+      context: .
+      dockerfile: ./Dockerfile
     ports:
       - 7827:7827
     restart: unless-stopped
@@ -14,24 +14,27 @@ services:
       # - -p=/app/index.ts # TODO this doesn't work because dependencies are not copied
       - -p=ipfs://QmNaNBhXJoFpRJeNQcnTH8Yh6Rf4pzJy6VSnfnQSZHysdZ
       - -h=http://host.docker.internal:11434
-    # healthcheck:
-    #   test: ["CMD", "curl", "-f", "http://subql-ai:7827/health"]
-    #   interval: 3s
-    #   timeout: 5s
-    #   retries: 10
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://subql-ai:7827/ready"]
+      interval: 3s
+      timeout: 5s
+      retries: 10
 
   # A simple chat UI
   ui:
     image: ghcr.io/open-webui/open-webui:main
     ports:
       - 8080:8080
     restart: always
+    depends_on:
+      "subql-ai":
+        condition: service_healthy
     environment:
-      - 'OPENAI_API_BASE_URLS=http://subql-ai:7827/v1'
-      - 'OPENAI_API_KEYS=foobar'
-      - 'WEBUI_AUTH=false'
+      - "OPENAI_API_BASE_URLS=http://subql-ai:7827/v1"
+      - "OPENAI_API_KEYS=foobar"
+      - "WEBUI_AUTH=false"
     volumes:
-     - open-webui:/app/backend/data
+      - open-webui:/app/backend/data
 
 volumes:
   open-webui:

src/app.ts

@@ -1,4 +1,3 @@
-import { resolve } from "@std/path/resolve";
 import ora from "ora";
 import { brightMagenta } from "@std/fmt/colors";
 import { Ollama } from "ollama";
@@ -11,7 +10,7 @@ import { Context, type IContext } from "./context/context.ts";
 import type { ISandbox } from "./sandbox/sandbox.ts";
 import * as lancedb from "@lancedb/lancedb";
 import type { IPFSClient } from "./ipfs.ts";
-import { loadProject, loadVectorStoragePath } from "./loader.ts";
+import { Loader } from "./loader.ts";
 import { getPrompt } from "./util.ts";
 
 export async function runApp(config: {
@@ -23,25 +22,20 @@ export async function runApp(config: {
   forceReload?: boolean;
 }): Promise<void> {
   const model = new Ollama({ host: config.host });
-  const projectPath = await loadProject(
+
+  const loader = new Loader(
     config.projectPath,
     config.ipfs,
     undefined,
     config.forceReload,
   );
-  const sandbox = await getDefaultSandbox(resolve(projectPath));
+
+  const sandbox = await getDefaultSandbox(loader);
 
   const ctx = await makeContext(
     sandbox,
     model,
-    (dbPath) =>
-      loadVectorStoragePath(
-        projectPath,
-        dbPath,
-        config.ipfs,
-        undefined,
-        config.forceReload,
-      ),
+    loader,
   );
 
   const runnerHost = new RunnerHost(() => {
@@ -59,9 +53,6 @@ export async function runApp(config: {
 
   switch (config.interface) {
     case "cli":
-      if (sandbox.userMessage) {
-        console.log(sandbox.userMessage);
-      }
       await cli(runnerHost);
       break;
     case "http":
@@ -73,18 +64,20 @@ export async function runApp(config: {
 async function makeContext(
   sandbox: ISandbox,
   model: Ollama,
-  loadVectorStoragePath: (vectorStoragePath: string) => Promise<string>,
+  loader: Loader,
 ): Promise<IContext> {
-  if (!sandbox.vectorStorage) {
+  if (!sandbox.manifest.vectorStorage) {
     return new Context(model);
   }
 
-  const { type, path } = sandbox.vectorStorage;
+  const { type } = sandbox.manifest.vectorStorage;
   if (type !== "lancedb") {
     throw new Error("Only lancedb vector storage is supported");
   }
-  const dbPath = await loadVectorStoragePath(path);
-  const connection = await lancedb.connect(dbPath);
+
+  const loadRes = await loader.getVectorDb();
+  if (!loadRes) throw new Error("Failed to load vector db");
+  const connection = await lancedb.connect(loadRes[0]);
 
   return new Context(model, connection);
 }

src/bundle.ts

@@ -1,4 +1,3 @@
-import { getProjectJson } from "./info.ts";
 import { resolve } from "@std/path/resolve";
 import { Tar } from "@std/archive/tar";
 import { walk } from "@std/fs/walk";
@@ -9,20 +8,39 @@ import type { IPFSClient } from "./ipfs.ts";
 // import * as esbuild from "https://deno.land/x/[email protected]/wasm.js";
 // import * as esbuild from "esbuild";
 import { denoPlugins } from "@luca/esbuild-deno-loader";
-import { getDefaultSandbox } from "./sandbox/index.ts";
 import { toReadableStream } from "@std/io/to-readable-stream";
 import { readerFromStreamReader } from "@std/io/reader-from-stream-reader";
 import { getSpinner } from "./util.ts";
+import { Loader } from "./loader.ts";
 
 export async function publishProject(
   projectPath: string,
   ipfs: IPFSClient,
-  sandboxFactory = getDefaultSandbox,
 ): Promise<string> {
-  projectPath = await Deno.realPath(projectPath);
-  const projectJson = await getProjectJson(projectPath, sandboxFactory);
-  let code = await generateBundle(projectPath);
-  const vectorDbPath = projectJson.vectorStorage?.path;
+  const loader = new Loader(projectPath, ipfs);
+
+  const [_, manifest, source] = await loader.getManifest();
+  if (source !== "local") {
+    throw new Error("Cannot bundle a project that isn't local");
+  }
+
+  // Upload project
+  const [project, projectSource] = await loader.getProject();
+  if (projectSource === "local") {
+    const spinner = getSpinner().start("Publishing project code");
+    try {
+      const code = await generateBundle(project);
+      const [{ cid: codeCid }] = await ipfs.addFile([code]);
+      manifest.entry = `ipfs://${codeCid}`;
+      spinner.succeed("Published project code");
+    } catch (e) {
+      spinner.fail("Failed to publish project code");
+      throw e;
+    }
+  }
+
+  // Upload vector db
+  const vectorDbPath = manifest.vectorStorage?.path;
   if (vectorDbPath) {
     // Resolve the db path relative to the project
     const dbPath = resolve(dirname(projectPath), vectorDbPath);
@@ -39,11 +57,9 @@ export async function publishProject(
 
         const [{ cid }] = await ipfs.addFile([dbBuf]);
 
-        code = updateProjectVectorDbPath(
-          code,
-          vectorDbPath,
-          `ipfs://${cid}`,
-        );
+        // Update manifest
+        manifest.vectorStorage!.path = `ipfs://${cid}`;
+
         spinner.succeed("Published vector db");
       } catch (e) {
         spinner.fail("Failed to publish project vectordb");
@@ -52,8 +68,9 @@ export async function publishProject(
     }
   }
 
+  // Upload manifest
   const spinner = getSpinner().start("Publishing project to IPFS");
-  const [{ cid }] = await ipfs.addFile([code]);
+  const [{ cid }] = await ipfs.addFile([JSON.stringify(manifest, null, 2)]);
   spinner.succeed("Published project to IPFS");
   return `ipfs://${cid}`;
 }
@@ -86,20 +103,6 @@ export async function generateBundle(projectPath: string): Promise<string> {
   }
 }
 
-/**
- * @param code The raw bundled code
- * @param currentPath The current db path that will get replaced
- * @param newPath The new db path to replace
- * @returns Updated raw bundled code
- */
-function updateProjectVectorDbPath(
-  code: string,
-  currentPath: string,
-  newPath: string,
-): string {
-  return code.replaceAll(currentPath, newPath);
-}
-
 /**
  * Archives the lancedb directory into a file for uploading
  * @param dbPath The path to the lancedb directory

src/bundle_test.ts

@@ -1,6 +1,5 @@
 import { generateBundle, publishProject } from "./bundle.ts";
 import { expect } from "jsr:@std/expect";
-import { UnsafeSandbox } from "./sandbox/unsafeSandbox.ts";
 import { IPFSClient } from "./ipfs.ts";
 
 Deno.test("Generates a bundle", async () => {
@@ -12,15 +11,14 @@ Deno.test("Generates a bundle", async () => {
 Deno.test("Publishing a project to ipfs", async () => {
   // WebWorkers don't work in tests, use the unsafe sandbox instead
   const cid = await publishProject(
-    "./subquery-delegator/index.ts",
+    "./subquery-delegator/project.ts",
     new IPFSClient(
       Deno.env.get("IPFS_ENDPOINT") ??
         "https://unauthipfs.subquery.network/ipfs/api/v0",
       {
         Authorization: `Bearer: ${Deno.env.get("SUBQL_ACCESS_TOKEN")}`,
       },
     ),
-    UnsafeSandbox.create,
   );
 
   // The example project could end up being modified so we only validate the response, not the content