Simple SQLite DB loader (currently unused) and update worker script to push to gh-pages

package.json

@@ -7,9 +7,11 @@
     "@swc/core": "^1.2.63",
     "@swc/helpers": "^0.2.12",
     "@types/node": "^16.3.2",
+    "@types/sql.js": "^1.4.2",
     "env2-shebang": "^0.0.2",
     "fs-extra": "^10.0.0",
     "jsonc-parser": "^3.0.0",
+    "sql.js": "^1.5.0",
     "strip-json-comments": "^3.1.1",
     "ts-node": "^10.1.0",
     "tslib": "^2.3.0",

src/db.ts

@@ -0,0 +1,65 @@
+import fsExtra from 'fs-extra';
+const { mkdtempSync, readFileSync, writeFileSync } = fsExtra;
+import { join } from "path";
+import { $ } from 'zx';
+import initSqlJs from 'sql.js';
+
+/*
+ * Does it make any kind of sense to store state in a proper sqlite database?
+ * I don't know, but I threw this module together to find out.
+ * 
+ * Theoretically, if this DB is published to GH pages, we can even read it on the
+ * frontend.  No idea if that will be useful or not.
+ * 
+ * IS UNTESTED!
+ */
+
+const SQLPromise = initSqlJs({});
+
+const dbFilename = 'db.sqlite';
+
+/**
+ * A sqlite database persisted as a file in its own git ref.
+ * 
+ * The git ref can be a branch, or it can be a non-branch ref.
+ * 
+ * Assumes that you're already in a directory with a git clone where we can create worktrees
+ * 
+ * NOTE:
+ * For better or worse, this has been hacked together under the assumption
+ * that this database file is the *only* file in the ref.
+ * So you need to give the database its own ref to live in.
+ */
+export async function createDatabaseFromGit(opts: {
+    gitRef?: string,
+    workdir?: string
+}) {
+    const {
+        workdir = process.cwd(),
+        gitRef = 'refs/heads/db'
+    } = opts;
+
+    await $`git fetch ${gitRef}`;
+    const tempDir = mkdtempSync(join(workdir, 'tmp'));
+    await $`git worktree add ${ tempDir } FETCH_HEAD`;
+    const db = readFileSync(join(tempDir, dbFilename));
+    const database = new (await SQLPromise).Database(db);
+
+    async function pushDatabaseToGit() {
+        const buffer = database.export();
+        writeFileSync(join(tempDir, dbFilename), buffer);
+        await $`git -C ${tempDir} add ${dbFilename}`;
+        await $`git -C ${tempDir} commit -m "update database"`;
+        await $`git -C ${tempDir} push HEAD:${gitRef}`;
+    }
+    async function discardWorktree() {
+        await $`git worktree remove ${ tempDir }`;
+    }
+
+    return {
+        tempDir,
+        database,
+        pushDatabaseToGit,
+        discardWorktree
+    };
+}

src/work.ts

@@ -36,10 +36,22 @@ if(errors.length) throw errors;
 // Create a temp directory to work in
 const tempDir = mkdtempSync(join(__root, 'tmp'));
 const docsOutputRoot = join(tempDir, 'docs');
+await $`git worktree add ${docsOutputRoot} orphan`;
+
+interface IndexEntry {
+    lib: string;
+    dir: string;
+}
+
+const indexEntries: IndexEntry[] = [];
 
 // For each lib, render it
 for(const lib of libsToBuild) {
     const libSanitizedDir = lib.replace(/[@\/]/g, '_');
+    indexEntries.push({
+        lib, dir: libSanitizedDir
+    });
+
     const workdir = join(tempDir, libSanitizedDir);
 
     mkdirpSync(workdir);
@@ -54,8 +66,8 @@ for(const lib of libsToBuild) {
     const entrypointPath = getPathToModuleEntrypoint(lib, workdir)!;
     assert(entrypointPath);
 
-    // cd(join(workdir, `node_modules/${lib}`));
-
+    // output docs to this directory
+    const outDir = join(docsOutputRoot, libSanitizedDir);
     // typedoc needs a tsconfig file.  Create one
     const tsconfigPath = join(workdir, `tsconfig.json`);
     writeFileSync(tsconfigPath, JSON.stringify({
@@ -64,11 +76,28 @@ for(const lib of libsToBuild) {
             target: 'esnext',
             module: 'esnext',
             moduleResolution: 'node'
+        },
+        typedocOptions: {
+            entryPoints: [entrypointPath],
+            out: outDir
         }
     }));
-    // output docs to this directory
-    const outDir = join(docsOutputRoot, libSanitizedDir);
 
     // NOTE not catching errors.  If non-zero exit code, will continue to the next lib
-    await $`typedoc --tsconfig ${tsconfigPath} --entryPoints ${entrypointPath} --out ${outDir}`;
-}
+    await $`typedoc --tsconfig ${tsconfigPath}`;
+}
+
+writeFileSync(join(docsOutputRoot, 'index.html'), `
+    <ul>
+    ${
+        indexEntries.map(({lib, dir}) => `
+            <li><a href="${dir}">${lib}</a></li>
+        `).join('')
+    }
+    </ul>
+`);
+writeFileSync(join(docsOutputRoot, '.nojekyll'), '');
+
+await $`git -C ${docsOutputRoot} add --all`;
+await $`git -C ${docsOutputRoot} commit -m "overwrite docs with new build (TODO stop overwriting everything!)"`;
+await $`git -C ${docsOutputRoot} push -f origin HEAD:gh-pages`;

yarn.lock

@@ -120,6 +120,11 @@
   resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.1.tgz#a6ca6a9a0ff366af433f42f5f0e124794ff6b8f1"
   integrity sha512-FTgBI767POY/lKNDNbIzgAX6miIDBs6NTCbdlDb8TrWovHsSvaVIZDlTqym29C6UqhzwcJx4CYr+AlrMywA0cA==
 
+"@types/emscripten@*":
+  version "1.39.5"
+  resolved "https://registry.yarnpkg.com/@types/emscripten/-/emscripten-1.39.5.tgz#eb0fb1048301df980b6f8a5ec3d63f7d1572bb73"
+  integrity sha512-DIOOg+POSrYl+OlNRHQuIEqCd8DCtynG57H862UCce16nXJX7J8eWxNGgOcf8Eyge8zXeSs27mz1UcFu8L/L7g==
+
 "@types/fs-extra@^9.0.11":
   version "9.0.12"
   resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-9.0.12.tgz#9b8f27973df8a7a3920e8461517ebf8a7d4fdfaf"
@@ -145,6 +150,14 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.3.2.tgz#655432817f83b51ac869c2d51dd8305fb8342e16"
   integrity sha512-jJs9ErFLP403I+hMLGnqDRWT0RYKSvArxuBVh2veudHV7ifEC1WAmjJADacZ7mRbA2nWgHtn8xyECMAot0SkAw==
 
+"@types/sql.js@^1.4.2":
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/@types/sql.js/-/sql.js-1.4.2.tgz#7bf4f9ec12013f48d23d77f42fe0945b37ac8808"
+  integrity sha512-3Mq6KcJp+1qJ6FfE7c16oNZcgGrPiUZWj8DSePRRZQCPhi4vawX/thGCzwqIg59tOCjdQI1eMC+pLk9FLfTXwQ==
+  dependencies:
+    "@types/emscripten" "*"
+    "@types/node" "*"
+
 ansi-styles@^4.1.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937"
@@ -430,6 +443,11 @@ source-map@^0.6.0, source-map@^0.6.1:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
 
+sql.js@^1.5.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/sql.js/-/sql.js-1.5.0.tgz#6625c3e41115194f6bfab80a5e53647f1378c3fb"
+  integrity sha512-Qqr6HgX/hCDpLFWdN0BNoNpYQ2c1tOl1c3HGI0cshjaFSAWszKICuLZ9CyFUvRFPpEGW8RzHzwuXWWvXVGTKBg==
+
 strip-json-comments@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"