feat(lockfile)!: Smaller lockfile (#45)

* feat(lockfile): Platform string * refactor: move platform types * deduplicate tasks dependecy def * deduplicate task allowedPortDeps * fix taskAllowPortDeps registration * fix bug * remove debug logging * relative file urls * merge install configs * merge allowed deps * fix task config with resolved deps * refactor: move other processing tasks to processManifest * fix type taskEnvBase * add github token * fix meta-cli release artifact url
metatypedev · Mar 8, 2024 · 423d38e · 423d38e
1 parent 6d35e91
commit 423d38e
Show file tree

Hide file tree

Showing 20 changed files with 751 additions and 1,857 deletions.
diff --git a/.ghjk/deno.lock b/.ghjk/deno.lock
diff --git a/.ghjk/lock.json b/.ghjk/lock.json
diff --git a/deno.lock b/deno.lock
diff --git a/deps/common.ts b/deps/common.ts
@@ -12,4 +12,5 @@ export * as std_path from "https://deno.land/[email protected]/path/mod.ts";
 export * as std_fs from "https://deno.land/[email protected]/fs/mod.ts";
 export * as dax from "https://deno.land/x/[email protected]/mod.ts";
 export * as jsonHash from "https://deno.land/x/[email protected]/mod.ts";
+export { default as objectHash } from "https://deno.land/x/[email protected]/mod.ts";
 export { default as deep_eql } from "https://deno.land/x/[email protected]/index.js";
diff --git a/host/mod.ts b/host/mod.ts
@@ -9,12 +9,12 @@ import {
   PathRef,
   stringHashHex,
 } from "../utils/mod.ts";
-import validators from "./types.ts";
+import validators, { SerializedConfig } from "./types.ts";
 import * as std_modules from "../modules/std.ts";
 import * as deno from "./deno.ts";
 import type { ModuleBase } from "../modules/mod.ts";
 import { GhjkCtx } from "../modules/types.ts";
-import portValidators from "../modules/ports/types.ts";
+import { serializePlatform } from "../modules/ports/types/platform.ts";
 
 export interface CliArgs {
   ghjkShareDir: string;
@@ -140,10 +140,7 @@ async function readConfig(gcx: GhjkCtx, hcx: HostCtx) {
 
   const ghjkfileHash = await fileHashHex(hcx, configPath);
 
-  let serializedConfig;
-  let envVarHashes;
-  let readFileHashes;
-  let listedFiles;
+  let configExt: SerializedConfigExt | null = null;
   // TODO: figure out cross platform lockfiles :O
   if (
     foundLockObj && // lockfile found
@@ -211,46 +208,46 @@ async function readConfig(gcx: GhjkCtx, hcx: HostCtx) {
       await fileListingsMatch() &&
       await envHashesMatch()
     ) {
-      serializedConfig = foundLockObj.config;
-      envVarHashes = foundHashObj.envVarHashes;
-      readFileHashes = foundHashObj.readFileHashes;
-      listedFiles = foundHashObj.listedFiles;
+      configExt = {
+        config: foundLockObj.config,
+        envVarHashes: foundHashObj.envVarHashes,
+        readFileHashes: foundHashObj.readFileHashes,
+        listedFiles: foundHashObj.listedFiles,
+      };
     }
   }
 
-  // if one is null, all are null but typescript
-  // doesn't know better so check both
-  if (!serializedConfig || !envVarHashes || !readFileHashes || !listedFiles) {
+  if (!configExt) {
     logger().info("serializing ghjkfile", configPath);
-    ({ config: serializedConfig, envVarHashes, listedFiles, readFileHashes } =
-      await readAndSerializeConfig(
-        hcx,
-        configPath,
-        curEnvVars,
-      ));
+    configExt = await readAndSerializeConfig(hcx, configPath, curEnvVars);
   }
 
   const newLockObj: zod.infer<typeof lockObjValidator> = {
     version: "0",
-    platform: [Deno.build.os, Deno.build.arch],
+    platform: serializePlatform(Deno.build),
     moduleEntries: {} as Record<string, unknown>,
-    config: serializedConfig,
+    config: configExt.config,
   };
   const newHashObj: zod.infer<typeof hashObjValidator> = {
     version: "0",
     ghjkfileHash,
-    envVarHashes,
-    readFileHashes,
-    listedFiles,
+    envVarHashes: configExt.envVarHashes,
+    readFileHashes: configExt.readFileHashes,
+    listedFiles: configExt.listedFiles,
   };
   const instances = [];
-  for (const man of serializedConfig.modules) {
+  for (const man of configExt.config.modules) {
     const mod = std_modules.map[man.id];
     if (!mod) {
       throw new Error(`unrecognized module specified by ghjk.ts: ${man.id}`);
     }
     const instance: ModuleBase<unknown, unknown> = new mod.ctor();
-    const pMan = await instance.processManifest(gcx, man, lockEntries[man.id]);
+    const pMan = await instance.processManifest(
+      gcx,
+      man,
+      lockEntries[man.id],
+      newLockObj.config.globalEnv,
+    );
     instances.push([man.id, instance, pMan] as const);
     subCommands[man.id] = instance.command(gcx, pMan);
   }
@@ -274,40 +271,51 @@ async function readConfig(gcx: GhjkCtx, hcx: HostCtx) {
   if (!foundHashObj || !deep_eql(newHashObj, foundHashObj)) {
     await hashFilePath.writeJsonPretty(newHashObj);
   }
-  return { subCommands, serializedConfig };
+  return { subCommands, serializedConfig: configExt.config };
 }
 
+type HashStore = Record<string, string | null | undefined>;
+
+type SerializedConfigExt = {
+  config: SerializedConfig;
+  envVarHashes: HashStore;
+  readFileHashes: HashStore;
+  listedFiles: string[];
+};
+
 async function readAndSerializeConfig(
   hcx: HostCtx,
   configPath: PathRef,
   envVars: Record<string, string>,
-) {
-  let raw;
-  let envVarHashes;
-  let readFileHashes;
-  let listedFiles;
+): Promise<SerializedConfigExt> {
   switch (configPath.extname()) {
     case "":
       logger().warning("config file has no extension, assuming deno config");
-      /* falls through */
+    /* falls through */
     case ".ts": {
+      logger().debug("serializing ts config", configPath);
       const res = await deno.getSerializedConfig(
         configPath.toFileUrl().href,
         envVars,
       );
-      raw = res.config;
-      envVarHashes = await hashEnvVars(envVars, res.accessedEnvKeys);
+      const envVarHashes = await hashEnvVars(envVars, res.accessedEnvKeys);
       const cwd = $.path(Deno.cwd());
       const cwdStr = cwd.toString();
-      listedFiles = res.listedFiles
+      const listedFiles = res.listedFiles
         .map((path) => cwd.resolve(path).toString().replace(cwdStr, "."));
       // FIXME: this breaks if the version of the file the config reads
       // has changed by this point
       // consider reading mtime of files when read by the serializer and comparing
       // them before hashing to make sure we get the same file
       // not sure what to do if it has changed though, re-serialize?
-      readFileHashes = await hashFiles(hcx, res.readFiles, cwd);
-      break;
+      const readFileHashes = await hashFiles(hcx, res.readFiles, cwd);
+
+      return {
+        config: validateRawConfig(res.config, configPath),
+        envVarHashes,
+        readFileHashes,
+        listedFiles,
+      };
     }
     // case ".jsonc":
     // case ".json":
@@ -318,26 +326,34 @@ async function readAndSerializeConfig(
         `unrecognized ghjkfile type provided at path: ${configPath}`,
       );
   }
+}
+
+function validateRawConfig(
+  raw: unknown,
+  configPath: PathRef,
+): SerializedConfig {
   const res = validators.serializedConfig.safeParse(raw);
   if (!res.success) {
     logger().error("zod error", res.error);
     logger().error("serializedConf", raw);
     throw new Error(`error parsing seralized config from ${configPath}`);
   }
-  const config = res.data;
-  return { config, envVarHashes, readFileHashes, listedFiles };
+
+  return res.data;
 }
 
 const lockObjValidator = zod.object({
   version: zod.string(),
-  platform: zod.tuple([portValidators.osEnum, portValidators.archEnum]),
+  platform: zod.string(), // TODO custom validator??
   moduleEntries: zod.record(zod.string(), zod.unknown()),
   config: validators.serializedConfig,
 });
 
-async function readLockFile(lockFilePath: PathRef) {
+type LockObject = zod.infer<typeof lockObjValidator>;
+
+async function readLockFile(lockFilePath: PathRef): Promise<LockObject | null> {
   const raw = await lockFilePath.readMaybeJson();
-  if (!raw) return;
+  if (!raw) return null;
   const res = lockObjValidator.safeParse(raw);
   if (!res.success) {
     throw new Error(`error parsing lockfile from ${lockFilePath}`, {
@@ -369,7 +385,7 @@ async function readHashFile(hashFilePath: PathRef) {
 }
 
 async function hashEnvVars(all: Record<string, string>, accessed: string[]) {
-  const hashes = {} as Record<string, string | null>;
+  const hashes = {} as HashStore;
   for (const key of accessed) {
     const val = all[key];
     if (!val) {
@@ -384,7 +400,7 @@ async function hashEnvVars(all: Record<string, string>, accessed: string[]) {
 
 async function hashFiles(hcx: HostCtx, readFiles: string[], cwd: PathRef) {
   const cwdStr = cwd.toString();
-  const readFileHashes = {} as Record<string, string | null>;
+  const readFileHashes = {} as HashStore;
   for (const path of readFiles) {
     const pathRef = cwd.resolve(path);
     const relativePath = pathRef

diff --git a/host/types.ts b/host/types.ts
@@ -1,13 +1,21 @@
 import { zod } from "../deps/common.ts";
 import moduleValidators from "../modules/types.ts";
+import portsValidator from "../modules/ports/types.ts";
+
+const globalEnv = zod.object({
+  installs: zod.record(zod.string(), portsValidator.installConfigFat),
+  allowedPortDeps: zod.record(zod.string(), portsValidator.allowedPortDep),
+});
 
 const serializedConfig = zod.object(
   {
     modules: zod.array(moduleValidators.moduleManifest),
+    globalEnv,
   },
 );
 
 export type SerializedConfig = zod.infer<typeof serializedConfig>;
+export type GlobalEnv = zod.infer<typeof globalEnv>;
 
 export default {
   serializedConfig,

diff --git a/mod.ts b/mod.ts
@@ -20,15 +20,15 @@ import * as std_ports from "./modules/ports/std.ts";
 import * as cpy from "./ports/cpy_bs.ts";
 import * as node from "./ports/node.ts";
 // hosts
-import type { SerializedConfig } from "./host/types.ts";
+import type { GlobalEnv, SerializedConfig } from "./host/types.ts";
 import * as std_modules from "./modules/std.ts";
 // tasks
 import type {
   TaskDef,
   TaskEnv,
   TasksModuleConfig,
 } from "./modules/tasks/types.ts";
-import { dax } from "./deps/common.ts";
+import { dax, jsonHash, objectHash } from "./deps/common.ts";
 
 const portsConfig: PortsModuleConfigBase = { installs: [] };
 
@@ -43,10 +43,14 @@ export type TaskFnDef = TaskDef & {
   fn: TaskFn;
   // command: cliffy_cmd.Command;
 };
-const tasks = {} as Record<
-  string,
-  TaskFnDef
->;
+
+// TODO tasks config
+const tasks = {} as Record<string, TaskFnDef>;
+
+const globalEnv: GlobalEnv = {
+  installs: {},
+  allowedPortDeps: {},
+};
 
 // FIXME: ses.lockdown to freeze primoridials
 // freeze the object to prevent malicious tampering of the secureConfig
@@ -64,33 +68,47 @@ export function install(...configs: InstallConfigFat[]) {
   }
 }
 
+function registerInstall(config: InstallConfigFat) {
+  // jsonHash.digest is async
+  const hash = objectHash(jsonHash.canonicalize(config as jsonHash.Tree));
+
+  if (!globalEnv.installs[hash]) {
+    globalEnv.installs[hash] = config;
+  }
+  return hash;
+}
+
+function registerAllowedPortDep(dep: AllowedPortDep) {
+  const hash = objectHash(jsonHash.canonicalize(dep as jsonHash.Tree));
+  if (!globalEnv.allowedPortDeps[hash]) {
+    globalEnv.allowedPortDeps[hash] = dep;
+  }
+  return hash;
+}
+
 /*
  * A nicer form of TaskFnDef for better ergonomics in the ghjkfile
  */
 export type TaskDefNice =
   & Omit<TaskFnDef, "env" | "name" | "dependsOn">
   & Partial<Pick<TaskFnDef, "dependsOn">>
-  & Partial<Omit<TaskEnv, "allowedPortDeps">>
-  & { allowedPortDeps?: AllowedPortDep[] };
+  & Partial<Pick<TaskEnv, "env">>
+  & { allowedPortDeps?: AllowedPortDep[]; installs?: InstallConfigFat[] };
 export function task(name: string, config: TaskDefNice) {
-  const allowedPortDeps = Object.fromEntries([
-    ...(config.allowedPortDeps ??
-      // only add the stdDeps if the task specifies installs
-      (config.installs ? stdDeps() : []))
-      .map((dep) =>
-        [
-          dep.manifest.name,
-          portsValidators.allowedPortDep.parse(dep),
-        ] as const
-      ),
-  ]);
+  const allowedPortDeps = [
+    ...(config.allowedPortDeps ?? (config.installs ? stdDeps() : [])),
+  ].map(registerAllowedPortDep);
+
+  // TODO validate installs?
+  const installs = (config.installs ?? []).map(registerInstall);
+
   tasks[name] = {
     name,
     fn: config.fn,
     desc: config.desc,
     dependsOn: config.dependsOn ?? [],
     env: {
-      installs: config.installs ?? [],
+      installs,
       env: config.env ?? {},
       allowedPortDeps,
     },
@@ -113,7 +131,7 @@ function addInstall(
   }
   const config = res.data;
   logger().debug("install added", config);
-  cx.installs.push(config);
+  cx.installs.push(registerInstall(config));
 }
 
 export function secureConfig(
@@ -164,7 +182,7 @@ async function getConfig(secureConfig: PortsModuleSecureConfig | undefined) {
         .map((dep) =>
           [
             dep.manifest.name,
-            portsValidators.allowedPortDep.parse(dep),
+            registerAllowedPortDep(portsValidators.allowedPortDep.parse(dep)),
           ] as const
         ),
     ]);
@@ -199,9 +217,10 @@ async function getConfig(secureConfig: PortsModuleSecureConfig | undefined) {
         id: std_modules.tasks,
         config: tasksConfig,
       }],
+      globalEnv,
     };
     return config;
   } catch (cause) {
-    throw new Error(`error constructing config for serializatino`, { cause });
+    throw new Error(`error constructing config for serialization`, { cause });
   }
 }