refactor(ports)!: streamlining and python support (#14)

* refactor(ports): simplify ports interface

* fix: example ghjk.ts

* fix: workaroud for port version conflicts

* fix: asdf installid bug

* wip: reuse installs across envs

* refact: simplify `PortModuleConfig`

* refactor: bring `asdf` into line

* feat(port): `python_bs`

* feat(port): `pipi`

* fix(ci): test-action hack

* fix: bad tar args

* fix: tests improvement and cleanup

* wip(tests): enable debug ghjk tree dumping

* feat(ci): add caching on deno dir

* fix(ports): removing wrong tmp dir

* feat(ports): `GITHUB_TOKEN` support

* refactor: remove python and node from default portDeps

* fix(tests): secure configs for tests

* fix(tests): serialized conf escaping issues

.github/workflows/tests.yml

@@ -10,6 +10,7 @@ on:
 
 env:
   DENO_VERSION: "1.38.5"
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   changes:
@@ -46,14 +47,23 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: ${{ env.DENO_VERSION }}
+
       - if: "${{ matrix.os == 'macos-latest' }}"
         run: brew install fish zsh
+      - name: Cache deno dir
+        if: "${{ matrix.os == 'macos-latest' }}"
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.DENO_DIR }}
+          key: deno-mac-${{ hashFiles('**/deno.lock') }}
+
       - if: "${{ matrix.e2eType == 'docker' }}"
         uses: docker/setup-buildx-action@v3
       - if: "${{ matrix.e2eType == 'docker' }}"
         uses: actions-hub/docker/cli@master
         env:
           SKIP_LOGIN: true
+
       - run: deno task test
 
   test-action:
@@ -67,5 +77,5 @@ jobs:
           GHJK_CONFIG: ./examples/protoc/ghjk.ts
       - run: |
           cd examples/protoc
-          . $BASH_ENV
+          . $(dirname $BASH_ENV)/env.bash
           protoc --version

README.md

@@ -32,9 +32,12 @@ In your project, create a configuration file `ghjk.ts`:
 
 ```ts
 export { ghjk } from "https://raw.githubusercontent.com/metatypedev/ghjk/main/mod.ts";
+import * as ghjk from "https://raw.githubusercontent.com/metatypedev/ghjk/main/mod.ts";
 import node from "https://raw.githubusercontent.com/metatypedev/ghjk/main/ports/node.ts";
 
-node({ version: "14.17.0" });
+ghjk.install(
+  node({ version: "14.17.0" }),
+);
 ```
 
 ## How it works
@@ -55,33 +58,18 @@ and looks as follows (abstracting away some implementation details):
     loaded ones (if any)
 - you can then
   - sync your runtime with `ghjk ports sync` which
-    - installs the missing tools at `$HOME/.local/share/ghjk/envs/$PWD/installs`
+    - installs the missing tools at `$HOME/.local/share/ghjk/ports/installs`
     - regenerates the shims with symlinks and environment variables
     - detects any violation of the enforced rules
-  - [ ] `ghjk list`: list installed tools and versions
-  - [ ] `ghjk outdated`: list outdated tools
-  - [ ] `ghjk cleanup`: remove unused tools and versions
+  - [ ] `ghjk ports list`: list installed tools and versions
+  - [ ] `ghjk ports outdated`: list outdated tools
+  - [ ] `ghjk ports cleanup`: remove unused tools and versions
 
 ## Extending `ghjk`
 
 ```ts
 ```
 
-## todo
-
-- multiple version of the same package (e.g. rust stable and rust nighted)
-- [ ] python with virtual env dir
-  - poetry
-  - pre-commit
-- [ ] rust toolchain
-- hash verifiable dependencies (timestamp)
-- hide the `Deno` object in an abstraction
-- support windows
-- [ ] installation tools
-  - [ ] untar
-  - [ ] xz
-  - [ ] git
-
 ## Development
 
 ```bash

check.ts

@@ -3,7 +3,7 @@ import { $ } from "./utils/mod.ts";
 
 const files = (await Array.fromAsync(
   $.path(import.meta.url).parentOrThrow().expandGlob("**/*.ts", {
-    exclude: ["./gh_action"],
+    exclude: [],
   }),
 )).map((ref) => ref.path.toString());
 

deno.jsonc

@@ -5,10 +5,10 @@
     "check": "deno run -A check.ts"
   },
   "fmt": {
-    "exclude": ["gh_action/*.js"]
+    "exclude": []
   },
   "lint": {
-    "exclude": ["gh_action/*", "ghjk.ts", "play.ts"],
+    "exclude": ["ghjk.ts", "play.ts"],
     "rules": {
       "exclude": ["no-explicit-any"]
     }

deps/common.ts

@@ -8,3 +8,5 @@ export * as std_url from "https://deno.land/[email protected]/url/mod.ts";
 export * as std_path from "https://deno.land/[email protected]/path/mod.ts";
 export * as std_fs from "https://deno.land/[email protected]/fs/mod.ts";
 export * as dax from "https://deno.land/x/[email protected]/mod.ts";
+export * as jsonHash from "https://deno.land/x/[email protected]/mod.ts";
+export * as equal from "https://deno.land/x/[email protected]/mod.ts";

examples/protoc/ghjk.ts

@@ -1,4 +1,7 @@
 export { ghjk } from "../../mod.ts";
+import { install } from "../../mod.ts";
 import protoc from "../../ports/protoc.ts";
 
-protoc({});
+install(
+  protoc(),
+);

ghjk.ts

@@ -1,4 +1,5 @@
 export { ghjk } from "./mod.ts";
+import * as ghjk from "./mod.ts";
 import node from "./ports/node.ts";
 import pnpm from "./ports/pnpm.ts";
 import cargo_binstall from "./ports/cargo-binstall.ts";
@@ -11,27 +12,42 @@ import mold from "./ports/mold.ts";
 import act from "./ports/act.ts";
 import asdf from "./ports/asdf.ts";
 import protoc from "./ports/protoc.ts";
-import earthly from "./ports/earthly.ts";
 import ruff from "./ports/ruff.ts";
 import whiz from "./ports/whiz.ts";
+import cpython from "./ports/cpy_bs.ts";
+import pipi from "./ports/pipi.ts";
 
-// node({});
-// wasmedge({});
-// pnpm({});
-// cargo_binstall({});
-// wasm_tools({});
-// wasm_opt({});
-// cargo_insta({});
-// jco({});
-// mold({
-//   replaceLd: true,
-// });
-act({});
-// asdf({
-//   pluginRepo: "https://github.com/asdf-community/asdf-cmake",
-//   installType: "version",
-// });
-// protoc({});
-// earthly({});
-// ruff({});
-// whiz({});
+// these are just for quick testing
+ghjk.install(
+  // node(),
+  // wasmedge(),
+  // pnpm(),
+  // cargo_binstall(),
+  // wasm_tools(),
+  // wasm_opt(),
+  // cargo_insta(),
+  // mold({
+  //   replaceLd: true,
+  // }),
+  // asdf({
+  // act(),
+  //   pluginRepo: "https://github.com/asdf-community/asdf-cmake",
+  //   installType: "version",
+  // }),
+  // ...pipi({ packageName: "pre-commit" }),
+  // protoc(),
+  // ruff(),
+  // whiz(),
+  // jco()[0],
+  // cpython(),
+);
+
+// these are used for developing ghjk
+ghjk.install(
+  act(),
+  ...pipi({ packageName: "pre-commit" }),
+);
+
+export const secureConfig = ghjk.secureConfig({
+  allowedPortDeps: [...ghjk.stdDeps({ enableRuntimes: true })],
+});

host/deno.ts

@@ -7,24 +7,13 @@ import { std_url } from "../deps/common.ts";
 
 import { inWorker } from "../utils/mod.ts";
 import logger, { setup as setupLogger } from "../utils/logger.ts";
-import type { PortsModuleConfigBase } from "../modules/ports/mod.ts";
 
 if (inWorker()) {
   initWorker();
 }
 
-declare global {
-  interface Window {
-    ports: PortsModuleConfigBase;
-  }
-}
-
 function initWorker() {
   setupLogger();
-  self.ports = {
-    ports: {},
-    installs: [],
-  };
 
   self.onmessage = onMsg;
 }

host/mod.ts

@@ -21,15 +21,14 @@ export async function cli(args: CliArgs) {
   );
   const envDir = envDirFromConfig(ghjkDir, configPath);
 
-  logger().debug({ configPath });
-  logger().debug({ envDir });
+  logger().debug({ configPath, envDir });
 
   const serializedConfig = await serializeConfig(configPath);
 
-  const ctx = { configPath, envDir };
+  const ctx = { ghjkDir, configPath, envDir };
   let cmd = new cliffy_cmd.Command()
     .name("ghjk")
-    .version("0.1.0") // FIXME: better way to resolve version
+    .version("0.1.1") // FIXME: better way to resolve version
     .description("Programmable runtime manager.")
     .action(function () {
       this.showHelp();
@@ -114,6 +113,11 @@ async function serializeConfig(configPath: string) {
         `unrecognized ghjk config type provided at path: ${configPath}`,
       );
   }
-  const serializedConfig = validators.serializedConfig.parse(serializedJson);
-  return serializedConfig;
+  const res = validators.serializedConfig.safeParse(serializedJson);
+  if (!res.success) {
+    logger().error("zod error", res.error);
+    logger().error("serializedConf", serializedJson);
+    throw new Error(`error parsing seralized config from ${configPath}`);
+  }
+  return res.data;
 }

install.ts

@@ -26,6 +26,8 @@ if (import.meta.main) {
     shellHookMarker: Deno.env.get("GHJK_INSTALL_HOOK_MARKER") ??
       defaultInstallArgs.shellHookMarker,
     noLockfile: !!noLockfile && noLockfile != "0" && noLockfile != "false",
+    ghjkDenoCacheDir: Deno.env.get("GHJK_INSTALL_DENO_DIR") ??
+      defaultInstallArgs.ghjkDenoCacheDir,
   });
 } else {
   throw new Error(

install/mod.ts

@@ -1,8 +1,6 @@
 //! this installs the different shell ghjk hooks in ~/.local/share/ghjk
 //! and a `ghjk` bin at ~/.local/share/bin
 
-// TODO: support for different environments to use different versions of ghjk
-
 import logger from "../utils/logger.ts";
 import { std_fs, std_path } from "../deps/cli.ts";
 import { $, dirs, importRaw } from "../utils/mod.ts";
@@ -111,6 +109,8 @@ interface InstallArgs {
   /// the deno exec to be used by the ghjk executable
   /// by default will be "deno" i.e. whatever the shell resolves that to
   ghjkExecDenoExec: string;
+  /// The cache dir to use by the ghjk deno installation
+  ghjkDenoCacheDir?: string;
   // Disable using a lockfile for the ghjk command
   noLockfile: boolean;
 }
@@ -124,6 +124,8 @@ export const defaultInstallArgs: InstallArgs = {
   // TODO: respect xdg dirs
   ghjkExecInstallDir: std_path.resolve(dirs().homeDir, ".local", "bin"),
   ghjkExecDenoExec: Deno.execPath(),
+  // the default behvaior kicks in with ghjkDenoCacheDir is falsy
+  // ghjkDenoCacheDir: undefined,
   noLockfile: false,
 };
 
@@ -188,10 +190,13 @@ export async function install(
           ? "--no-lock"
           : `--lock $GHJK_DIR/deno.lock`;
 
+        // use an isolated cache by default
+        const denoCacheDir = args.ghjkDenoCacheDir ??
+          std_path.resolve(ghjkDir, "deno");
         await Deno.writeTextFile(
           exePath,
           `#!/bin/sh 
-GHJK_DIR="$\{GHJK_DIR:-${ghjkDir}}" DENO_DIR=${args.ghjkExecInstallDir}/cache
+GHJK_DIR="$\{GHJK_DIR:-${ghjkDir}}" DENO_DIR="$\{GHJK_DENO_DIR:-${denoCacheDir}}"
 ${args.ghjkExecDenoExec} run --unstable-worker-options -A ${lockFlag} ${
             import.meta.resolve("../main.ts")
           } $*`,