From 9430a8d61ed1208fb185a71fb805d23d365711f5 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Wed, 14 Aug 2024 10:27:49 +0300 Subject: [PATCH 01/16] wip: python venv --- .ghjk/lock.json | 27 +++++++++++++++++++++++ examples/kitchen/.gitignore | 1 + examples/kitchen/ghjk.ts | 5 +++++ files/mod.ts | 16 +++++++++++++- library/py.ts | 44 +++++++++++++++++++++++++++++++++++++ modules/envs/mod.ts | 2 +- 6 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 examples/kitchen/.gitignore create mode 100644 library/py.ts diff --git a/.ghjk/lock.json b/.ghjk/lock.json index 88c7320..7928670 100644 --- a/.ghjk/lock.json +++ b/.ghjk/lock.json @@ -80,6 +80,33 @@ "buildDepConfigs": {}, "portRef": "deno_ghrel@0.1.0", "specifiedVersion": true + }, + "bciqao2s3r3r33ruox4qknfrxqrmemuccxn64dze2ylojrzp2bwvt4ji": { + "version": "3.8.0", + "buildDepConfigs": { + "cpy_bs_ghrel": { + "version": "3.12.4", + "buildDepConfigs": { + "tar_aa": { + "version": "1.35", + "buildDepConfigs": {}, + "portRef": "tar_aa@0.1.0", + "specifiedVersion": false + }, + "zstd_aa": { + "version": "v1.5.6,", + "buildDepConfigs": {}, + "portRef": "zstd_aa@0.1.0", + "specifiedVersion": false + } + }, + "portRef": "cpy_bs_ghrel@0.1.0", + "specifiedVersion": false + } + }, + "portRef": "pipi_pypi@0.1.0", + "packageName": "pre-commit", + "specifiedVersion": false } } }, diff --git a/examples/kitchen/.gitignore b/examples/kitchen/.gitignore new file mode 100644 index 0000000..21d0b89 --- /dev/null +++ b/examples/kitchen/.gitignore @@ -0,0 +1 @@ +.venv/ diff --git a/examples/kitchen/ghjk.ts b/examples/kitchen/ghjk.ts index a88897e..a175549 100644 --- a/examples/kitchen/ghjk.ts +++ b/examples/kitchen/ghjk.ts @@ -1,6 +1,7 @@ import { stdDeps } from "../../files/mod.ts"; import { file } from "../../mod.ts"; import * as ports from "../../ports/mod.ts"; +import { pyEnv } from "../../library/py.ts"; const ghjk = file({ // configre an empty env so that no ports are avail by default in our workdir @@ -120,3 +121,7 @@ env("dev") workingDir: "..", fn: ($) => $`ls`, })); + +// -- +env("venv") + .use(pyEnv({ version: "3.8.18", releaseTag: "20240224" })); diff --git a/files/mod.ts b/files/mod.ts index 44c0322..53167f2 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -120,7 +120,7 @@ export type DenoTaskDefArgs = TaskDefArgs & { * execution of a specific task, we identify each using a hash. * The {@field fn} is `toString`ed in the hash input. * If a ghjkfile is produing identical anonymous tasks for - * instance, it can provide a none to disambiguate beteween each + * instance, it can provide a nonce to disambiguate between each * through hash differences. * * NOTE: the nonce must be stable across serialization. @@ -1076,6 +1076,20 @@ export class EnvBuilder { this.#onExitHookTasks.push(...taskKey); return this; } + + use( + setup: ( + builder: EnvBuilder, + ghjk: { task(args: DenoTaskDefArgs): void }, + ) => void, + ) { + setup(this, { + task: (args) => { + return this.#file.addTask({ ...args, ty: "denoFile@v1" }); + }, + }); + return this; + } } export function stdDeps(args = { enableRuntimes: false }) { diff --git a/library/py.ts b/library/py.ts new file mode 100644 index 0000000..5b3c3ba --- /dev/null +++ b/library/py.ts @@ -0,0 +1,44 @@ +import { EnvBuilder } from "../files/mod.ts"; +import { cpy_bs } from "../ports/mod.ts"; +import * as ports from "../ports/mod.ts"; + +interface PyEnvConfig { + /** Python version */ + version: string; + releatTag: string; + /** venv dir, relative to Ghjk dir; default: ".venv" */ + dir?: string; + /** create the venv if missing; default: true */ + create?: boolean; +} + +export function pyEnv( + { version, releaseTag, dir = ".venv", create = true }: PyEnvConfig = {}, +) { + return (builder: EnvBuilder, ghjk) => { + console.log({ version, releaseTag }); + if (create) { + builder.onEnter(ghjk.task({ + name: "activate-py-venv", + installs: [ + ports.cpy_bs({ version, releaseTag }), + ports.jq_ghrel(), + ], + vars: { STUFF: "stuffier" }, + fn: async ($, { workingDir }) => { + console.log("dir", { dir, workingDir }); + const venvDir = $.path(workingDir).join(dir); + console.log(await venvDir.exists()); + if (!(await venvDir.exists())) { + await $`echo "Creating python venv at ${dir}"`; + await $`python3 -m venv ${dir}`; + } + await $`python3 --version`; + await $`echo $STUFF; jq --version`; + return $`echo enter`; + }, + installs: [], + })); + } + }; +} diff --git a/modules/envs/mod.ts b/modules/envs/mod.ts index 6258f72..3778540 100644 --- a/modules/envs/mod.ts +++ b/modules/envs/mod.ts @@ -394,7 +394,7 @@ async function activateEnv(envKey: string) { const shell = await detectShellPath(); if (!shell) { throw new Error( - "unable to detct shell in use. Use `--shell` flag to explicitly pass shell program.", + "unable to detect shell in use. Use `--shell` flag to explicitly pass shell program.", ); } // FIXME: the ghjk process will be around and consumer resources From 6d6e72d651047cdafa31707819ce6f475bb005a0 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Thu, 15 Aug 2024 15:02:40 +0300 Subject: [PATCH 02/16] feat: bin dir --- .ghjk/lock.json | 27 ---------------------- examples/kitchen/ghjk.ts | 1 - files/mod.ts | 48 ++++++++++++++++++++++++++++++++++++++-- library/py.ts | 25 +++++++++++---------- modules/envs/posix.ts | 7 +++++- modules/envs/reducer.ts | 40 ++++++++++++++++++++++++++++++++- modules/envs/types.ts | 6 +++++ 7 files changed, 110 insertions(+), 44 deletions(-) diff --git a/.ghjk/lock.json b/.ghjk/lock.json index 7928670..30c21bb 100644 --- a/.ghjk/lock.json +++ b/.ghjk/lock.json @@ -17,33 +17,6 @@ "portRef": "act_ghrel@0.1.0", "specifiedVersion": false }, - "bciqao2s3r3r33ruox4qknfrxqrmemuccxn64dze2ylojrzp2bwvt4ji": { - "version": "3.7.1", - "buildDepConfigs": { - "cpy_bs_ghrel": { - "version": "3.12.4", - "buildDepConfigs": { - "tar_aa": { - "version": "1.34", - "buildDepConfigs": {}, - "portRef": "tar_aa@0.1.0", - "specifiedVersion": false - }, - "zstd_aa": { - "version": "v1.4.8,", - "buildDepConfigs": {}, - "portRef": "zstd_aa@0.1.0", - "specifiedVersion": false - } - }, - "portRef": "cpy_bs_ghrel@0.1.0", - "specifiedVersion": false - } - }, - "portRef": "pipi_pypi@0.1.0", - "packageName": "pre-commit", - "specifiedVersion": false - }, "bciqij3g6mmbjn4a6ps4eipcy2fmw2zumgv5a3gbxycthroffihwquoi": { "version": "3.12.4", "buildDepConfigs": { diff --git a/examples/kitchen/ghjk.ts b/examples/kitchen/ghjk.ts index a175549..695a2f1 100644 --- a/examples/kitchen/ghjk.ts +++ b/examples/kitchen/ghjk.ts @@ -122,6 +122,5 @@ env("dev") fn: ($) => $`ls`, })); -// -- env("venv") .use(pyEnv({ version: "3.8.18", releaseTag: "20240224" })); diff --git a/files/mod.ts b/files/mod.ts index 53167f2..57579b8 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -663,6 +663,14 @@ export class Ghjkfile { prov, ); }), + ...final.binDirs.map((path) => { + const prov: WellKnownProvision = { ty: "posix.binDir", path }; + return prov; + }), + ...final.dynBinDirs.map((taskKey) => { + const prov = { ty: "posix.binDirDyn", taskKey }; + return prov; + }), // env hooks ...hooks, ], @@ -897,6 +905,8 @@ type EnvFinalizer = () => { inherit: string | string[] | boolean; vars: Record; dynVars: Record; + binDirs: string[]; + dynBinDirs: string[]; desc?: string; onEnterHookTasks: string[]; onExitHookTasks: string[]; @@ -908,8 +918,12 @@ export type EnvDefArgsPartial = export type DynEnvValue = | (() => string | number) - | (($_: typeof $) => string | number) - | (($_: typeof $) => Promise); + | (($_: typeof $, args: ExecTaskArgs) => string | number) + | (($_: typeof $, args: ExecTaskArgs) => Promise); + +export type DynPathValue = + | ((...params: Parameters) => string) + | ((...params: Parameters) => Promise); // // /** @@ -956,6 +970,8 @@ export class EnvBuilder { #inherit: string | string[] | boolean = true; #vars: Record = {}; #dynVars: Record = {}; + #binDirs: string[] = []; + #dynBinDirs: string[] = []; #desc?: string; #onEnterHookTasks: string[] = []; #onExitHookTasks: string[] = []; @@ -977,6 +993,8 @@ export class EnvBuilder { Object.entries(this.#vars).map(([key, val]) => [key, val.toString()]), ), dynVars: this.#dynVars, + binDirs: this.#binDirs, + dynBinDirs: this.#dynBinDirs, desc: this.#desc, onExitHookTasks: this.#onExitHookTasks, onEnterHookTasks: this.#onEnterHookTasks, @@ -1053,6 +1071,32 @@ export class EnvBuilder { return this; } + /** + * Add a directory to $PATH + */ + binDir(path: string | DynPathValue) { + switch (typeof path) { + case "string": + // TODO: validate path + this.#binDirs.push(path); + break; + + case "function": { + const taskKey = this.#file.addTask({ + ty: "denoFile@v1", + fn: path, + }); + this.#dynBinDirs.push(taskKey); + break; + } + + default: + throw new Error(`type "${typeof path}" is not supported for path`); + } + + return this; + } + /** * Description of the environment. */ diff --git a/library/py.ts b/library/py.ts index 5b3c3ba..b6bc98a 100644 --- a/library/py.ts +++ b/library/py.ts @@ -1,5 +1,4 @@ import { EnvBuilder } from "../files/mod.ts"; -import { cpy_bs } from "../ports/mod.ts"; import * as ports from "../ports/mod.ts"; interface PyEnvConfig { @@ -16,29 +15,31 @@ export function pyEnv( { version, releaseTag, dir = ".venv", create = true }: PyEnvConfig = {}, ) { return (builder: EnvBuilder, ghjk) => { - console.log({ version, releaseTag }); + builder.install( + ports.cpy_bs({ version, releaseTag }), + ); if (create) { builder.onEnter(ghjk.task({ name: "activate-py-venv", - installs: [ - ports.cpy_bs({ version, releaseTag }), - ports.jq_ghrel(), - ], vars: { STUFF: "stuffier" }, fn: async ($, { workingDir }) => { - console.log("dir", { dir, workingDir }); const venvDir = $.path(workingDir).join(dir); - console.log(await venvDir.exists()); if (!(await venvDir.exists())) { await $`echo "Creating python venv at ${dir}"`; await $`python3 -m venv ${dir}`; } - await $`python3 --version`; - await $`echo $STUFF; jq --version`; - return $`echo enter`; }, - installs: [], })); } + + builder.var("VIRTUAL_ENV", ($, { workingDir }) => { + const venvDir = $.path(workingDir).join(dir); + return venvDir.toString(); + }); + + builder.binDir(($, { workingDir }) => { + const path = $.path(workingDir).join(dir).join("bin"); + return path.toString(); + }); }; } diff --git a/modules/envs/posix.ts b/modules/envs/posix.ts index 5e775f5..34e1c49 100644 --- a/modules/envs/posix.ts +++ b/modules/envs/posix.ts @@ -38,6 +38,7 @@ export async function cookPosixEnv( const binPaths = [] as string[]; const libPaths = [] as string[]; const includePaths = [] as string[]; + const binDirs = [] as string[]; const vars = { GHJK_ENV: envKey, } as Record; @@ -74,6 +75,9 @@ export async function cookPosixEnv( vars[wellKnownProv.key] = wellKnownProv.val; // installSetIds.push(wellKnownProv.installSetIdProvision!.id); break; + case "posix.binDir": + binDirs.push(wellKnownProv.path); + break; case "hook.onEnter.posixExec": onEnterHooks.push([wellKnownProv.program, wellKnownProv.arguments]); break; @@ -119,8 +123,9 @@ export async function cookPosixEnv( default: throw new Error(`unsupported os ${Deno.build.os}`); } + binDirs.push(`${envDir}/shims/bin`); const pathVars = { - PATH: `${envDir}/shims/bin`, + PATH: binDirs.join(":"), LIBRARY_PATH: `${envDir}/shims/lib`, [LD_LIBRARY_ENV]: `${envDir}/shims/lib`, C_INCLUDE_PATH: `${envDir}/shims/include`, diff --git a/modules/envs/reducer.ts b/modules/envs/reducer.ts index 64cfd70..3137c0c 100644 --- a/modules/envs/reducer.ts +++ b/modules/envs/reducer.ts @@ -9,7 +9,7 @@ import type { WellKnownEnvRecipeX, WellKnownProvision, } from "./types.ts"; -import { envVarDynTy, wellKnownProvisionTypes } from "./types.ts"; +import { binPathDynTy, envVarDynTy, wellKnownProvisionTypes } from "./types.ts"; import validators from "./types.ts"; export type ProvisionReducerStore = Map< @@ -37,6 +37,10 @@ export function getProvisionReducerStore( envVarDynTy, installDynEnvReducer(gcx) as ProvisionReducer, ); + store?.set( + binPathDynTy, + installDynBinPathReducer(gcx) as ProvisionReducer, + ); return store; } @@ -126,3 +130,37 @@ export function installDynEnvReducer(gcx: GhjkCtx) { return output; }; } + +export function installDynBinPathReducer(gcx: GhjkCtx) { + return async (provisions: Provision[]) => { + const output = []; + const badProvisions = []; + const taskCtx = getTasksCtx(gcx); + + for (const provision of provisions) { + const ty = "posix.binDir"; + const key = provision.taskKey; + + const taskGraph = taskCtx.taskGraph; + const taskConf = taskCtx.config; + + const targetKey = Object.entries(taskConf.tasks) + .find(([_, task]) => task.key === key)?.[0]; + + if (targetKey) { + const results = await execTask(gcx, taskConf, taskGraph, targetKey, []); + output.push({ ...provision, ty, path: results[key] as string }); + } else { + badProvisions.push(provision); + } + + if (badProvisions.length >= 1) { + throw new Error("cannot deduce task from keys", { + cause: { badProvisions }, + }); + } + + return output; + } + }; +} diff --git a/modules/envs/types.ts b/modules/envs/types.ts index 1a0372c..372091d 100644 --- a/modules/envs/types.ts +++ b/modules/envs/types.ts @@ -22,11 +22,13 @@ export const installProvisionTypes = [ ] as const; export const envVarDynTy = "posix.envVarDyn"; +export const binPathDynTy = "posix.binDirDyn"; // we separate the posix file types in a separate // array in the interest of type inference export const wellKnownProvisionTypes = [ "posix.envVar", + "posix.binDir", ...posixFileProvisionTypes, ...hookProvisionTypes, ...installProvisionTypes, @@ -40,6 +42,10 @@ const wellKnownProvision = zod.discriminatedUnion( key: moduleValidators.envVarName, val: zod.string(), }), + zod.object({ + ty: zod.literal(wellKnownProvisionTypes[1]), + path: absolutePath, + }), ...hookProvisionTypes.map((ty) => zod.object({ ty: zod.literal(ty), From 93e3ba812e7e3b30e4b45ab62e13d584a9a45624 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Thu, 15 Aug 2024 16:02:23 +0300 Subject: [PATCH 03/16] fix typo --- library/py.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/library/py.ts b/library/py.ts index b6bc98a..ec466bd 100644 --- a/library/py.ts +++ b/library/py.ts @@ -4,7 +4,7 @@ import * as ports from "../ports/mod.ts"; interface PyEnvConfig { /** Python version */ version: string; - releatTag: string; + releaseTag: string; /** venv dir, relative to Ghjk dir; default: ".venv" */ dir?: string; /** create the venv if missing; default: true */ @@ -21,7 +21,6 @@ export function pyEnv( if (create) { builder.onEnter(ghjk.task({ name: "activate-py-venv", - vars: { STUFF: "stuffier" }, fn: async ($, { workingDir }) => { const venvDir = $.path(workingDir).join(dir); if (!(await venvDir.exists())) { From 15cd07dcaa76448cf8f5d334cb481bb2a14f150e Mon Sep 17 00:00:00 2001 From: Natoandro Date: Thu, 15 Aug 2024 16:06:14 +0300 Subject: [PATCH 04/16] fix typing errors --- files/mod.ts | 5 ++--- modules/envs/reducer.ts | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/files/mod.ts b/files/mod.ts index 57579b8..f48477a 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -917,9 +917,8 @@ export type EnvDefArgsPartial = & Omit; export type DynEnvValue = - | (() => string | number) - | (($_: typeof $, args: ExecTaskArgs) => string | number) - | (($_: typeof $, args: ExecTaskArgs) => Promise); + | ((...params: Parameters) => string | number) + | ((...params: Parameters) => Promise); export type DynPathValue = | ((...params: Parameters) => string) diff --git a/modules/envs/reducer.ts b/modules/envs/reducer.ts index 3137c0c..f93692c 100644 --- a/modules/envs/reducer.ts +++ b/modules/envs/reducer.ts @@ -139,7 +139,7 @@ export function installDynBinPathReducer(gcx: GhjkCtx) { for (const provision of provisions) { const ty = "posix.binDir"; - const key = provision.taskKey; + const key = provision.taskKey as string; const taskGraph = taskCtx.taskGraph; const taskConf = taskCtx.config; From 7fde9d5bd326c068ce9146800f8b0d4b0cdcc187 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Mon, 19 Aug 2024 10:09:27 +0300 Subject: [PATCH 05/16] more provisions for posix dirs --- examples/kitchen/ghjk.ts | 4 +-- files/mod.ts | 65 +++++++++++++++++++++++++--------------- modules/envs/posix.ts | 28 +++++++++++------ modules/envs/reducer.ts | 26 +++++++++++----- modules/envs/types.ts | 42 ++++++++++++++++++++++---- {library => std}/py.ts | 2 +- 6 files changed, 118 insertions(+), 49 deletions(-) rename {library => std}/py.ts (96%) diff --git a/examples/kitchen/ghjk.ts b/examples/kitchen/ghjk.ts index 695a2f1..7edc7db 100644 --- a/examples/kitchen/ghjk.ts +++ b/examples/kitchen/ghjk.ts @@ -1,7 +1,7 @@ import { stdDeps } from "../../files/mod.ts"; import { file } from "../../mod.ts"; import * as ports from "../../ports/mod.ts"; -import { pyEnv } from "../../library/py.ts"; +import { pyEnv } from "../../std/py.ts"; const ghjk = file({ // configre an empty env so that no ports are avail by default in our workdir @@ -123,4 +123,4 @@ env("dev") })); env("venv") - .use(pyEnv({ version: "3.8.18", releaseTag: "20240224" })); + .mixin(pyEnv({ version: "3.8.18", releaseTag: "20240224" })); diff --git a/files/mod.ts b/files/mod.ts index f48477a..715c1eb 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -37,8 +37,11 @@ import type { ExecTaskArgs } from "../modules/tasks/deno.ts"; import { TaskDefHashed, TasksModuleConfig } from "../modules/tasks/types.ts"; // envs import { + DynamicPathVarProvision, type EnvRecipe, type EnvsModuleConfig, + PosixDirProvision, + PosixDirProvisionType, type Provision, type WellKnownProvision, } from "../modules/envs/types.ts"; @@ -663,14 +666,8 @@ export class Ghjkfile { prov, ); }), - ...final.binDirs.map((path) => { - const prov: WellKnownProvision = { ty: "posix.binDir", path }; - return prov; - }), - ...final.dynBinDirs.map((taskKey) => { - const prov = { ty: "posix.binDirDyn", taskKey }; - return prov; - }), + ...final.posixDirs, + ...final.dynamicPathVars, // env hooks ...hooks, ], @@ -905,7 +902,8 @@ type EnvFinalizer = () => { inherit: string | string[] | boolean; vars: Record; dynVars: Record; - binDirs: string[]; + posixDirs: Array; + dynamicPathVars: Array; dynBinDirs: string[]; desc?: string; onEnterHookTasks: string[]; @@ -920,7 +918,7 @@ export type DynEnvValue = | ((...params: Parameters) => string | number) | ((...params: Parameters) => Promise); -export type DynPathValue = +export type DynamicPathVarFn = | ((...params: Parameters) => string) | ((...params: Parameters) => Promise); @@ -969,8 +967,8 @@ export class EnvBuilder { #inherit: string | string[] | boolean = true; #vars: Record = {}; #dynVars: Record = {}; - #binDirs: string[] = []; - #dynBinDirs: string[] = []; + #posixDirs: Array = []; + #dynamicPathVars: Array = []; #desc?: string; #onEnterHookTasks: string[] = []; #onExitHookTasks: string[] = []; @@ -992,8 +990,8 @@ export class EnvBuilder { Object.entries(this.#vars).map(([key, val]) => [key, val.toString()]), ), dynVars: this.#dynVars, - binDirs: this.#binDirs, - dynBinDirs: this.#dynBinDirs, + posixDirs: this.#posixDirs, + dynamicPathVars: this.#dynamicPathVars, desc: this.#desc, onExitHookTasks: this.#onExitHookTasks, onEnterHookTasks: this.#onEnterHookTasks, @@ -1071,31 +1069,50 @@ export class EnvBuilder { } /** - * Add a directory to $PATH + * Add a directory to the path environment variable + * $PATH, $LD_LIBRARY_PATH, $INCLUDE_PATH, etc. */ - binDir(path: string | DynPathValue) { - switch (typeof path) { - case "string": - // TODO: validate path - this.#binDirs.push(path); + pathVar(type: PosixDirProvisionType, val: string | DynamicPathVarFn) { + switch (typeof val) { + case "string": { + const prov = { ty: type, path: val }; + this.#posixDirs.push(unwrapZodRes( + envsValidators.pathVarProvision.safeParse(prov), + prov, + )); break; + } case "function": { const taskKey = this.#file.addTask({ ty: "denoFile@v1", - fn: path, + fn: val, }); - this.#dynBinDirs.push(taskKey); + const prov = { ty: type + ".dynamic", taskKey }; + this.#dynamicPathVars.push(unwrapZodRes( + envsValidators.dynamicPathVarProvision.safeParse(prov), + prov, + )); break; } default: - throw new Error(`type "${typeof path}" is not supported for path`); + throw new Error(`type "${typeof val}" is not supported for path`); } return this; } + execDir(val: string | DynamicPathVarFn) { + return this.pathVar("posix.execDir", val); + } + sharedLibDir(val: string | DynamicPathVarFn) { + return this.pathVar("posix.sharedLibDir", val); + } + headerDir(val: string | DynamicPathVarFn) { + return this.pathVar("posix.headerDir", val); + } + /** * Description of the environment. */ @@ -1120,7 +1137,7 @@ export class EnvBuilder { return this; } - use( + mixin( setup: ( builder: EnvBuilder, ghjk: { task(args: DenoTaskDefArgs): void }, diff --git a/modules/envs/posix.ts b/modules/envs/posix.ts index 34e1c49..21f2d26 100644 --- a/modules/envs/posix.ts +++ b/modules/envs/posix.ts @@ -38,7 +38,9 @@ export async function cookPosixEnv( const binPaths = [] as string[]; const libPaths = [] as string[]; const includePaths = [] as string[]; - const binDirs = [] as string[]; + const execDirs = [] as string[]; + const sharedLibDirs = [] as string[]; + const headerDirs = [] as string[]; const vars = { GHJK_ENV: envKey, } as Record; @@ -75,8 +77,14 @@ export async function cookPosixEnv( vars[wellKnownProv.key] = wellKnownProv.val; // installSetIds.push(wellKnownProv.installSetIdProvision!.id); break; - case "posix.binDir": - binDirs.push(wellKnownProv.path); + case "posix.execDir": + execDirs.push(wellKnownProv.path); + break; + case "posix.sharedLibDir": + sharedLibDirs.push(wellKnownProv.path); + break; + case "posix.headerDir": + headerDirs.push(wellKnownProv.path); break; case "hook.onEnter.posixExec": onEnterHooks.push([wellKnownProv.program, wellKnownProv.arguments]); @@ -123,13 +131,15 @@ export async function cookPosixEnv( default: throw new Error(`unsupported os ${Deno.build.os}`); } - binDirs.push(`${envDir}/shims/bin`); + execDirs.push(`${envDir}/shims/bin`); + sharedLibDirs.push(`${envDir}/shims/lib`); + headerDirs.push(`${envDir}/shims/include`); const pathVars = { - PATH: binDirs.join(":"), - LIBRARY_PATH: `${envDir}/shims/lib`, - [LD_LIBRARY_ENV]: `${envDir}/shims/lib`, - C_INCLUDE_PATH: `${envDir}/shims/include`, - CPLUS_INCLUDE_PATH: `${envDir}/shims/include`, + PATH: execDirs.join(":"), + LIBRARY_PATH: sharedLibDirs.join(":"), + [LD_LIBRARY_ENV]: sharedLibDirs.join(":"), + C_INCLUDE_PATH: headerDirs.join(":"), + CPLUS_INCLUDE_PATH: headerDirs.join(":"), }; if (createShellLoaders) { // write loader for the env vars mandated by the installs diff --git a/modules/envs/reducer.ts b/modules/envs/reducer.ts index f93692c..c27172d 100644 --- a/modules/envs/reducer.ts +++ b/modules/envs/reducer.ts @@ -4,12 +4,17 @@ import { getTasksCtx } from "../tasks/inter.ts"; import type { GhjkCtx } from "../types.ts"; import type { EnvRecipeX, + PosixDirProvisionType, Provision, ProvisionReducer, WellKnownEnvRecipeX, WellKnownProvision, } from "./types.ts"; -import { binPathDynTy, envVarDynTy, wellKnownProvisionTypes } from "./types.ts"; +import { + envVarDynTy, + posixDirProvisionTypes, + wellKnownProvisionTypes, +} from "./types.ts"; import validators from "./types.ts"; export type ProvisionReducerStore = Map< @@ -37,10 +42,15 @@ export function getProvisionReducerStore( envVarDynTy, installDynEnvReducer(gcx) as ProvisionReducer, ); - store?.set( - binPathDynTy, - installDynBinPathReducer(gcx) as ProvisionReducer, - ); + for (const ty of posixDirProvisionTypes) { + store?.set( + ty + ".dynamic", + installDynamicPathVarReducer(gcx, ty) as ProvisionReducer< + Provision, + Provision + >, + ); + } return store; } @@ -131,14 +141,16 @@ export function installDynEnvReducer(gcx: GhjkCtx) { }; } -export function installDynBinPathReducer(gcx: GhjkCtx) { +export function installDynamicPathVarReducer( + gcx: GhjkCtx, + ty: PosixDirProvisionType, +) { return async (provisions: Provision[]) => { const output = []; const badProvisions = []; const taskCtx = getTasksCtx(gcx); for (const provision of provisions) { - const ty = "posix.binDir"; const key = provision.taskKey as string; const taskGraph = taskCtx.taskGraph; diff --git a/modules/envs/types.ts b/modules/envs/types.ts index 372091d..263cb61 100644 --- a/modules/envs/types.ts +++ b/modules/envs/types.ts @@ -12,6 +12,27 @@ const posixFileProvisionTypes = [ "posix.headerFile", ] as const; +export const posixDirProvisionTypes = [ + "posix.execDir", + "posix.sharedLibDir", + "posix.headerDir", +] as const; + +export type PosixDirProvisionType = typeof posixDirProvisionTypes[number]; + +const posixDirProvision = zod.object({ + ty: zod.enum(posixDirProvisionTypes), + path: absolutePath, +}); + +export type PosixDirProvision = zod.infer; + +const dynamicPathVarProvisionTypes = [ + "posix.execDir.dynamic", + "posix.sharedLibDir.dynamic", + "posix.headerDir.dynamic", +] as const; + export const hookProvisionTypes = [ "hook.onEnter.posixExec", "hook.onExit.posixExec", @@ -22,13 +43,12 @@ export const installProvisionTypes = [ ] as const; export const envVarDynTy = "posix.envVarDyn"; -export const binPathDynTy = "posix.binDirDyn"; // we separate the posix file types in a separate // array in the interest of type inference export const wellKnownProvisionTypes = [ "posix.envVar", - "posix.binDir", + ...posixDirProvisionTypes, ...posixFileProvisionTypes, ...hookProvisionTypes, ...installProvisionTypes, @@ -42,10 +62,12 @@ const wellKnownProvision = zod.discriminatedUnion( key: moduleValidators.envVarName, val: zod.string(), }), - zod.object({ - ty: zod.literal(wellKnownProvisionTypes[1]), - path: absolutePath, - }), + ...posixDirProvisionTypes.map((ty) => + zod.object({ + ty: zod.literal(ty), + path: absolutePath, + }) + ), ...hookProvisionTypes.map((ty) => zod.object({ ty: zod.literal(ty), @@ -92,10 +114,18 @@ const envVarDynProvision = zod.object({ taskKey: zod.string(), }); +const dynamicPathVarProvision = zod.object({ + ty: zod.enum(dynamicPathVarProvisionTypes), + taskKey: zod.string(), +}); +export type DynamicPathVarProvision = zod.infer; + const validators = { provision, wellKnownProvision, envVarDynProvision, + posixDirProvision, + dynamicPathVarProvision, envRecipe, envsModuleConfig, wellKnownEnvRecipe, diff --git a/library/py.ts b/std/py.ts similarity index 96% rename from library/py.ts rename to std/py.ts index ec466bd..6fd7a5a 100644 --- a/library/py.ts +++ b/std/py.ts @@ -36,7 +36,7 @@ export function pyEnv( return venvDir.toString(); }); - builder.binDir(($, { workingDir }) => { + builder.execDir(($, { workingDir }) => { const path = $.path(workingDir).join(dir).join("bin"); return path.toString(); }); From 930a169b022f7a44cd832b95ec5751fd0c1b0dc8 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Mon, 19 Aug 2024 10:18:31 +0300 Subject: [PATCH 06/16] add to EnvDefArgs --- files/mod.ts | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/files/mod.ts b/files/mod.ts index 715c1eb..594542d 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -74,6 +74,9 @@ export type EnvDefArgs = { inherit?: EnvParent; desc?: string; vars?: Record; + execDirs?: string[]; + sharedLibDirs?: string[]; + headerDirs?: string[]; /** * Task to execute when environment is activated. */ @@ -283,6 +286,21 @@ export class Ghjkfile { if (args.vars) { env.vars(args.vars); } + if (args.execDirs) { + for (const dir of args.execDirs) { + env.execDir(dir); + } + } + if (args.sharedLibDirs) { + for (const dir of args.sharedLibDirs) { + env.sharedLibDir(dir); + } + } + if (args.headerDirs) { + for (const dir of args.headerDirs) { + env.headerDir(dir); + } + } if (args.onEnter) { env.onEnter(...args.onEnter); } @@ -667,7 +685,7 @@ export class Ghjkfile { ); }), ...final.posixDirs, - ...final.dynamicPathVars, + ...final.dynamicPosixDirs, // env hooks ...hooks, ], @@ -903,7 +921,7 @@ type EnvFinalizer = () => { vars: Record; dynVars: Record; posixDirs: Array; - dynamicPathVars: Array; + dynamicPosixDirs: Array; dynBinDirs: string[]; desc?: string; onEnterHookTasks: string[]; @@ -968,7 +986,7 @@ export class EnvBuilder { #vars: Record = {}; #dynVars: Record = {}; #posixDirs: Array = []; - #dynamicPathVars: Array = []; + #dynamicPosixDirs: Array = []; #desc?: string; #onEnterHookTasks: string[] = []; #onExitHookTasks: string[] = []; @@ -991,7 +1009,7 @@ export class EnvBuilder { ), dynVars: this.#dynVars, posixDirs: this.#posixDirs, - dynamicPathVars: this.#dynamicPathVars, + dynamicPosixDirs: this.#dynamicPosixDirs, desc: this.#desc, onExitHookTasks: this.#onExitHookTasks, onEnterHookTasks: this.#onEnterHookTasks, @@ -1072,7 +1090,7 @@ export class EnvBuilder { * Add a directory to the path environment variable * $PATH, $LD_LIBRARY_PATH, $INCLUDE_PATH, etc. */ - pathVar(type: PosixDirProvisionType, val: string | DynamicPathVarFn) { + posixDir(type: PosixDirProvisionType, val: string | DynamicPathVarFn) { switch (typeof val) { case "string": { const prov = { ty: type, path: val }; @@ -1089,7 +1107,7 @@ export class EnvBuilder { fn: val, }); const prov = { ty: type + ".dynamic", taskKey }; - this.#dynamicPathVars.push(unwrapZodRes( + this.#dynamicPosixDirs.push(unwrapZodRes( envsValidators.dynamicPathVarProvision.safeParse(prov), prov, )); @@ -1104,13 +1122,13 @@ export class EnvBuilder { } execDir(val: string | DynamicPathVarFn) { - return this.pathVar("posix.execDir", val); + return this.posixDir("posix.execDir", val); } sharedLibDir(val: string | DynamicPathVarFn) { - return this.pathVar("posix.sharedLibDir", val); + return this.posixDir("posix.sharedLibDir", val); } headerDir(val: string | DynamicPathVarFn) { - return this.pathVar("posix.headerDir", val); + return this.posixDir("posix.headerDir", val); } /** From d533d825a20e22de992baa8cfee3ed73c1b9b502 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Mon, 19 Aug 2024 10:22:18 +0300 Subject: [PATCH 07/16] optional python install --- examples/kitchen/ghjk.ts | 2 +- std/py.ts | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/examples/kitchen/ghjk.ts b/examples/kitchen/ghjk.ts index 7edc7db..f4b4832 100644 --- a/examples/kitchen/ghjk.ts +++ b/examples/kitchen/ghjk.ts @@ -123,4 +123,4 @@ env("dev") })); env("venv") - .mixin(pyEnv({ version: "3.8.18", releaseTag: "20240224" })); + .mixin(pyEnv({ install: { version: "3.8.18", releaseTag: "20240224" } })); diff --git a/std/py.ts b/std/py.ts index 6fd7a5a..ba5b1c2 100644 --- a/std/py.ts +++ b/std/py.ts @@ -2,9 +2,11 @@ import { EnvBuilder } from "../files/mod.ts"; import * as ports from "../ports/mod.ts"; interface PyEnvConfig { - /** Python version */ - version: string; - releaseTag: string; + install?: { + /** Python version */ + version: string; + releaseTag: string; + }; /** venv dir, relative to Ghjk dir; default: ".venv" */ dir?: string; /** create the venv if missing; default: true */ @@ -12,12 +14,15 @@ interface PyEnvConfig { } export function pyEnv( - { version, releaseTag, dir = ".venv", create = true }: PyEnvConfig = {}, + { install, dir = ".venv", create = true }: PyEnvConfig = {}, ) { return (builder: EnvBuilder, ghjk) => { - builder.install( - ports.cpy_bs({ version, releaseTag }), - ); + if (install) { + const { version, releaseTag } = install; + builder.install( + ports.cpy_bs({ version, releaseTag }), + ); + } if (create) { builder.onEnter(ghjk.task({ name: "activate-py-venv", From db405752cbb30bc7410ff2aca153cc36b37c4ff6 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Mon, 19 Aug 2024 10:26:52 +0300 Subject: [PATCH 08/16] fix typing errors --- files/mod.ts | 11 +++++------ modules/envs/types.ts | 12 +++++++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/files/mod.ts b/files/mod.ts index 594542d..a4499ea 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -37,7 +37,7 @@ import type { ExecTaskArgs } from "../modules/tasks/deno.ts"; import { TaskDefHashed, TasksModuleConfig } from "../modules/tasks/types.ts"; // envs import { - DynamicPathVarProvision, + DynamicPosixDirProvision, type EnvRecipe, type EnvsModuleConfig, PosixDirProvision, @@ -921,8 +921,7 @@ type EnvFinalizer = () => { vars: Record; dynVars: Record; posixDirs: Array; - dynamicPosixDirs: Array; - dynBinDirs: string[]; + dynamicPosixDirs: Array; desc?: string; onEnterHookTasks: string[]; onExitHookTasks: string[]; @@ -986,7 +985,7 @@ export class EnvBuilder { #vars: Record = {}; #dynVars: Record = {}; #posixDirs: Array = []; - #dynamicPosixDirs: Array = []; + #dynamicPosixDirs: Array = []; #desc?: string; #onEnterHookTasks: string[] = []; #onExitHookTasks: string[] = []; @@ -1095,7 +1094,7 @@ export class EnvBuilder { case "string": { const prov = { ty: type, path: val }; this.#posixDirs.push(unwrapZodRes( - envsValidators.pathVarProvision.safeParse(prov), + envsValidators.posixDirProvision.safeParse(prov), prov, )); break; @@ -1108,7 +1107,7 @@ export class EnvBuilder { }); const prov = { ty: type + ".dynamic", taskKey }; this.#dynamicPosixDirs.push(unwrapZodRes( - envsValidators.dynamicPathVarProvision.safeParse(prov), + envsValidators.dynamicPosixDirProvision.safeParse(prov), prov, )); break; diff --git a/modules/envs/types.ts b/modules/envs/types.ts index 263cb61..22cfdbd 100644 --- a/modules/envs/types.ts +++ b/modules/envs/types.ts @@ -27,7 +27,7 @@ const posixDirProvision = zod.object({ export type PosixDirProvision = zod.infer; -const dynamicPathVarProvisionTypes = [ +const dynamicPosixDirProvisionTypes = [ "posix.execDir.dynamic", "posix.sharedLibDir.dynamic", "posix.headerDir.dynamic", @@ -114,18 +114,20 @@ const envVarDynProvision = zod.object({ taskKey: zod.string(), }); -const dynamicPathVarProvision = zod.object({ - ty: zod.enum(dynamicPathVarProvisionTypes), +const dynamicPosixDirProvision = zod.object({ + ty: zod.enum(dynamicPosixDirProvisionTypes), taskKey: zod.string(), }); -export type DynamicPathVarProvision = zod.infer; +export type DynamicPosixDirProvision = zod.infer< + typeof dynamicPosixDirProvision +>; const validators = { provision, wellKnownProvision, envVarDynProvision, posixDirProvision, - dynamicPathVarProvision, + dynamicPosixDirProvision, envRecipe, envsModuleConfig, wellKnownEnvRecipe, From 22cb91ca8c6ed92bffab666aa41925f09cdb2424 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Wed, 28 Aug 2024 10:26:03 +0300 Subject: [PATCH 09/16] fix variable inheritance --- examples/kitchen/ghjk.ts | 5 +- files/MergedEnvs.ts | 123 +++++++++++++++++++++++++++++++++++++++ files/mod.ts | 111 +++++++++++------------------------ std/py.ts | 2 +- 4 files changed, 162 insertions(+), 79 deletions(-) create mode 100644 files/MergedEnvs.ts diff --git a/examples/kitchen/ghjk.ts b/examples/kitchen/ghjk.ts index f4b4832..138e12f 100644 --- a/examples/kitchen/ghjk.ts +++ b/examples/kitchen/ghjk.ts @@ -107,7 +107,8 @@ env("python") ports.cpy_bs({ version: "3.8.18", releaseTag: "20240224" }), ports.tar(), ports.zstd(), - ); + ) + .mixin(pyEnv()); env("dev") // we can inherit from many envs @@ -123,4 +124,4 @@ env("dev") })); env("venv") - .mixin(pyEnv({ install: { version: "3.8.18", releaseTag: "20240224" } })); + .inherit(["python"]); diff --git a/files/MergedEnvs.ts b/files/MergedEnvs.ts new file mode 100644 index 0000000..6b622ce --- /dev/null +++ b/files/MergedEnvs.ts @@ -0,0 +1,123 @@ +import { deep_eql } from "../deps/common.ts"; +import getLogger from "../utils/logger.ts"; + +const logger = getLogger(import.meta); + +type Var = + | { kind: "static"; value: string; parentName: string } + | { kind: "dynamic"; taskId: string; parentName: string }; + +export class ParentEnvs { + #childName: string; + #vars: Map = new Map(); + #installs: Set = new Set(); + #onEnterHooks: string[] = []; + #onExitHooks: string[] = []; + #allowedBuildDeps: Map = new Map(); + + constructor(childName: string) { + this.#childName = childName; + } + + addHooks(onEnterHooks: string[], onExitHooks: string[]) { + this.#onEnterHooks.push(...onEnterHooks); + this.#onExitHooks.push(...onExitHooks); + } + + mergeVars(parentName: string, vars: Record) { + for (const [key, value] of Object.entries(vars)) { + const conflict = this.#vars.get(key); + + if ( + conflict && !(conflict.kind === "static" && conflict.value === value) + ) { + logger.warn( + "environment variable conflict on multiple env inheritance, parent 2 was chosen", + { + child: this.#childName, + parent1: conflict.parentName, + parent2: parentName, + variable: key, + }, + ); + } + + this.#vars.set(key, { kind: "static", value, parentName }); + } + } + + mergeDynVars(parentName: string, dynVars: Record) { + for (const [key, taskId] of Object.entries(dynVars)) { + const conflict = this.#vars.get(key); + + if ( + conflict && !(conflict.kind === "dynamic" && conflict.taskId === taskId) + ) { + logger.warn( + "dynamic environment variable conflict on multiple env inheritance, parent 2 was chosen", + { + child: this.#childName, + parent1: conflict.parentName, + parent2: parentName, + variable: key, + }, + ); + } + + this.#vars.set(key, { kind: "dynamic", taskId, parentName }); + } + } + + mergeInstalls( + parentName: string, + installs: Set, + allowedBuildDeps: Record, + ) { + this.#installs = this.#installs.union(installs); + + for (const [key, val] of Object.entries(allowedBuildDeps)) { + const conflict = this.#allowedBuildDeps.get(key); + if (conflict && !deep_eql(val, conflict[0])) { + logger.warn( + "allowedBuildDeps conflict on multiple env inheritance, parent 2 was chosen", + { + child: this.#childName, + parent1: conflict[1], + parent2: parentName, + variable: key, + }, + ); + } + + this.#allowedBuildDeps.set(key, [val, parentName]); + } + } + + finalize() { + const vars: Record = {}; + const dynVars: Record = {}; + + for (const [key, value] of this.#vars) { + if (value.kind === "static") { + vars[key] = value.value; + } else { + dynVars[key] = value.taskId; + } + } + + return { + installSet: { + installs: this.#installs, + allowedBuildDeps: Object.fromEntries( + [...this.#allowedBuildDeps.entries()].map(( + [key, [val]], + ) => [key, val]), + ), + }, + onEnterHookTasks: this.#onEnterHooks, + onExitHookTasks: this.#onExitHooks, + vars, + dynVars, + }; + } +} diff --git a/files/mod.ts b/files/mod.ts index a4499ea..d2f5887 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -47,6 +47,7 @@ import { } from "../modules/envs/types.ts"; import envsValidators from "../modules/envs/types.ts"; import modulesValidators from "../modules/types.ts"; +import { ParentEnvs } from "./MergedEnvs.ts"; const validators = { envVars: zod.record( @@ -139,6 +140,14 @@ export type DenoTaskDefArgs = TaskDefArgs & { type TaskDefTyped = DenoTaskDefArgs & { ty: "denoFile@v1" }; +export type FinalizedEnvs = { + finalized: ReturnType; + installSetId?: string; + vars: Record; + dynVars: Record; + envHash: string; +}; + export class Ghjkfile { #installSets = new Map< string, @@ -149,15 +158,7 @@ export class Ghjkfile { #tasks = new Map(); #bb = new Map(); #seenEnvs: Record = {}; - #finalizedEnvs: Record< - string, - { - finalized: ReturnType; - installSetId?: string; - vars: Record; - envHash: string; - } - > = {}; + #finalizedEnvs: Record = {}; /* dump() { return { @@ -390,6 +391,7 @@ export class Ghjkfile { }; return config; } catch (cause) { + logger.error(`error constructing config for serialization`, { cause }); throw new Error(`error constructing config for serialization`, { cause }); } } @@ -414,75 +416,27 @@ export class Ghjkfile { } #mergeEnvs(keys: string[], childName: string) { - const mergedVars = {} as Record; - let mergedInstalls = new Set(); - const mergedOnEnterHooks = []; - const mergedOnExitHooks = []; - const mergedAllowedBuildDeps = {} as Record< - string, - [string, string] | undefined - >; + const parentEnvs = new ParentEnvs(childName); for (const parentName of keys) { - const { vars, installSetId, finalized } = this.#finalizedEnvs[parentName]; - mergedOnEnterHooks.push(...finalized.onEnterHookTasks); - mergedOnExitHooks.push(...finalized.onExitHookTasks); - for (const [key, val] of Object.entries(vars)) { - const conflict = mergedVars[key]; - // if parents share a parent themselves, they will have - // the same item so it's not exactly a conflict - if (conflict && val !== conflict[0]) { - logger.warn( - "environment variable conflict on multiple env inheritance, parent2 was chosen", - { - child: childName, - parent1: conflict[1], - parent2: parentName, - variable: key, - }, - ); - } - mergedVars[key] = [val, parentName]; - } - if (!installSetId) { - continue; - } - const set = this.#installSets.get(installSetId)!; - mergedInstalls = mergedInstalls.union(set.installs); - for ( - const [key, val] of Object.entries(set.allowedBuildDeps) - ) { - const conflict = mergedAllowedBuildDeps[key]; - if (conflict && !deep_eql(val, conflict[0])) { - logger.warn( - "allowedBuildDeps conflict on multiple env inheritance, parent2 was chosen", - { - child: childName, - parent1: conflict[1], - parent2: parentName, - depPort: key, - }, - ); - } - mergedAllowedBuildDeps[key] = [val, parentName]; + const { installSetId, vars, dynVars, finalized } = + this.#finalizedEnvs[parentName]; + parentEnvs.addHooks( + finalized.onEnterHookTasks, + finalized.onExitHookTasks, + ); + parentEnvs.mergeVars(parentName, vars); + parentEnvs.mergeDynVars(parentName, dynVars); + if (installSetId) { + const set = this.#installSets.get(installSetId)!; + parentEnvs.mergeInstalls( + parentName, + set.installs, + set.allowedBuildDeps, + ); } } - const outInstallSet = { - installs: mergedInstalls, - allowedBuildDeps: Object.fromEntries( - Object.entries(mergedAllowedBuildDeps).map(( - [key, val], - ) => [key, val![0]]), - ), - }; - const outVars = Object.fromEntries( - Object.entries(mergedVars).map(([key, val]) => [key, val![0]]), - ); - return { - installSet: outInstallSet, - onEnterHookTasks: mergedOnEnterHooks, - onExitHookTasks: mergedOnExitHooks, - vars: outVars, - }; + + return parentEnvs.finalize(); } #resolveEnvBases( @@ -592,6 +546,10 @@ export class Ghjkfile { ...base.vars, ...final.vars, }; + const finalDynVars = { + ...base.dynVars, + ...final.dynVars, + }; let finalInstallSetId: string | undefined; { @@ -675,7 +633,7 @@ export class Ghjkfile { const prov: WellKnownProvision = { ty: "posix.envVar", key, val }; return prov; }), - ...Object.entries(final.dynVars).map(( + ...Object.entries(finalDynVars).map(( [key, val], ) => { const prov = { ty: "posix.envVarDyn", key, taskKey: val }; @@ -704,6 +662,7 @@ export class Ghjkfile { this.#finalizedEnvs[final.key] = { installSetId: finalInstallSetId, vars: finalVars, + dynVars: finalDynVars, finalized: final, envHash, }; diff --git a/std/py.ts b/std/py.ts index ba5b1c2..529c7d8 100644 --- a/std/py.ts +++ b/std/py.ts @@ -25,7 +25,7 @@ export function pyEnv( } if (create) { builder.onEnter(ghjk.task({ - name: "activate-py-venv", + name: "create-py-venv", fn: async ($, { workingDir }) => { const venvDir = $.path(workingDir).join(dir); if (!(await venvDir.exists())) { From 801dc825033672ca4665eeda8b73ac47b13e6301 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Wed, 28 Aug 2024 10:40:56 +0300 Subject: [PATCH 10/16] fix posix dir inheritance --- files/MergedEnvs.ts | 16 ++++++++++++++++ files/mod.ts | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/files/MergedEnvs.ts b/files/MergedEnvs.ts index 6b622ce..c158503 100644 --- a/files/MergedEnvs.ts +++ b/files/MergedEnvs.ts @@ -1,4 +1,8 @@ import { deep_eql } from "../deps/common.ts"; +import { + DynamicPosixDirProvision, + PosixDirProvision, +} from "../modules/envs/types.ts"; import getLogger from "../utils/logger.ts"; const logger = getLogger(import.meta); @@ -10,6 +14,8 @@ type Var = export class ParentEnvs { #childName: string; #vars: Map = new Map(); + #posixDirs: Array = []; + #dynamicPosixDirs: Array = []; #installs: Set = new Set(); #onEnterHooks: string[] = []; #onExitHooks: string[] = []; @@ -68,6 +74,14 @@ export class ParentEnvs { } } + mergePosixDirs( + posixDirs: Array, + dynamicPosixDirs: Array, + ) { + this.#posixDirs.push(...posixDirs); + this.#dynamicPosixDirs.push(...dynamicPosixDirs); + } + mergeInstalls( parentName: string, installs: Set, @@ -118,6 +132,8 @@ export class ParentEnvs { onExitHookTasks: this.#onExitHooks, vars, dynVars, + posixDirs: this.#posixDirs, + dynamicPosixDirs: this.#dynamicPosixDirs, }; } } diff --git a/files/mod.ts b/files/mod.ts index d2f5887..5e26360 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -426,6 +426,10 @@ export class Ghjkfile { ); parentEnvs.mergeVars(parentName, vars); parentEnvs.mergeDynVars(parentName, dynVars); + parentEnvs.mergePosixDirs( + finalized.posixDirs, + finalized.dynamicPosixDirs, + ); if (installSetId) { const set = this.#installSets.get(installSetId)!; parentEnvs.mergeInstalls( @@ -642,7 +646,9 @@ export class Ghjkfile { prov, ); }), + ...base.posixDirs, ...final.posixDirs, + ...base.dynamicPosixDirs, ...final.dynamicPosixDirs, // env hooks ...hooks, From cce840c16f9405755d7440a417aaa9b0d6ee3bda Mon Sep 17 00:00:00 2001 From: Natoandro Date: Wed, 28 Aug 2024 10:43:56 +0300 Subject: [PATCH 11/16] remove unused import --- files/mod.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/mod.ts b/files/mod.ts index 5e26360..e881dcf 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -4,7 +4,7 @@ // here to make the resulting config reasonably stable // across repeated serializaitons. No random identifiers. -import { deep_eql, multibase32, multibase64, zod } from "../deps/common.ts"; +import { multibase32, multibase64, zod } from "../deps/common.ts"; // ports specific imports import portsValidators from "../modules/ports/types.ts"; From 643adcd1203e1d8ac368a6c9f20463703f8d7707 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Wed, 28 Aug 2024 10:52:08 +0300 Subject: [PATCH 12/16] change python 3.12 version --- .ghjk/lock.json | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.ghjk/lock.json b/.ghjk/lock.json index 30c21bb..42f2d6d 100644 --- a/.ghjk/lock.json +++ b/.ghjk/lock.json @@ -17,25 +17,6 @@ "portRef": "act_ghrel@0.1.0", "specifiedVersion": false }, - "bciqij3g6mmbjn4a6ps4eipcy2fmw2zumgv5a3gbxycthroffihwquoi": { - "version": "3.12.4", - "buildDepConfigs": { - "tar_aa": { - "version": "1.34", - "buildDepConfigs": {}, - "portRef": "tar_aa@0.1.0", - "specifiedVersion": false - }, - "zstd_aa": { - "version": "v1.4.8,", - "buildDepConfigs": {}, - "portRef": "zstd_aa@0.1.0", - "specifiedVersion": false - } - }, - "portRef": "cpy_bs_ghrel@0.1.0", - "specifiedVersion": false - }, "bciqj4p5hoqweghbuvz52rupja7sqze34z63dd62nz632c5zxikv6ezy": { "version": "1.34", "buildDepConfigs": {}, @@ -58,16 +39,16 @@ "version": "3.8.0", "buildDepConfigs": { "cpy_bs_ghrel": { - "version": "3.12.4", + "version": "3.12.5", "buildDepConfigs": { "tar_aa": { - "version": "1.35", + "version": "1.34", "buildDepConfigs": {}, "portRef": "tar_aa@0.1.0", "specifiedVersion": false }, "zstd_aa": { - "version": "v1.5.6,", + "version": "v1.4.8,", "buildDepConfigs": {}, "portRef": "zstd_aa@0.1.0", "specifiedVersion": false @@ -80,6 +61,25 @@ "portRef": "pipi_pypi@0.1.0", "packageName": "pre-commit", "specifiedVersion": false + }, + "bciqij3g6mmbjn4a6ps4eipcy2fmw2zumgv5a3gbxycthroffihwquoi": { + "version": "3.12.5", + "buildDepConfigs": { + "tar_aa": { + "version": "1.34", + "buildDepConfigs": {}, + "portRef": "tar_aa@0.1.0", + "specifiedVersion": false + }, + "zstd_aa": { + "version": "v1.4.8,", + "buildDepConfigs": {}, + "portRef": "zstd_aa@0.1.0", + "specifiedVersion": false + } + }, + "portRef": "cpy_bs_ghrel@0.1.0", + "specifiedVersion": false } } }, From 7b787a260060acbb83ba6d6a62538fe1794375cf Mon Sep 17 00:00:00 2001 From: Natoandro Date: Fri, 30 Aug 2024 00:50:46 +0300 Subject: [PATCH 13/16] debug --- files/MergedEnvs.ts | 1 + files/mod.ts | 5 +++-- modules/envs/posix.ts | 1 + modules/envs/reducer.ts | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/files/MergedEnvs.ts b/files/MergedEnvs.ts index c158503..948a4d9 100644 --- a/files/MergedEnvs.ts +++ b/files/MergedEnvs.ts @@ -78,6 +78,7 @@ export class ParentEnvs { posixDirs: Array, dynamicPosixDirs: Array, ) { + logger.debug("merge posix dirs", { posixDirs, dynamicPosixDirs }); this.#posixDirs.push(...posixDirs); this.#dynamicPosixDirs.push(...dynamicPosixDirs); } diff --git a/files/mod.ts b/files/mod.ts index e881dcf..4b1de78 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -544,6 +544,7 @@ export class Ghjkfile { const final = all[item]; const base = this.#mergeEnvs(final.envBaseResolved ?? [], final.key); + logger.debug("base", final.key, base); // console.log({ parents: final.envBaseResolved, child: final.key, base }); const finalVars = { @@ -646,10 +647,10 @@ export class Ghjkfile { prov, ); }), - ...base.posixDirs, ...final.posixDirs, - ...base.dynamicPosixDirs, + ...base.posixDirs, ...final.dynamicPosixDirs, + ...base.dynamicPosixDirs, // env hooks ...hooks, ], diff --git a/modules/envs/posix.ts b/modules/envs/posix.ts index 21f2d26..6dce0b0 100644 --- a/modules/envs/posix.ts +++ b/modules/envs/posix.ts @@ -22,6 +22,7 @@ export async function cookPosixEnv( }, ) { logger.debug("cooking env", envKey, { envDir }); + logger.debug("recipe", recipe); const reducedRecipe = await reduceStrangeProvisions(gcx, recipe); await $.removeIfExists(envDir); // create the shims for the user's environment diff --git a/modules/envs/reducer.ts b/modules/envs/reducer.ts index c27172d..26c5d9e 100644 --- a/modules/envs/reducer.ts +++ b/modules/envs/reducer.ts @@ -1,4 +1,5 @@ import { unwrapZodRes } from "../../port.ts"; +import logger from "../../utils/logger.ts"; import { execTask } from "../tasks/exec.ts"; import { getTasksCtx } from "../tasks/inter.ts"; import type { GhjkCtx } from "../types.ts"; @@ -66,6 +67,7 @@ export async function reduceStrangeProvisions( const reducerStore = getProvisionReducerStore(gcx); // Replace by `Object.groupBy` once the types for it are fixed const bins = {} as Record; + logger(import.meta).debug("provides", env.provides); for (const item of env.provides) { let bin = bins[item.ty]; if (!bin) { From c1ac9e559a79a097e57ad02f59742a15abe72935 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Fri, 30 Aug 2024 22:04:05 +0300 Subject: [PATCH 14/16] cookbook --- files/cookbook.ts | 176 +++++++++++ files/{MergedEnvs.ts => merged_envs.ts} | 51 ++- files/mod.ts | 399 ++++++++---------------- modules/envs/posix.ts | 190 ++++++----- modules/envs/reducer.ts | 23 +- 5 files changed, 442 insertions(+), 397 deletions(-) create mode 100644 files/cookbook.ts rename files/{MergedEnvs.ts => merged_envs.ts} (72%) diff --git a/files/cookbook.ts b/files/cookbook.ts new file mode 100644 index 0000000..926bf77 --- /dev/null +++ b/files/cookbook.ts @@ -0,0 +1,176 @@ +import { + EnvFinalizer, + FinalizedEnv, + InlineTaskHookProvision, + objectHashSafe, + TaskDefTyped, +} from "./mod.ts"; +import { + EnvRecipe, + EnvsModuleConfig, + WellKnownProvision, +} from "../modules/envs/types.ts"; +import { InstallSetRefProvision, unwrapZodRes } from "../port.ts"; +import { InstallSet, MergedEnvs } from "./merged_envs.ts"; +import envsValidators from "../modules/envs/types.ts"; + +export type Final = ReturnType & { + envBaseResolved: null | string[]; +}; + +interface MergedEntries { + vars: Record; + dynVars: Record; +} + +export class Cookbook { + #moduleConfig: EnvsModuleConfig; + + constructor( + public installSets: Map, + public finalizedEnvs: Record, + public tasks: Map, + defaultEnv: string, + ) { + this.#moduleConfig = { + envs: {}, + defaultEnv, + envsNamed: {}, + }; + } + + public registerEnv(final: Final, merged: MergedEnvs) { + const recipe = new RecipeBuilder(this, merged).build(); + + const installSetId = this.#getInstallSetId(final, merged.installSet); + if (installSetId) { + const prov: InstallSetRefProvision = { + ty: "ghjk.ports.InstallSetRef", + setId: installSetId, + }; + recipe.provides.push(prov); + } + + const hash = objectHashSafe(recipe); + this.finalizedEnvs[final.key] = { + installSetId, + finalized: final, + vars: merged.vars, + dynVars: merged.dynVars, + envHash: hash, + }; + + this.#moduleConfig.envs[hash] = recipe; + if (final.name) { + this.#moduleConfig.envsNamed[final.name] = hash; + } + } + + get moduleConfig() { + return this.#moduleConfig; + } + + #getInstallSetId(final: Final, baseSet: InstallSet): string | undefined { + const installSet = this.installSets.get(final.installSetId); + if (installSet) { + installSet.installs = installSet.installs.union( + baseSet.installs, + ); + for ( + const [key, val] of Object.entries( + baseSet.allowedBuildDeps, + ) + ) { + // prefer the port dep config of the child over any + // similar deps in the base + if (!installSet.allowedBuildDeps[key]) { + installSet.allowedBuildDeps[key] = val; + } + } + return final.installSetId; + } // if there's no install set found under the id + else { + // implies that the env has not ports explicitly configured + if (final.envBaseResolved) { + // has a singluar parent + if (final.envBaseResolved.length == 1) { + return this.finalizedEnvs[final.envBaseResolved[0]] + .installSetId; + } else { + this.installSets.set( + final.installSetId, + baseSet, + ); + return final.installSetId; + } + } + } + } +} + +class RecipeBuilder { + constructor( + private book: Cookbook, + private compactEnv: MergedEnvs, + ) {} + + build(): EnvRecipe { + return { + desc: this.compactEnv.desc, + provides: [ + ...Object.entries(this.compactEnv.vars).map(([key, val]) => { + const prov: WellKnownProvision = { ty: "posix.envVar", key, val }; + return prov; + }), + ...Object.entries(this.compactEnv.dynVars).map(([key, val]) => { + const prov = { ty: "posix.envVarDyn", key, taskKey: val }; + return unwrapZodRes( + envsValidators.envVarDynProvision.safeParse(prov), + prov, + ); + }), + ...this.compactEnv.posixDirs, + ...this.compactEnv.dynamicPosixDirs, + // env hooks + ...this.#getHooks(), + ], + }; + } + + #getHooks(): InlineTaskHookProvision[] { + return [ + ...this.compactEnv.onEnterHookTasks.map( + (key) => [key, "hook.onEnter.ghjkTask"] as const, + ), + ...this.compactEnv.onExitHookTasks.map( + (key) => [key, "hook.onExit.ghjkTask"] as const, + ), + ].map(([taskKey, ty]) => { + const task = this.book.tasks.get(taskKey); + if (!task) { + throw new Error("unable to find task for onEnterHook", { + cause: { + env: this.compactEnv.name, + taskKey, + }, + }); + } + if (task.ty == "denoFile@v1") { + const prov: InlineTaskHookProvision = { + ty, + taskKey, + }; + return prov; + } + throw new Error( + `unsupported task type "${task.ty}" used for environment hook`, + { + cause: { + taskKey, + task, + }, + }, + ); + }); + } +} diff --git a/files/MergedEnvs.ts b/files/merged_envs.ts similarity index 72% rename from files/MergedEnvs.ts rename to files/merged_envs.ts index 948a4d9..b64e921 100644 --- a/files/MergedEnvs.ts +++ b/files/merged_envs.ts @@ -4,6 +4,7 @@ import { PosixDirProvision, } from "../modules/envs/types.ts"; import getLogger from "../utils/logger.ts"; +import { Final } from "./cookbook.ts"; const logger = getLogger(import.meta); @@ -11,6 +12,23 @@ type Var = | { kind: "static"; value: string; parentName: string } | { kind: "dynamic"; taskId: string; parentName: string }; +export interface InstallSet { + installs: Set; + allowedBuildDeps: Record; +} + +export interface MergedEnvs { + desc: string | undefined; + name: string | undefined; + installSet: InstallSet; + onEnterHookTasks: string[]; + onExitHookTasks: string[]; + vars: Record; + dynVars: Record; + posixDirs: PosixDirProvision[]; + dynamicPosixDirs: DynamicPosixDirProvision[]; +} + export class ParentEnvs { #childName: string; #vars: Map = new Map(); @@ -35,7 +53,8 @@ export class ParentEnvs { const conflict = this.#vars.get(key); if ( - conflict && !(conflict.kind === "static" && conflict.value === value) + conflict && + !(conflict.kind === "static" && conflict.value === value) ) { logger.warn( "environment variable conflict on multiple env inheritance, parent 2 was chosen", @@ -57,7 +76,8 @@ export class ParentEnvs { const conflict = this.#vars.get(key); if ( - conflict && !(conflict.kind === "dynamic" && conflict.taskId === taskId) + conflict && + !(conflict.kind === "dynamic" && conflict.taskId === taskId) ) { logger.warn( "dynamic environment variable conflict on multiple env inheritance, parent 2 was chosen", @@ -108,7 +128,7 @@ export class ParentEnvs { } } - finalize() { + withChild(child: Final): MergedEnvs { const vars: Record = {}; const dynVars: Record = {}; @@ -121,20 +141,27 @@ export class ParentEnvs { } return { + desc: child.desc, + name: child.name, + // installSets are not merged here... installSet: { installs: this.#installs, allowedBuildDeps: Object.fromEntries( - [...this.#allowedBuildDeps.entries()].map(( - [key, [val]], - ) => [key, val]), + [...this.#allowedBuildDeps.entries()].map(([key, [val]]) => [ + key, + val, + ]), ), }, - onEnterHookTasks: this.#onEnterHooks, - onExitHookTasks: this.#onExitHooks, - vars, - dynVars, - posixDirs: this.#posixDirs, - dynamicPosixDirs: this.#dynamicPosixDirs, + onEnterHookTasks: [...this.#onEnterHooks, ...child.onEnterHookTasks], + onExitHookTasks: [...this.#onExitHooks, ...child.onExitHookTasks], + vars: { ...vars, ...child.vars }, + dynVars: { ...dynVars, ...child.dynVars }, + posixDirs: [...child.posixDirs, ...this.#posixDirs], + dynamicPosixDirs: [ + ...child.dynamicPosixDirs, + ...this.#dynamicPosixDirs, + ], }; } } diff --git a/files/mod.ts b/files/mod.ts index 4b1de78..d58803c 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -47,7 +47,8 @@ import { } from "../modules/envs/types.ts"; import envsValidators from "../modules/envs/types.ts"; import modulesValidators from "../modules/types.ts"; -import { ParentEnvs } from "./MergedEnvs.ts"; +import { InstallSet, ParentEnvs } from "./merged_envs.ts"; +import { Cookbook, Final } from "./cookbook.ts"; const validators = { envVars: zod.record( @@ -138,9 +139,9 @@ export type DenoTaskDefArgs = TaskDefArgs & { nonce?: string; }; -type TaskDefTyped = DenoTaskDefArgs & { ty: "denoFile@v1" }; +export type TaskDefTyped = DenoTaskDefArgs & { ty: "denoFile@v1" }; -export type FinalizedEnvs = { +export type FinalizedEnv = { finalized: ReturnType; installSetId?: string; vars: Record; @@ -149,16 +150,13 @@ export type FinalizedEnvs = { }; export class Ghjkfile { - #installSets = new Map< - string, - { installs: Set; allowedBuildDeps: Record } - >(); + #installSets = new Map(); #seenInstallConfs = new Map(); #seenAllowedDepPorts = new Map(); #tasks = new Map(); #bb = new Map(); #seenEnvs: Record = {}; - #finalizedEnvs: Record = {}; + #finalizedEnvs: Record = {}; /* dump() { return { @@ -203,9 +201,7 @@ export class Ghjkfile { ) { const set = this.#getSet(setId); set.allowedBuildDeps = Object.fromEntries( - reduceAllowedDeps(deps).map(( - dep, - ) => { + reduceAllowedDeps(deps).map((dep) => { const hash = objectHashSafe(dep); this.#seenAllowedDepPorts.set(hash, dep); return [dep.manifest.name, hash]; @@ -241,9 +237,7 @@ export class Ghjkfile { } : {}), }); - key = multibase64.base64urlpad.encode( - multibase32.base32.decode(key), - ); + key = multibase64.base64urlpad.encode(multibase32.base32.decode(key)); break; } default: @@ -311,9 +305,7 @@ export class Ghjkfile { return env; } - async execTask( - { key, workingDir, envVars, argv }: ExecTaskArgs, - ) { + async execTask({ key, workingDir, envVars, argv }: ExecTaskArgs) { const task = this.#tasks.get(key); if (!task) { throw new Error(`no task defined under "${key}"`); @@ -337,13 +329,14 @@ export class Ghjkfile { } } - toConfig( - { defaultEnv, defaultBaseEnv }: { - defaultEnv: string; - defaultBaseEnv: string; - ghjkfileUrl: string; - }, - ) { + toConfig({ + defaultEnv, + defaultBaseEnv, + }: { + defaultEnv: string; + defaultBaseEnv: string; + ghjkfileUrl: string; + }) { // make sure referenced envs exist this.addEnv(defaultEnv, { name: defaultEnv }); this.addEnv(defaultBaseEnv, { name: defaultBaseEnv }); @@ -351,8 +344,10 @@ export class Ghjkfile { // crearte the envs used by the tasks const taskToEnvMap = {} as Record; for ( - const [key, { inherit, vars, installs, allowedBuildDeps }] of this.#tasks - .entries() + const [ + key, + { inherit, vars, installs, allowedBuildDeps }, + ] of this.#tasks.entries() ) { const envKey = `____task_env_${key}`; this.addEnv(envKey, { @@ -370,24 +365,25 @@ export class Ghjkfile { defaultBaseEnv, taskToEnvMap, ); - const tasksConfig = this.#processTasks( - envsConfig, - taskToEnvMap, - ); + const tasksConfig = this.#processTasks(envsConfig, taskToEnvMap); const portsConfig = this.#processInstalls(); const config: SerializedConfig = { blackboard: Object.fromEntries(this.#bb.entries()), - modules: [{ - id: std_modules.ports, - config: portsConfig, - }, { - id: std_modules.tasks, - config: tasksConfig, - }, { - id: std_modules.envs, - config: envsConfig, - }], + modules: [ + { + id: std_modules.ports, + config: portsConfig, + }, + { + id: std_modules.tasks, + config: tasksConfig, + }, + { + id: std_modules.envs, + config: envsConfig, + }, + ], }; return config; } catch (cause) { @@ -415,9 +411,13 @@ export class Ghjkfile { return hash; } - #mergeEnvs(keys: string[], childName: string) { + #mergeEnvs(final: Final) { + const parents = final.envBaseResolved ?? []; + const childName = final.key; + logger.debug({ childName, parents }); const parentEnvs = new ParentEnvs(childName); - for (const parentName of keys) { + for (const parentName of parents) { + logger.debug("finalized envs", this.#finalizedEnvs[parentName]); const { installSetId, vars, dynVars, finalized } = this.#finalizedEnvs[parentName]; parentEnvs.addHooks( @@ -440,7 +440,7 @@ export class Ghjkfile { } } - return parentEnvs.finalize(); + return parentEnvs.withChild(final); } #resolveEnvBases( @@ -503,9 +503,12 @@ export class Ghjkfile { const deps = new Map(); const revDeps = new Map(); for ( - const [_key, [_builder, finalizer]] of Object.entries(this.#seenEnvs) + const [_key, [_builder, finalizer]] of Object.entries( + this.#seenEnvs, + ) ) { const final = finalizer(); + logger.debug("seen env", _key, final.key); const envBaseResolved = this.#resolveEnvBases( final.inherit, @@ -529,12 +532,13 @@ export class Ghjkfile { } } - const moduleConfig: EnvsModuleConfig = { - envs: {}, - defaultEnv, - envsNamed: {}, - }; const workingSet = indie; + const cookbook = new Cookbook( + this.#installSets, + this.#finalizedEnvs, + this.#tasks, + defaultEnv, + ); // console.log({ // indie, // deps, @@ -543,142 +547,10 @@ export class Ghjkfile { const item = workingSet.pop()!; const final = all[item]; - const base = this.#mergeEnvs(final.envBaseResolved ?? [], final.key); - logger.debug("base", final.key, base); - // console.log({ parents: final.envBaseResolved, child: final.key, base }); + logger.debug("base", final.envBaseResolved); + const mergedEnvs = this.#mergeEnvs(final); - const finalVars = { - ...base.vars, - ...final.vars, - }; - const finalDynVars = { - ...base.dynVars, - ...final.dynVars, - }; - - let finalInstallSetId: string | undefined; - { - const installSet = this.#installSets.get(final.installSetId); - if (installSet) { - installSet.installs = installSet.installs - .union(base.installSet.installs); - for ( - const [key, val] of Object.entries(base.installSet.allowedBuildDeps) - ) { - // prefer the port dep config of the child over any - // similar deps in the base - if (!installSet.allowedBuildDeps[key]) { - installSet.allowedBuildDeps[key] = val; - } - } - finalInstallSetId = final.installSetId; - } // if there's no install set found under the id - else { - // implies that the env has not ports explicitly configured - if (final.envBaseResolved) { - // has a singluar parent - if (final.envBaseResolved.length == 1) { - finalInstallSetId = - this.#finalizedEnvs[final.envBaseResolved[0]].installSetId; - } else { - this.#installSets.set(final.installSetId, base.installSet); - finalInstallSetId = final.installSetId; - } - } - } - } - const hooks = [ - ...base.onEnterHookTasks.map( - (key) => [key, "hook.onEnter.ghjkTask"] as const, - ), - ...final.onEnterHookTasks.map( - (key) => [key, "hook.onEnter.ghjkTask"] as const, - ), - ...base.onExitHookTasks.map( - (key) => [key, "hook.onExit.ghjkTask"] as const, - ), - ...final.onExitHookTasks.map( - (key) => [key, "hook.onExit.ghjkTask"] as const, - ), - ].map(([taskKey, ty]) => { - const task = this.#tasks.get(taskKey); - if (!task) { - throw new Error("unable to find task for onEnterHook", { - cause: { - env: final.name, - taskKey, - }, - }); - } - if (task.ty == "denoFile@v1") { - const prov: InlineTaskHookProvision = { - ty, - taskKey, - }; - return prov; - } - throw new Error( - `unsupported task type "${task.ty}" used for environment hook`, - { - cause: { - taskKey, - task, - }, - }, - ); - }); - - // the actual final final recipe - const recipe: EnvRecipe = { - desc: final.desc, - provides: [ - ...Object.entries(finalVars).map(( - [key, val], - ) => { - const prov: WellKnownProvision = { ty: "posix.envVar", key, val }; - return prov; - }), - ...Object.entries(finalDynVars).map(( - [key, val], - ) => { - const prov = { ty: "posix.envVarDyn", key, taskKey: val }; - return unwrapZodRes( - envsValidators.envVarDynProvision.safeParse(prov), - prov, - ); - }), - ...final.posixDirs, - ...base.posixDirs, - ...final.dynamicPosixDirs, - ...base.dynamicPosixDirs, - // env hooks - ...hooks, - ], - }; - - if (finalInstallSetId) { - const prov: InstallSetRefProvision = { - ty: "ghjk.ports.InstallSetRef", - setId: finalInstallSetId, - }; - recipe.provides.push(prov); - } - - // hashing takes care of deduplication - const envHash = objectHashSafe(recipe); - this.#finalizedEnvs[final.key] = { - installSetId: finalInstallSetId, - vars: finalVars, - dynVars: finalDynVars, - finalized: final, - envHash, - }; - // hashing takes care of deduplication - moduleConfig.envs[envHash] = recipe; - - if (final.name) { - moduleConfig.envsNamed[final.name] = envHash; - } + cookbook.registerEnv(final, mergedEnvs); for (const revDepKey of revDeps.get(final.key) ?? []) { const revDepDeps = deps.get(revDepKey)!; @@ -705,7 +577,8 @@ export class Ghjkfile { }, }); } - return moduleConfig; + + return cookbook.moduleConfig; } #processTasks( @@ -722,9 +595,9 @@ export class Ghjkfile { ); for (const [key, args] of this.#tasks) { if (args.dependsOn && args.dependsOn.length > 0) { - const depKeys = - (Array.isArray(args.dependsOn) ? args.dependsOn : [args.dependsOn]) - .map((nameOrKey) => nameToKey[nameOrKey] ?? nameOrKey); + const depKeys = ( + Array.isArray(args.dependsOn) ? args.dependsOn : [args.dependsOn] + ).map((nameOrKey) => nameToKey[nameOrKey] ?? nameOrKey); deps.set(key, depKeys); for (const depKey of depKeys) { const depRevDeps = revDeps.get(depKey); @@ -759,14 +632,15 @@ export class Ghjkfile { ? workingDir.toString() : workingDir, desc, - ...dependsOn + ...(dependsOn ? { dependsOn: (Array.isArray(dependsOn) ? dependsOn : [dependsOn]) - ?.map((keyOrHash) => - localToFinalKey[nameToKey[keyOrHash] ?? keyOrHash] + ?.map( + (keyOrHash) => + localToFinalKey[nameToKey[keyOrHash] ?? keyOrHash], ), } - : {}, + : {}), envKey: envHash, }; const taskHash = objectHash(def); @@ -826,26 +700,24 @@ export class Ghjkfile { // reduce task based env hooks for (const [_name, env] of Object.entries(envsConfig.envs)) { - env.provides = env.provides.map( - (prov) => { - if ( - prov.ty == "hook.onEnter.ghjkTask" || - prov.ty == "hook.onExit.ghjkTask" - ) { - const inlineProv = prov as InlineTaskHookProvision; - const taskKey = localToFinalKey[inlineProv.taskKey]; - const out: WellKnownProvision = { - ty: /onEnter/.test(prov.ty) - ? "hook.onEnter.posixExec" - : "hook.onExit.posixExec", - program: "ghjk", - arguments: ["x", taskKey], - }; - return out; - } - return prov; - }, - ); + env.provides = env.provides.map((prov) => { + if ( + prov.ty == "hook.onEnter.ghjkTask" || + prov.ty == "hook.onExit.ghjkTask" + ) { + const inlineProv = prov as InlineTaskHookProvision; + const taskKey = localToFinalKey[inlineProv.taskKey]; + const out: WellKnownProvision = { + ty: /onEnter/.test(prov.ty) + ? "hook.onEnter.posixExec" + : "hook.onExit.posixExec", + program: "ghjk", + arguments: ["x", taskKey], + }; + return out; + } + return prov; + }); } return moduleConfig; @@ -855,31 +727,26 @@ export class Ghjkfile { const out: PortsModuleConfigHashed = { sets: {}, }; - for ( - const [setId, set] of this.#installSets.entries() - ) { + for (const [setId, set] of this.#installSets.entries()) { out.sets[setId] = { - installs: [...set.installs.values()] - .map((instHash) => - this.#addToBlackboard(this.#seenInstallConfs.get(instHash)) - ), - allowedBuildDeps: this.#addToBlackboard(Object.fromEntries( - Object.entries(set.allowedBuildDeps).map( - ( - [key, depHash], - ) => [ + installs: [...set.installs.values()].map((instHash) => + this.#addToBlackboard(this.#seenInstallConfs.get(instHash)) + ), + allowedBuildDeps: this.#addToBlackboard( + Object.fromEntries( + Object.entries(set.allowedBuildDeps).map(([key, depHash]) => [ key, this.#addToBlackboard(this.#seenAllowedDepPorts.get(depHash)), - ], + ]), ), - )), + ), }; } return out; } } -type EnvFinalizer = () => { +export type EnvFinalizer = () => { key: string; name?: string; installSetId: string; @@ -893,9 +760,7 @@ type EnvFinalizer = () => { onExitHookTasks: string[]; }; -export type EnvDefArgsPartial = - & { name?: string } - & Omit; +export type EnvDefArgsPartial = { name?: string } & Omit; export type DynEnvValue = | ((...params: Parameters) => string | number) @@ -1017,7 +882,8 @@ export class EnvBuilder { * Add multiple environment variable. */ vars(envVars: Record) { - const vars = {}, dynVars = {}; + const vars = {}, + dynVars = {}; for (const [k, v] of Object.entries(envVars)) { switch (typeof v) { case "string": @@ -1044,10 +910,7 @@ export class EnvBuilder { this.#vars, unwrapZodRes(validators.envVars.safeParse(vars), { envVars: vars }), ); - Object.assign( - this.#dynVars, - dynVars, - ); + Object.assign(this.#dynVars, dynVars); return this; } @@ -1059,10 +922,9 @@ export class EnvBuilder { switch (typeof val) { case "string": { const prov = { ty: type, path: val }; - this.#posixDirs.push(unwrapZodRes( - envsValidators.posixDirProvision.safeParse(prov), - prov, - )); + this.#posixDirs.push( + unwrapZodRes(envsValidators.posixDirProvision.safeParse(prov), prov), + ); break; } @@ -1072,10 +934,12 @@ export class EnvBuilder { fn: val, }); const prov = { ty: type + ".dynamic", taskKey }; - this.#dynamicPosixDirs.push(unwrapZodRes( - envsValidators.dynamicPosixDirProvision.safeParse(prov), - prov, - )); + this.#dynamicPosixDirs.push( + unwrapZodRes( + envsValidators.dynamicPosixDirProvision.safeParse(prov), + prov, + ), + ); break; } @@ -1136,16 +1000,9 @@ export class EnvBuilder { } export function stdDeps(args = { enableRuntimes: false }) { - const out: AllowedPortDep[] = [ - ...Object.values(std_ports.map), - ]; + const out: AllowedPortDep[] = [...Object.values(std_ports.map)]; if (args.enableRuntimes) { - out.push( - ...reduceAllowedDeps([ - node.default(), - cpy.default(), - ]), - ); + out.push(...reduceAllowedDeps([node.default(), cpy.default()])); } return out; } @@ -1172,7 +1029,7 @@ function task$( return custom$; } -type InlineTaskHookProvision = Provision & { +export type InlineTaskHookProvision = Provision & { ty: "hook.onExit.ghjkTask" | "hook.onEnter.ghjkTask"; taskKey: string; }; @@ -1180,26 +1037,24 @@ type InlineTaskHookProvision = Provision & { export function reduceAllowedDeps( deps: (AllowedPortDep | InstallConfigFat)[], ): AllowedPortDep[] { - return deps.map( - (dep: any) => { - { - const res = portsValidators.allowedPortDep.safeParse(dep); - if (res.success) return res.data; - } - const inst = unwrapZodRes( - portsValidators.installConfigFat.safeParse(dep), - dep, - "invalid allowed dep object, provide either InstallConfigFat or AllowedPortDep objects", - ); - const out: AllowedPortDep = { - manifest: inst.port, - defaultInst: thinInstallConfig(inst), - }; - return portsValidators.allowedPortDep.parse(out); - }, - ); + return deps.map((dep: any) => { + { + const res = portsValidators.allowedPortDep.safeParse(dep); + if (res.success) return res.data; + } + const inst = unwrapZodRes( + portsValidators.installConfigFat.safeParse(dep), + dep, + "invalid allowed dep object, provide either InstallConfigFat or AllowedPortDep objects", + ); + const out: AllowedPortDep = { + manifest: inst.port, + defaultInst: thinInstallConfig(inst), + }; + return portsValidators.allowedPortDep.parse(out); + }); } -function objectHashSafe(obj: unknown) { +export function objectHashSafe(obj: unknown) { return objectHash(JSON.parse(JSON.stringify(obj))); } diff --git a/modules/envs/posix.ts b/modules/envs/posix.ts index 6dce0b0..d50f45a 100644 --- a/modules/envs/posix.ts +++ b/modules/envs/posix.ts @@ -12,17 +12,21 @@ import getLogger from "../../utils/logger.ts"; const logger = getLogger(import.meta); -export async function cookPosixEnv( - { gcx, recipe, envKey, envDir, createShellLoaders = false }: { - gcx: GhjkCtx; - recipe: EnvRecipeX; - envKey: string; - envDir: string; - createShellLoaders?: boolean; - }, -) { +export async function cookPosixEnv({ + gcx, + recipe, + envKey, + envDir, + createShellLoaders = false, +}: { + gcx: GhjkCtx; + recipe: EnvRecipeX; + envKey: string; + envDir: string; + createShellLoaders?: boolean; +}) { logger.debug("cooking env", envKey, { envDir }); - logger.debug("recipe", recipe); + // logger.debug("recipe", recipe); const reducedRecipe = await reduceStrangeProvisions(gcx, recipe); await $.removeIfExists(envDir); // create the shims for the user's environment @@ -50,76 +54,69 @@ export async function cookPosixEnv( // FIXME: detect shim conflicts // FIXME: better support for multi installs - await Promise.all(reducedRecipe.provides.map((item) => { - if (!wellKnownProvisionTypes.includes(item.ty)) { - return Promise.resolve(); - } + await Promise.all( + reducedRecipe.provides.map((item) => { + if (!wellKnownProvisionTypes.includes(item.ty)) { + return Promise.resolve(); + } - const wellKnownProv = item as WellKnownProvision; - switch (wellKnownProv.ty) { - case "posix.exec": - binPaths.push(wellKnownProv.absolutePath); - break; - case "posix.sharedLib": - libPaths.push(wellKnownProv.absolutePath); - break; - case "posix.headerFile": - includePaths.push(wellKnownProv.absolutePath); - break; - // case "posix.envVarDyn": - case "posix.envVar": - if (vars[wellKnownProv.key]) { - throw new Error( - `env var conflict cooking unix env: key "${wellKnownProv.key}" has entries "${ - vars[wellKnownProv.key] - }" and "${wellKnownProv.val}"`, + const wellKnownProv = item as WellKnownProvision; + switch (wellKnownProv.ty) { + case "posix.exec": + binPaths.push(wellKnownProv.absolutePath); + break; + case "posix.sharedLib": + libPaths.push(wellKnownProv.absolutePath); + break; + case "posix.headerFile": + includePaths.push(wellKnownProv.absolutePath); + break; + // case "posix.envVarDyn": + case "posix.envVar": + if (vars[wellKnownProv.key]) { + throw new Error( + `env var conflict cooking unix env: key "${wellKnownProv.key}" has entries "${ + vars[wellKnownProv.key] + }" and "${wellKnownProv.val}"`, + ); + } + vars[wellKnownProv.key] = wellKnownProv.val; + // installSetIds.push(wellKnownProv.installSetIdProvision!.id); + break; + case "posix.execDir": + execDirs.push(wellKnownProv.path); + break; + case "posix.sharedLibDir": + sharedLibDirs.push(wellKnownProv.path); + break; + case "posix.headerDir": + headerDirs.push(wellKnownProv.path); + break; + case "hook.onEnter.posixExec": + onEnterHooks.push([wellKnownProv.program, wellKnownProv.arguments]); + break; + case "hook.onExit.posixExec": + onExitHooks.push([wellKnownProv.program, wellKnownProv.arguments]); + break; + case "ghjk.ports.Install": + // do nothing + break; + default: + throw Error( + `unsupported provision type: ${(wellKnownProv as any).ty}`, ); - } - vars[wellKnownProv.key] = wellKnownProv.val; - // installSetIds.push(wellKnownProv.installSetIdProvision!.id); - break; - case "posix.execDir": - execDirs.push(wellKnownProv.path); - break; - case "posix.sharedLibDir": - sharedLibDirs.push(wellKnownProv.path); - break; - case "posix.headerDir": - headerDirs.push(wellKnownProv.path); - break; - case "hook.onEnter.posixExec": - onEnterHooks.push([wellKnownProv.program, wellKnownProv.arguments]); - break; - case "hook.onExit.posixExec": - onExitHooks.push([wellKnownProv.program, wellKnownProv.arguments]); - break; - case "ghjk.ports.Install": - // do nothing - break; - default: - throw Error( - `unsupported provision type: ${(wellKnownProv as any).ty}`, - ); - } - })); - void await Promise.all([ + } + }), + ); + void (await Promise.all([ // bin shims - await shimLinkPaths( - binPaths, - binShimDir, - ), + await shimLinkPaths(binPaths, binShimDir), // lib shims - await shimLinkPaths( - libPaths, - libShimDir, - ), + await shimLinkPaths(libPaths, libShimDir), // include shims - await shimLinkPaths( - includePaths, - includeShimDir, - ), + await shimLinkPaths(includePaths, includeShimDir), $.path(envDir).join("recipe.json").writeJsonPretty(reducedRecipe), - ]); + ])); // FIXME: prevent malicious env manipulations let LD_LIBRARY_ENV: string; switch (Deno.build.os) { @@ -162,10 +159,7 @@ export async function cookPosixEnv( } /// This expands globs found in the targetPaths -async function shimLinkPaths( - targetPaths: string[], - shimDir: Path, -) { +async function shimLinkPaths(targetPaths: string[], shimDir: Path) { // map of filename to shimPath const shims: Record = {}; // a work sack to append to incase there are globs expanded @@ -174,8 +168,9 @@ async function shimLinkPaths( const file = foundTargetPaths.pop()!; if (std_path.isGlob(file)) { foundTargetPaths.push( - ...(await Array.fromAsync(std_fs.expandGlob(file))) - .map((entry) => entry.path), + ...(await Array.fromAsync(std_fs.expandGlob(file))).map( + (entry) => entry.path, + ), ); continue; } @@ -221,9 +216,7 @@ async function writeActivators( const shareDirVar = "_ghjk_share_dir"; pathVars = { ...Object.fromEntries( - Object.entries(pathVars).map(( - [key, val], - ) => [ + Object.entries(pathVars).map(([key, val]) => [ key, val .replace(gcx.ghjkDir.toString(), "$" + ghjkDirVar) @@ -236,11 +229,12 @@ async function writeActivators( const onEnterHooksEscaped = onEnterHooks.map(([cmd, args]) => [cmd == "ghjk" ? ghjkShimName : cmd, ...args] .join(" ") - .replaceAll("'", "'\\''") + .replaceAll("'", "'\\''"), ); const onExitHooksEscaped = onExitHooks.map(([cmd, args]) => [cmd == "ghjk" ? ghjkShimName : cmd, ...args] - .join(" ").replaceAll("'", "'\\''") + .join(" ") + .replaceAll("'", "'\\''"), ); // ghjk.sh sets the DENO_DIR so we can usually @@ -270,9 +264,9 @@ async function writeActivators( // by defaulting to a value that's guranteed to // be differeint than `key` // TODO: avoid invalid key values elsewhere - const safeComparisionKey = `$\{${key}:-_${ - val.replace(/['"]/g, "").slice(0, 2) - }}`; + const safeComparisionKey = `$\{${key}:-_${val + .replace(/['"]/g, "") + .slice(0, 2)}}`; return [ // we only restore the old $KEY value at cleanup if value of $KEY // is the one set by the activate script @@ -283,13 +277,13 @@ async function writeActivators( // string (that's why we "escaped single quote" the value) // NOTE: the addition sign at the end `GHJK_CLEANUP_POSIX=$GHJK_CLEANUP_POSIX'[ \"${safeComparisionKey}\" = '\\''${safeVal}'\\'' ] && '` + - // we want to capture the old $key value here so we wrap those - // with double quotes but the rest is in single quotes - // within the value of $key - // i.e. export KEY='OLD $VALUE OF KEY' - // but $VALUE won't be expanded when the cleanup actually runs - // we also unset the key if it wasn't previously set - `$([ -z "$\{${key}+x}" ] && echo 'export ${key}= '\\'"$\{${key}:-unreachable}""';" || echo 'unset ${key};');`, + // we want to capture the old $key value here so we wrap those + // with double quotes but the rest is in single quotes + // within the value of $key + // i.e. export KEY='OLD $VALUE OF KEY' + // but $VALUE won't be expanded when the cleanup actually runs + // we also unset the key if it wasn't previously set + `$([ -z "$\{${key}+x}" ] && echo 'export ${key}= '\\'"$\{${key}:-unreachable}""';" || echo 'unset ${key};');`, `export ${key}='${safeVal}';`, ``, ]; @@ -357,7 +351,7 @@ async function writeActivators( // - we don't have to deal with 'set -o nounset' return [ `set --global --append GHJK_CLEANUP_FISH 'test "$${key}" = \\'${safeVal}\\'; and '` + - `(if set -q ${key}; echo 'set --global --export ${key} \\'' "$${key}" "';"; else; echo 'set -e ${key};'; end;);`, + `(if set -q ${key}; echo 'set --global --export ${key} \\'' "$${key}" "';"; else; echo 'set -e ${key};'; end;);`, `set --global --export ${key} '${val}';`, ``, ]; @@ -384,8 +378,8 @@ async function writeActivators( , ``, ` # on exit hooks`, - ...onExitHooksEscaped.map((cmd) => - ` set --global --append GHJK_CLEANUP_FISH '${cmd};';` + ...onExitHooksEscaped.map( + (cmd) => ` set --global --append GHJK_CLEANUP_FISH '${cmd};';`, ), `end`, ], diff --git a/modules/envs/reducer.ts b/modules/envs/reducer.ts index 26c5d9e..23d8cfd 100644 --- a/modules/envs/reducer.ts +++ b/modules/envs/reducer.ts @@ -28,13 +28,9 @@ export type ProvisionReducerStore = Map< * environment provisions, {@link ProvisionReducer}s can be registered * here. */ -export function getProvisionReducerStore( - gcx: GhjkCtx, -) { +export function getProvisionReducerStore(gcx: GhjkCtx) { const id = "provisionReducerStore"; - let store = gcx.blackboard.get(id) as - | ProvisionReducerStore - | undefined; + let store = gcx.blackboard.get(id) as ProvisionReducerStore | undefined; if (!store) { store = new Map(); gcx.blackboard.set(id, store); @@ -60,14 +56,10 @@ export function getProvisionReducerStore( * {@link WellKnownProvision}, looks for reducers in * {@link ProvisionReducer} to convert it to one. */ -export async function reduceStrangeProvisions( - gcx: GhjkCtx, - env: EnvRecipeX, -) { +export async function reduceStrangeProvisions(gcx: GhjkCtx, env: EnvRecipeX) { const reducerStore = getProvisionReducerStore(gcx); // Replace by `Object.groupBy` once the types for it are fixed const bins = {} as Record; - logger(import.meta).debug("provides", env.provides); for (const item of env.provides) { let bin = bins[item.ty]; if (!bin) { @@ -97,7 +89,7 @@ export async function reduceStrangeProvisions( validators.wellKnownProvision.safeParse(prov), { prov }, `error parsing reduced provision`, - ) + ), ), ); } @@ -128,7 +120,7 @@ export function installDynEnvReducer(gcx: GhjkCtx) { if (targetKey) { // console.log("key", key, " maps to target ", targetKey); const results = await execTask(gcx, taskConf, taskGraph, targetKey, []); - output.push({ ...provision, ty, val: results[key] as any ?? "" }); + output.push({ ...provision, ty, val: (results[key] as any) ?? "" }); } else { badProvisions.push(provision); } @@ -158,8 +150,9 @@ export function installDynamicPathVarReducer( const taskGraph = taskCtx.taskGraph; const taskConf = taskCtx.config; - const targetKey = Object.entries(taskConf.tasks) - .find(([_, task]) => task.key === key)?.[0]; + const targetKey = Object.entries(taskConf.tasks).find( + ([_, task]) => task.key === key, + )?.[0]; if (targetKey) { const results = await execTask(gcx, taskConf, taskGraph, targetKey, []); From 4325795354c989efbdcfd756096a1bca199a53ab Mon Sep 17 00:00:00 2001 From: Natoandro Date: Sat, 31 Aug 2024 08:13:50 +0300 Subject: [PATCH 15/16] Use the merged base instead of the finalzed base --- files/cookbook.ts | 7 +++++-- files/merged_envs.ts | 1 - files/mod.ts | 28 +++++++++++----------------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/files/cookbook.ts b/files/cookbook.ts index 926bf77..0b6c07a 100644 --- a/files/cookbook.ts +++ b/files/cookbook.ts @@ -13,11 +13,14 @@ import { import { InstallSetRefProvision, unwrapZodRes } from "../port.ts"; import { InstallSet, MergedEnvs } from "./merged_envs.ts"; import envsValidators from "../modules/envs/types.ts"; +import getLogger from "../utils/logger.ts"; export type Final = ReturnType & { envBaseResolved: null | string[]; }; +const logger = getLogger(import.meta); + interface MergedEntries { vars: Record; dynVars: Record; @@ -55,11 +58,11 @@ export class Cookbook { this.finalizedEnvs[final.key] = { installSetId, finalized: final, - vars: merged.vars, - dynVars: merged.dynVars, + merged, envHash: hash, }; + logger.debug("registering env", { key: final.key, name: final.name, hash }); this.#moduleConfig.envs[hash] = recipe; if (final.name) { this.#moduleConfig.envsNamed[final.name] = hash; diff --git a/files/merged_envs.ts b/files/merged_envs.ts index b64e921..d43df7c 100644 --- a/files/merged_envs.ts +++ b/files/merged_envs.ts @@ -98,7 +98,6 @@ export class ParentEnvs { posixDirs: Array, dynamicPosixDirs: Array, ) { - logger.debug("merge posix dirs", { posixDirs, dynamicPosixDirs }); this.#posixDirs.push(...posixDirs); this.#dynamicPosixDirs.push(...dynamicPosixDirs); } diff --git a/files/mod.ts b/files/mod.ts index d58803c..46bf709 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -11,7 +11,6 @@ import portsValidators from "../modules/ports/types.ts"; import type { AllowedPortDep, InstallConfigFat, - InstallSetRefProvision, PortsModuleConfigHashed, } from "../modules/ports/types.ts"; import getLogger from "../utils/logger.ts"; @@ -38,7 +37,6 @@ import { TaskDefHashed, TasksModuleConfig } from "../modules/tasks/types.ts"; // envs import { DynamicPosixDirProvision, - type EnvRecipe, type EnvsModuleConfig, PosixDirProvision, PosixDirProvisionType, @@ -47,7 +45,7 @@ import { } from "../modules/envs/types.ts"; import envsValidators from "../modules/envs/types.ts"; import modulesValidators from "../modules/types.ts"; -import { InstallSet, ParentEnvs } from "./merged_envs.ts"; +import { InstallSet, MergedEnvs, ParentEnvs } from "./merged_envs.ts"; import { Cookbook, Final } from "./cookbook.ts"; const validators = { @@ -144,8 +142,7 @@ export type TaskDefTyped = DenoTaskDefArgs & { ty: "denoFile@v1" }; export type FinalizedEnv = { finalized: ReturnType; installSetId?: string; - vars: Record; - dynVars: Record; + merged: MergedEnvs; envHash: string; }; @@ -413,22 +410,21 @@ export class Ghjkfile { #mergeEnvs(final: Final) { const parents = final.envBaseResolved ?? []; + logger.debug("merging envs", { base: parents, child: final.key }); const childName = final.key; - logger.debug({ childName, parents }); const parentEnvs = new ParentEnvs(childName); for (const parentName of parents) { - logger.debug("finalized envs", this.#finalizedEnvs[parentName]); - const { installSetId, vars, dynVars, finalized } = - this.#finalizedEnvs[parentName]; + const { installSetId, merged: base } = this.#finalizedEnvs[parentName]; + // FIXME unique?? parentEnvs.addHooks( - finalized.onEnterHookTasks, - finalized.onExitHookTasks, + base.onEnterHookTasks, + base.onExitHookTasks, ); - parentEnvs.mergeVars(parentName, vars); - parentEnvs.mergeDynVars(parentName, dynVars); + parentEnvs.mergeVars(parentName, base.vars); + parentEnvs.mergeDynVars(parentName, base.dynVars); parentEnvs.mergePosixDirs( - finalized.posixDirs, - finalized.dynamicPosixDirs, + base.posixDirs, + base.dynamicPosixDirs, ); if (installSetId) { const set = this.#installSets.get(installSetId)!; @@ -508,7 +504,6 @@ export class Ghjkfile { ) ) { const final = finalizer(); - logger.debug("seen env", _key, final.key); const envBaseResolved = this.#resolveEnvBases( final.inherit, @@ -547,7 +542,6 @@ export class Ghjkfile { const item = workingSet.pop()!; const final = all[item]; - logger.debug("base", final.envBaseResolved); const mergedEnvs = this.#mergeEnvs(final); cookbook.registerEnv(final, mergedEnvs); From 840a6464145fca2423dac91fb135862e1d8a8694 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Sat, 31 Aug 2024 15:25:53 +0300 Subject: [PATCH 16/16] fix pre-commit --- modules/envs/posix.ts | 28 +++++++++++++++------------- modules/envs/reducer.ts | 3 +-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/modules/envs/posix.ts b/modules/envs/posix.ts index d50f45a..e490864 100644 --- a/modules/envs/posix.ts +++ b/modules/envs/posix.ts @@ -229,12 +229,12 @@ async function writeActivators( const onEnterHooksEscaped = onEnterHooks.map(([cmd, args]) => [cmd == "ghjk" ? ghjkShimName : cmd, ...args] .join(" ") - .replaceAll("'", "'\\''"), + .replaceAll("'", "'\\''") ); const onExitHooksEscaped = onExitHooks.map(([cmd, args]) => [cmd == "ghjk" ? ghjkShimName : cmd, ...args] .join(" ") - .replaceAll("'", "'\\''"), + .replaceAll("'", "'\\''") ); // ghjk.sh sets the DENO_DIR so we can usually @@ -264,9 +264,11 @@ async function writeActivators( // by defaulting to a value that's guranteed to // be differeint than `key` // TODO: avoid invalid key values elsewhere - const safeComparisionKey = `$\{${key}:-_${val - .replace(/['"]/g, "") - .slice(0, 2)}}`; + const safeComparisionKey = `$\{${key}:-_${ + val + .replace(/['"]/g, "") + .slice(0, 2) + }}`; return [ // we only restore the old $KEY value at cleanup if value of $KEY // is the one set by the activate script @@ -277,13 +279,13 @@ async function writeActivators( // string (that's why we "escaped single quote" the value) // NOTE: the addition sign at the end `GHJK_CLEANUP_POSIX=$GHJK_CLEANUP_POSIX'[ \"${safeComparisionKey}\" = '\\''${safeVal}'\\'' ] && '` + - // we want to capture the old $key value here so we wrap those - // with double quotes but the rest is in single quotes - // within the value of $key - // i.e. export KEY='OLD $VALUE OF KEY' - // but $VALUE won't be expanded when the cleanup actually runs - // we also unset the key if it wasn't previously set - `$([ -z "$\{${key}+x}" ] && echo 'export ${key}= '\\'"$\{${key}:-unreachable}""';" || echo 'unset ${key};');`, + // we want to capture the old $key value here so we wrap those + // with double quotes but the rest is in single quotes + // within the value of $key + // i.e. export KEY='OLD $VALUE OF KEY' + // but $VALUE won't be expanded when the cleanup actually runs + // we also unset the key if it wasn't previously set + `$([ -z "$\{${key}+x}" ] && echo 'export ${key}= '\\'"$\{${key}:-unreachable}""';" || echo 'unset ${key};');`, `export ${key}='${safeVal}';`, ``, ]; @@ -351,7 +353,7 @@ async function writeActivators( // - we don't have to deal with 'set -o nounset' return [ `set --global --append GHJK_CLEANUP_FISH 'test "$${key}" = \\'${safeVal}\\'; and '` + - `(if set -q ${key}; echo 'set --global --export ${key} \\'' "$${key}" "';"; else; echo 'set -e ${key};'; end;);`, + `(if set -q ${key}; echo 'set --global --export ${key} \\'' "$${key}" "';"; else; echo 'set -e ${key};'; end;);`, `set --global --export ${key} '${val}';`, ``, ]; diff --git a/modules/envs/reducer.ts b/modules/envs/reducer.ts index 23d8cfd..04ff5f1 100644 --- a/modules/envs/reducer.ts +++ b/modules/envs/reducer.ts @@ -1,5 +1,4 @@ import { unwrapZodRes } from "../../port.ts"; -import logger from "../../utils/logger.ts"; import { execTask } from "../tasks/exec.ts"; import { getTasksCtx } from "../tasks/inter.ts"; import type { GhjkCtx } from "../types.ts"; @@ -89,7 +88,7 @@ export async function reduceStrangeProvisions(gcx: GhjkCtx, env: EnvRecipeX) { validators.wellKnownProvision.safeParse(prov), { prov }, `error parsing reduced provision`, - ), + ) ), ); }