From f8549e0c717d5ebd56c62402ee95b4c21cdd695f Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:04:20 +0300 Subject: [PATCH] fix: flakeouts --- docs/faq.md | 20 +++ ports/asdf_plugin_git.ts | 2 +- ports/npmi.ts | 7 +- src/deno_ports/mod.ts | 94 ++++++++++- src/deno_utils/mod.ts | 181 +------------------- src/ghjk/main.rs | 2 +- src/ghjk_ts/file.ts | 26 +-- src/ghjk_ts/mod.ts | 11 +- src/sys_deno/ports/deps.ts | 4 - src/sys_deno/ports/mod.ts | 270 +----------------------------- src/sys_deno/ports/std.ts | 77 ++++----- src/sys_deno/ports/std_runtime.ts | 2 +- src/sys_deno/ports/sync.ts | 3 +- src/sys_deno/ports/types.ts | 57 +++++-- src/sys_deno/ports/utils.ts | 140 ---------------- tests/ports.ts | 2 - tests/portsOutdated.ts | 59 ------- tools/check.ts | 2 +- 18 files changed, 219 insertions(+), 740 deletions(-) create mode 100644 docs/faq.md delete mode 100644 src/sys_deno/ports/utils.ts delete mode 100644 tests/portsOutdated.ts diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..bcbc4d6 --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,20 @@ +# Frequently Asked Questions + +This list is incomplete; you can help by [expanding it](https://github.com/metatypedev/ghjk/discussions). + +## How to increase verbosity? + +Set the `GHJK_LOG` environment variable to `debug` to enable more verbose logging. + +```bash +export GHJK_LOG=debug +``` + +For even more logs, one can set the `RUST_LOG` to `debug` or `trace`. +This is usually too verbose to be useful so a more targeted logging level is required. + +```bash +export GHJK_LOG='info,ghjk=debug' +``` +Setting `info,ghjk=debug`. +Please refer to the tracing [docs](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives) to learn how to manage log verbosity per module. diff --git a/ports/asdf_plugin_git.ts b/ports/asdf_plugin_git.ts index ddd99ea..ad51983 100644 --- a/ports/asdf_plugin_git.ts +++ b/ports/asdf_plugin_git.ts @@ -52,6 +52,7 @@ export type AsdfPluginInstallConf = */ export default function conf(config: AsdfPluginInstallConf) { return { + ...readGhVars(), ...confValidator.parse(config), port: manifest, }; @@ -59,7 +60,6 @@ export default function conf(config: AsdfPluginInstallConf) { export function buildDep(): AllowedPortDep { return { - ...readGhVars(), manifest, defaultInst: { portRef: getPortRef(manifest), diff --git a/ports/npmi.ts b/ports/npmi.ts index 4b8fbc8..24e86eb 100644 --- a/ports/npmi.ts +++ b/ports/npmi.ts @@ -85,9 +85,12 @@ export class Port extends PortBase { const conf = confValidator.parse(args.config); await $.raw`${depExecShimPath(std_ports.node_org, "npm", args.depArts) // provide prefix flat to avoid looking at package.json in parent dirs - } install --prefix ${args.tmpDirPath} --no-fund ${conf.packageName}@${args.installVersion}` + } install --prefix ${args.tmpDirPath} --no-update-notifier --no-fund ${conf.packageName}@${args.installVersion}` .cwd(args.tmpDirPath) - .env(pathsWithDepArts(args.depArts, args.platform.os)); + .env({ + ...pathsWithDepArts(args.depArts, args.platform.os), + NO_UPDATE_NOTIFIER: "1", + }); await std_fs.move(args.tmpDirPath, args.downloadPath); } diff --git a/src/deno_ports/mod.ts b/src/deno_ports/mod.ts index a27bc0a..2878e22 100644 --- a/src/deno_ports/mod.ts +++ b/src/deno_ports/mod.ts @@ -15,9 +15,16 @@ export * from "./unar/mod.ts"; export { default as portsValidators } from "../sys_deno/ports/types.ts"; export { serializePlatform } from "../sys_deno/ports/types/platform.ts"; +import { $ } from "../deno_utils/mod.ts"; import { std_url } from "../deps.ts"; import { PortBase } from "../sys_deno/ports/base.ts"; -import type { ArchEnum, ListAllArgs, OsEnum } from "../sys_deno/ports/types.ts"; +import type { + ArchEnum, + DepArts, + ListAllArgs, + OsEnum, + PortDep, +} from "../sys_deno/ports/types.ts"; import { serializePlatform } from "../sys_deno/ports/types/platform.ts"; export function dwnUrlOut(url: string) { @@ -43,3 +50,88 @@ export async function defaultLatestStable( } return allVers[allVers.length - 1]; } + +export function depExecShimPath( + dep: PortDep, + execName: string, + depArts: DepArts, +) { + const path = tryDepExecShimPath(dep, execName, depArts); + if (!path) { + throw new Error( + `unable to find shim path for bin "${execName}" of dep ${dep.name}`, + ); + } + return path; +} + +export function depEnv( + dep: PortDep, + depArts: DepArts, +) { + return depArts[dep.name]?.env ?? {}; +} + +export function tryDepExecShimPath( + dep: PortDep, + execName: string, + depArts: DepArts, +) { + const shimPaths = depArts[dep.name]; + if (!shimPaths) { + return; + } + // FIXME: match despite `.exe` extension on windows + const path = shimPaths.execs[execName]; + if (!path) { + return; + } + return path; +} + +export function pathsWithDepArts( + depArts: DepArts, + os: OsEnum, +) { + const pathSet = new Set(); + const libSet = new Set(); + const includesSet = new Set(); + for (const [_, { execs, libs, includes }] of Object.entries(depArts)) { + for (const [_, binPath] of Object.entries(execs)) { + pathSet.add($.path(binPath).parentOrThrow()); + } + for (const [_, libPath] of Object.entries(libs)) { + libSet.add($.path(libPath).parentOrThrow()); + } + for (const [_, incPath] of Object.entries(includes)) { + includesSet.add($.path(incPath).parentOrThrow()); + } + } + + let LD_LIBRARY_ENV: string; + switch (os) { + case "darwin": + LD_LIBRARY_ENV = "DYLD_LIBRARY_PATH"; + break; + case "linux": + LD_LIBRARY_ENV = "LD_LIBRARY_PATH"; + break; + default: + throw new Error(`unsupported os ${os}`); + } + return { + PATH: `${[...pathSet.keys()].join(":")}:${Deno.env.get("PATH") ?? ""}`, + LIBRARY_PATH: `${[...libSet.keys()].join(":")}:${ + Deno.env.get("LIBRARY_PATH") ?? "" + }`, + C_INCLUDE_PATH: `${[...includesSet.keys()].join(":")}:${ + Deno.env.get("C_INCLUDE_PATH") ?? "" + }`, + CPLUS_INCLUDE_PATH: `${[...includesSet.keys()].join(":")}:${ + Deno.env.get("CPLUS_INCLUDE_PATH") ?? "" + }`, + [LD_LIBRARY_ENV]: `${[...libSet.keys()].join(":")}:${ + Deno.env.get(LD_LIBRARY_ENV) ?? "" + }`, + }; +} diff --git a/src/deno_utils/mod.ts b/src/deno_utils/mod.ts index 20a9f71..6962d53 100644 --- a/src/deno_utils/mod.ts +++ b/src/deno_utils/mod.ts @@ -1,5 +1,9 @@ /// +// class re-exports are tricky. We want al importers +// of path to get it from here so we rename in common.ts +export { _DaxPath as Path } from "./deps.ts"; + import { _DaxPath as Path, dax, @@ -13,19 +17,9 @@ import { zod, zod_val_err, } from "./deps.ts"; -// class re-exports are tricky. We want al importers -// of path to get it from here so we rename in common.ts -export { _DaxPath as Path } from "./deps.ts"; import logger, { isColorfulTty } from "./logger.ts"; -// NOTE: only use type imports only when getting stuff from "./modules" -import type { - DepArts, - InstallConfigFat, - InstallConfigResolvedX, - OsEnum, - PortDep, - PortManifest, -} from "../sys_deno/ports/types.ts"; +// NOTE: only use type imports only when getting stuff from "./sys_deno" +import type { OsEnum } from "../sys_deno/ports/types.ts"; export type DeArrayify = T extends Array ? Inner : T; const literalSchema = zod.union([ @@ -47,91 +41,6 @@ export function dbg(val: T, ...more: unknown[]) { return val; } -export function pathsWithDepArts( - depArts: DepArts, - os: OsEnum, -) { - const pathSet = new Set(); - const libSet = new Set(); - const includesSet = new Set(); - for (const [_, { execs, libs, includes }] of Object.entries(depArts)) { - for (const [_, binPath] of Object.entries(execs)) { - pathSet.add($.path(binPath).parentOrThrow()); - } - for (const [_, libPath] of Object.entries(libs)) { - libSet.add($.path(libPath).parentOrThrow()); - } - for (const [_, incPath] of Object.entries(includes)) { - includesSet.add($.path(incPath).parentOrThrow()); - } - } - - let LD_LIBRARY_ENV: string; - switch (os) { - case "darwin": - LD_LIBRARY_ENV = "DYLD_LIBRARY_PATH"; - break; - case "linux": - LD_LIBRARY_ENV = "LD_LIBRARY_PATH"; - break; - default: - throw new Error(`unsupported os ${os}`); - } - return { - PATH: `${[...pathSet.keys()].join(":")}:${Deno.env.get("PATH") ?? ""}`, - LIBRARY_PATH: `${[...libSet.keys()].join(":")}:${ - Deno.env.get("LIBRARY_PATH") ?? "" - }`, - C_INCLUDE_PATH: `${[...includesSet.keys()].join(":")}:${ - Deno.env.get("C_INCLUDE_PATH") ?? "" - }`, - CPLUS_INCLUDE_PATH: `${[...includesSet.keys()].join(":")}:${ - Deno.env.get("CPLUS_INCLUDE_PATH") ?? "" - }`, - [LD_LIBRARY_ENV]: `${[...libSet.keys()].join(":")}:${ - Deno.env.get(LD_LIBRARY_ENV) ?? "" - }`, - }; -} - -export function depExecShimPath( - dep: PortDep, - execName: string, - depArts: DepArts, -) { - const path = tryDepExecShimPath(dep, execName, depArts); - if (!path) { - throw new Error( - `unable to find shim path for bin "${execName}" of dep ${dep.name}`, - ); - } - return path; -} - -export function depEnv( - dep: PortDep, - depArts: DepArts, -) { - return depArts[dep.name]?.env ?? {}; -} - -export function tryDepExecShimPath( - dep: PortDep, - execName: string, - depArts: DepArts, -) { - const shimPaths = depArts[dep.name]; - if (!shimPaths) { - return; - } - // FIXME: match despite `.exe` extension on windows - const path = shimPaths.execs[execName]; - if (!path) { - return; - } - return path; -} - const syncSha256Hasher = multihasher.from({ code: multisha2.sha256.code, name: multisha2.sha256.name, @@ -168,15 +77,6 @@ export function objectHash( return stringHash(json_canonicalize(object)); } -export function getPortRef(manifest: PortManifest) { - return `${manifest.name}@${manifest.version}`; -} - -export function getInstallHash(install: InstallConfigResolvedX) { - const fullHashHex = objectHash(JSON.parse(JSON.stringify(install))); - return `${install.portRef}!${fullHashHex}`; -} - export function defaultCommandBuilder() { const builder = new dax.CommandBuilder() .printCommand(true); @@ -187,16 +87,6 @@ export function defaultCommandBuilder() { return builder; } -// type Last = T extends readonly [...any, infer R] ? R -// : DeArrayify; -// -// type Ser[]> = T extends -// readonly [...Promise[], infer R] ? { (...promises: T): R } -// : { -// (...promises: T): DeArrayify; -// }; -// - let requestBuilder; try { requestBuilder = new dax.RequestBuilder() @@ -250,29 +140,10 @@ export const $ = dax.build$( depth: 10, }); }, + co: ((values: any[]) => Promise.all(values)) as typeof Promise.all, collector: promiseCollector, - /* co( - values: T, - ): Promise<{ -readonly [P in keyof T]: Awaited }> { - return Promise.all(values); - }, */ - // coIter( - // items: Iterable, - // fn: (item:T) => PromiseLike, - // opts: { - // limit: "cpu" | number; - // } = { - // limit: "cpu" - // }, - // ): Promise[]> { - // const limit = opts.limit == "cpu" ? AVAIL_CONCURRENCY : opts.limit; - // const promises = [] as PromiseLike[]; - // let freeSlots = limit; - // do { - // } while(true); - // return Promise.all(promises); - // } + pathToString(path: Path) { return path.toString(); }, @@ -514,14 +385,6 @@ export function asyncRc(val: T, onDrop: (val: T) => Promise) { return rc; } -export function thinInstallConfig(fat: InstallConfigFat) { - const { port, ...lite } = fat; - return { - portRef: getPortRef(port), - ...lite, - }; -} - export type OrRetOf = T extends () => infer Inner ? Inner : T; /** @@ -672,31 +535,3 @@ export function promiseCollector(promises: Promise[] = []) { }, }; } - -// /** -// * Blocks the event loop till the promise is resolved -// */ -// export function deasyncPromise(promise: Promise) { -// // Note: npm:deasync does not work on deno -// // TODO: better impl or use a lib? -// const sleepSync = (timeout: number) => { -// Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, timeout); -// }; - -// let resolved = false; -// let ret: T | undefined, err; - -// promise -// .then((r) => { ret = r; }) -// .catch((e) => { err = e; }) -// .finally(() => { resolved = true; }); - -// while (!resolved) { -// sleepSync(100); -// } - -// if (err) { -// throw err; -// } -// return ret; -// } diff --git a/src/ghjk/main.rs b/src/ghjk/main.rs index 0a2ab48..81af760 100644 --- a/src/ghjk/main.rs +++ b/src/ghjk/main.rs @@ -18,6 +18,7 @@ mod interlude { pub use eyre::{format_err as ferr, Context, Result as Res, WrapErr}; pub use futures::{future::BoxFuture, FutureExt}; pub use indexmap::IndexMap; + pub use itertools::Itertools; pub use serde::{Deserialize, Serialize}; pub use serde_json::json; pub use smallvec::smallvec as svec; @@ -74,7 +75,6 @@ fn main() -> Res { } } -use itertools::Itertools; use shadow_rs::shadow; shadow!(shadow); diff --git a/src/ghjk_ts/file.ts b/src/ghjk_ts/file.ts index 02e7220..73de5db 100644 --- a/src/ghjk_ts/file.ts +++ b/src/ghjk_ts/file.ts @@ -7,7 +7,7 @@ import { deep_eql, multibase32, multibase64, zod } from "../deps.ts"; // ports specific imports -import portsValidators from "../sys_deno/ports/types.ts"; +import portsValidators, { reduceAllowedDeps } from "../sys_deno/ports/types.ts"; import type { AllowedPortDep, InstallConfigFat, @@ -21,7 +21,6 @@ import { defaultCommandBuilder, objectHash, Path, - thinInstallConfig, unwrapZodRes, } from "../deno_utils/mod.ts"; import * as std_ports from "../sys_deno/ports/std.ts"; @@ -1118,29 +1117,6 @@ type InlineTaskHookProvision = Provision & { taskKey: string; }; -export function reduceAllowedDeps( - deps: (AllowedPortDep | InstallConfigFat)[], -): AllowedPortDep[] { - return deps.map( - (dep: any) => { - { - const res = portsValidators.allowedPortDep.safeParse(dep); - if (res.success) return res.data; - } - const inst = unwrapZodRes( - portsValidators.installConfigFat.safeParse(dep), - dep, - "invalid allowed dep object, provide either InstallConfigFat or AllowedPortDep objects", - ); - const out: AllowedPortDep = { - manifest: inst.port, - defaultInst: thinInstallConfig(inst), - }; - return portsValidators.allowedPortDep.parse(out); - }, - ); -} - function objectHashSafe(obj: unknown) { return objectHash(JSON.parse(JSON.stringify(obj))); } diff --git a/src/ghjk_ts/mod.ts b/src/ghjk_ts/mod.ts index 33ec9d2..8b26a41 100644 --- a/src/ghjk_ts/mod.ts +++ b/src/ghjk_ts/mod.ts @@ -5,15 +5,16 @@ import "../deno_utils/setup_logger.ts"; // ports specific imports -import type { - AllowedPortDep, - InstallConfigFat, +import { + type AllowedPortDep, + type InstallConfigFat, + reduceAllowedDeps, } from "../sys_deno/ports/types.ts"; import logger from "../deno_utils/logger.ts"; import { $ } from "../deno_utils/mod.ts"; -import { EnvBuilder, Ghjkfile, reduceAllowedDeps, stdDeps } from "./file.ts"; +import { EnvBuilder, Ghjkfile, stdDeps } from "./file.ts"; import type { DenoTaskDefArgs, EnvDefArgs, TaskFn } from "./file.ts"; -// WARN: this module has side-effects and only ever import +// WARN: following module has side-effects and only ever import // types from it import type { ExecTaskArgs } from "../sys_deno/tasks/deno.ts"; diff --git a/src/sys_deno/ports/deps.ts b/src/sys_deno/ports/deps.ts index 16f22e4..194dd5d 100644 --- a/src/sys_deno/ports/deps.ts +++ b/src/sys_deno/ports/deps.ts @@ -1,5 +1 @@ -//! dependencies used by the cli - export * from "../../deps.ts"; - -export { Table } from "https://deno.land/x/cliffy@v1.0.0-rc.4/table/table.ts"; diff --git a/src/sys_deno/ports/mod.ts b/src/sys_deno/ports/mod.ts index cc59465..12dbaa9 100644 --- a/src/sys_deno/ports/mod.ts +++ b/src/sys_deno/ports/mod.ts @@ -2,41 +2,26 @@ export * from "./types.ts"; -import { Table, zod } from "./deps.ts"; -import { $, Json, unwrapZodRes } from "../../deno_utils/mod.ts"; +import { zod } from "./deps.ts"; +import { Json, unwrapZodRes } from "../../deno_utils/mod.ts"; import logger from "../../deno_utils/logger.ts"; import validators, { - installProvisionTy, installSetProvisionTy, installSetRefProvisionTy, } from "./types.ts"; -import envsValidators from "../envs/types.ts"; -import type { - AllowedPortDep, - InstallConfigResolved, - InstallProvision, - InstallSetRefProvision, - InstallSetX, - PortsModuleConfigX, -} from "./types.ts"; -import { type GhjkCtx, type ModuleManifest } from "../types.ts"; +import type { InstallSetX, PortsModuleConfigX } from "./types.ts"; +import { type ModuleManifest } from "../types.ts"; import { ModuleBase } from "../mod.ts"; import { buildInstallGraph, - getDepConfig, - getPortImpl, getResolutionMemo, - getShimmedDepArts, - resolveAndInstall, syncCtxFromGhjk, -} from "./sync.ts"; // TODO: rename to install.ts +} from "./sync.ts"; import type { Blackboard } from "../types.ts"; import { getProvisionReducerStore } from "../envs/reducer.ts"; import { installSetReducer, installSetRefReducer } from "./reducers.ts"; import type { Provision, ProvisionReducer } from "../envs/types.ts"; import { getPortsCtx } from "./inter.ts"; -import { updateInstall } from "./utils.ts"; -import { getEnvsCtx } from "../envs/inter.ts"; import type { CliCommand } from "../types.ts"; export type PortsCtx = { @@ -137,32 +122,6 @@ export class PortsModule extends ModuleBase { } }, }, - { - name: "outdated", - about: "Show a version table for installs.", - flags: { - updateInstall: { - short: "u", - long: "update-install", - value_name: "INSTALL ID", - }, - updateAll: { - short: "a", - long: "update-all", - action: "SetTrue", - }, - }, - action: async function ( - { flags: { updateInstall, updateAll } }, - ) { - await outdatedCommand( - gcx, - pcx, - updateInstall as string | undefined, - updateAll as boolean | undefined, - ); - }, - }, ], }]; return out; @@ -200,222 +159,3 @@ export class PortsModule extends ModuleBase { }; } } - -async function outdatedCommand( - gcx: GhjkCtx, - pcx: PortsCtx, - updateInstallFlag?: string, - updateAllFlag?: boolean, -) { - const envsCtx = getEnvsCtx(gcx); - const envName = envsCtx.activeEnv; - - const installSets = pcx.config.sets; - - let currInstallSetId; - { - const activeEnvName = envsCtx.activeEnv; - const activeEnv = envsCtx.config - .envs[ - envsCtx.config.envsNamed[activeEnvName] ?? activeEnvName - ]; - if (!activeEnv) { - throw new Error( - `No env found under given name "${activeEnvName}"`, - ); - } - - const instSetRef = activeEnv.provides.filter((prov) => - prov.ty === installSetRefProvisionTy - )[0] as InstallSetRefProvision; - - currInstallSetId = instSetRef.setId; - } - const currInstallSet = installSets[currInstallSetId]; - const allowedDeps = currInstallSet.allowedBuildDeps; - - const rows = []; - const { - installedPortsVersions: installed, - latestPortsVersions: latest, - installConfigs, - } = await getOldNewVersionComparison( - gcx, - envName, - allowedDeps, - ); - for (let [installId, installedVersion] of installed.entries()) { - let latestVersion = latest.get(installId); - if (!latestVersion) { - throw new Error( - `Couldn't find the latest version for install id: ${installId}`, - ); - } - - if (latestVersion[0] === "v") { - latestVersion = latestVersion.slice(1); - } - if (installedVersion[0] === "v") { - installedVersion = installedVersion.slice(1); - } - - const config = installConfigs.get(installId); - - if (!config) { - throw new Error( - `Config not found for install id: ${installId}`, - ); - } - - if (config["specifiedVersion"]) { - latestVersion = "=" + latestVersion; - } - - const presentableConfig = { ...config }; - ["buildDepConfigs", "version", "specifiedVersion"].map( - (key) => { - delete presentableConfig[key]; - }, - ); - const row = [ - $.inspect(presentableConfig), - installedVersion, - latestVersion, - ]; - rows.push(row); - } - - if (updateInstallFlag) { - const installId = updateInstallFlag; - const newVersion = latest.get(installId); - if (!newVersion) { - logger().info( - `Error while fetching the latest version for: ${installId}`, - ); - return; - } - await updateInstall(gcx, installId, newVersion, allowedDeps); - return; - } - - if (updateAllFlag) { - for (const [installId, newVersion] of latest.entries()) { - await updateInstall(gcx, installId, newVersion, allowedDeps); - } - return; - } - - const _versionTable = new Table() - .header(["Install Config", "Old Version", "New Version"]) - .body(rows) - .border() - .padding(1) - .indent(2) - .maxColWidth(30) - .render(); -} - -async function getOldNewVersionComparison( - gcx: GhjkCtx, - envName: string, - allowedDeps: Record, -) { - await using scx = await syncCtxFromGhjk(gcx); - - const envDir = $.path(gcx.ghjkDir).join("envs", envName); - const recipePath = envDir.join("recipe.json").toString(); - - // read from `recipe.json` and get installSetIds - const recipeJson = JSON.parse(await Deno.readTextFile(recipePath)); - const reducedRecipe = unwrapZodRes( - envsValidators.envRecipe.safeParse(recipeJson), - { - envName, - recipePath, - }, - "error parsing recipe.json", - ); - - const installProvisions = reducedRecipe.provides.filter((prov) => - prov.ty === installProvisionTy - ) as InstallProvision[]; - - const db = scx.db.val; - - const installedPortsVersions = new Map(); - const latestPortsVersions = new Map(); - const installConfigs = new Map(); - - // get the current/installed version for the ports - for ( - const installProv of installProvisions - ) { - const installId = installProv.instId; - const install = await db.get(installId); - - if (!install) { - throw new Error("InstallId not found in InstallsDb", { - cause: { - installId, - }, - }); - } - - const manifest = install.manifest; - const config = install.conf; - - const resolvedResolutionDeps = [] as [string, string][]; - for (const dep of manifest.resolutionDeps ?? []) { - const { manifest: depManifest, config: depConf } = getDepConfig( - allowedDeps, - manifest, - config, - dep, - ); - - // TODO: avoid reinstall, infact just do a resolve - const depInstId = await resolveAndInstall( - scx, - allowedDeps, - depManifest, - depConf, - ); - resolvedResolutionDeps.push([depInstId.installId, depManifest.name]); - } - - const depShimsRootPath = await Deno.makeTempDir({ - dir: scx.tmpPath, - prefix: `shims_resDeps_${manifest.name}_`, - }); - const resolutionDepArts = await getShimmedDepArts( - scx, - depShimsRootPath, - resolvedResolutionDeps, - ); - - const port = getPortImpl(manifest); - const listAllArgs = { - depArts: resolutionDepArts, - config, - manifest, - }; - - // get the current Version - const currentVersion = config.version; - installedPortsVersions.set(installId, currentVersion); - - // get the latest version of the port - const latestStable = await port.latestStable(listAllArgs); - latestPortsVersions.set(installId, latestStable); - - installConfigs.set(installId, config); - - await $.removeIfExists(depShimsRootPath); - } - - return { - installedPortsVersions: installedPortsVersions, - latestPortsVersions: latestPortsVersions, - installConfigs: installConfigs, - }; -} diff --git a/src/sys_deno/ports/std.ts b/src/sys_deno/ports/std.ts index 33f9ae2..48a5595 100644 --- a/src/sys_deno/ports/std.ts +++ b/src/sys_deno/ports/std.ts @@ -1,51 +1,34 @@ //! This plugin exports the list of standard ports other //! plugins are expected to depend on. -import validators, { - type AllowedPortDepX, - type PortDep, - type PortManifest, -} from "./types.ts"; -import { manifest as man_tar_aa } from "../../../ports/tar.ts"; -import { manifest as man_unzip_aa } from "../../../ports/unzip.ts"; -import { manifest as man_zstd_aa } from "../../../ports/zstd.ts"; -import { manifest as man_git_aa } from "../../../ports/git.ts"; -import { manifest as man_curl_aa } from "../../../ports/curl.ts"; -import { manifest as man_cbin_ghrel } from "../../../ports/cargo-binstall.ts"; -import { manifest as man_node_org } from "../../../ports/node.ts"; -import { manifest as man_asdf_plugin_git } from "../../../ports/asdf_plugin_git.ts"; -import { manifest as man_cpy_bs_ghrel } from "../../../ports/cpy_bs.ts"; -import { manifest as man_rustup_rustlang } from "../../../ports/rustup.ts"; -import { manifest as man_rust_rustup } from "../../../ports/rust.ts"; -import { getPortRef } from "../../deno_utils/mod.ts"; +import { type PortDep, reduceAllowedDeps } from "./types.ts"; +import * as man_tar_aa from "../../../ports/tar.ts"; +import * as man_unzip_aa from "../../../ports/unzip.ts"; +import * as man_zstd_aa from "../../../ports/zstd.ts"; +import * as man_git_aa from "../../../ports/git.ts"; +import * as man_curl_aa from "../../../ports/curl.ts"; +import * as man_cbin_ghrel from "../../../ports/cargo-binstall.ts"; +import * as man_node_org from "../../../ports/node.ts"; +import * as man_asdf_plugin_git from "../../../ports/asdf_plugin_git.ts"; +import * as man_cpy_bs_ghrel from "../../../ports/cpy_bs.ts"; +import * as man_rustup_rustlang from "../../../ports/rustup.ts"; +import * as man_rust_rustup from "../../../ports/rust.ts"; -const aaPorts: PortManifest[] = [ +/** + * The default set of allowed port deps. + */ +const defaultAllowedDeps = reduceAllowedDeps([ + // ambient ports man_tar_aa, man_git_aa, man_curl_aa, man_unzip_aa, man_zstd_aa, -]; - -const denoPorts: PortManifest[] = [ + // denoFile ports man_rustup_rustlang, man_cbin_ghrel, -]; - -/** - * The default set of allowed port deps. - */ -const defaultAllowedDeps: AllowedPortDepX[] = [ - ...aaPorts, - ...denoPorts, ] - .map((manifest) => ({ - manifest, - defaultInst: { - portRef: getPortRef(manifest), - }, - })) - .map((portDep) => validators.allowedPortDep.parse(portDep)); + .map((portMods) => portMods.default())); export const map = Object.freeze( Object.fromEntries( @@ -54,45 +37,45 @@ export const map = Object.freeze( ); export const tar_aa = Object.freeze({ - name: man_tar_aa.name, + name: man_tar_aa.manifest.name, } as PortDep); export const zstd_aa = Object.freeze({ - name: man_zstd_aa.name, + name: man_zstd_aa.manifest.name, } as PortDep); export const unzip_aa = Object.freeze({ - name: man_unzip_aa.name, + name: man_unzip_aa.manifest.name, } as PortDep); export const git_aa = Object.freeze({ - name: man_git_aa.name, + name: man_git_aa.manifest.name, } as PortDep); export const curl_aa = Object.freeze({ - name: man_curl_aa.name, + name: man_curl_aa.manifest.name, } as PortDep); export const rustup_rustlang = Object.freeze({ - name: man_rustup_rustlang.name, + name: man_rustup_rustlang.manifest.name, } as PortDep); export const rust_rustup = Object.freeze({ - name: man_rust_rustup.name, + name: man_rust_rustup.manifest.name, } as PortDep); export const cbin_ghrel = Object.freeze({ - name: man_cbin_ghrel.name, + name: man_cbin_ghrel.manifest.name, } as PortDep); export const node_org = Object.freeze({ - name: man_node_org.name, + name: man_node_org.manifest.name, } as PortDep); export const cpy_bs_ghrel = Object.freeze({ - name: man_cpy_bs_ghrel.name, + name: man_cpy_bs_ghrel.manifest.name, } as PortDep); export const asdf_plugin_git = Object.freeze({ - name: man_asdf_plugin_git.name, + name: man_asdf_plugin_git.manifest.name, } as PortDep); diff --git a/src/sys_deno/ports/std_runtime.ts b/src/sys_deno/ports/std_runtime.ts index 8654827..ab1206d 100644 --- a/src/sys_deno/ports/std_runtime.ts +++ b/src/sys_deno/ports/std_runtime.ts @@ -11,7 +11,7 @@ export default [ // commonly used by the pipi port for installation using pip cpy_bs.default(), // commonly used by the cargobi port for building crates - rust.default(), + rust.default({ profile: "minimal" }), // used by the asdf port for installing asdf plugins asdf_plugin_git.buildDep(), ]; diff --git a/src/sys_deno/ports/sync.ts b/src/sys_deno/ports/sync.ts index 59a00eb..9adc512 100644 --- a/src/sys_deno/ports/sync.ts +++ b/src/sys_deno/ports/sync.ts @@ -18,13 +18,12 @@ import type { PortDep, PortManifestX, } from "./types.ts"; +import { getInstallHash, getPortRef } from "./types.ts"; import { DenoWorkerPort } from "./worker.ts"; import { AmbientAccessPort } from "./ambient.ts"; import { $, AVAIL_CONCURRENCY, - getInstallHash, - getPortRef, objectHash, type Rc, rc, diff --git a/src/sys_deno/ports/types.ts b/src/sys_deno/ports/types.ts index 6e24852..6863bac 100644 --- a/src/sys_deno/ports/types.ts +++ b/src/sys_deno/ports/types.ts @@ -2,7 +2,11 @@ import { semver, zod } from "./deps.ts"; import moduleValidators from "../types.ts"; -import { relativeToRepoRoot } from "../../deno_utils/mod.ts"; +import { + objectHash, + relativeToRepoRoot, + unwrapZodRes, +} from "../../deno_utils/mod.ts"; import { ALL_ARCH, ALL_OS, archEnum, osEnum } from "./types/platform.ts"; export { ALL_ARCH, ALL_OS, archEnum, osEnum }; @@ -109,10 +113,7 @@ const stdInstallConfigFat = installConfigBaseFat.merge(zod.object({})) const stdInstallConfigLite = installConfigBaseLite.merge(zod.object({})) .passthrough(); -const installConfigLite = - // zod.union([ - stdInstallConfigLite; -// ]); +const installConfigLite = stdInstallConfigLite; const installConfigFat = stdInstallConfigFat; const installConfigResolved = installConfigLite.merge(zod.object({ @@ -132,11 +133,6 @@ const installConfigResolved = installConfigLite.merge(zod.object({ // fields meant for sibs. // Which's to say ordering matters const installConfig = zod.union([ - // NOTE: generated types break if we make a union of other unions - // so we get the schemas of those unions instead - // https://github.com/colinhacks/zod/discussions/3010 - // ...installConfigLite.options, - // ...installConfigFat.options, stdInstallConfigLite, stdInstallConfigFat, ]); @@ -395,7 +391,6 @@ export interface PortArgsBase { export interface ListAllArgs { depArts: DepArts; manifest: PortManifestX; - // FIXME: switch to X type when https://github.com/colinhacks/zod/issues/2864 is resolved config: InstallConfigLiteX; } @@ -415,3 +410,43 @@ export interface InstallArgs extends PortArgsBase { export type DownloadArtifacts = zod.infer; export type InstallArtifacts = zod.infer; + +export function reduceAllowedDeps( + deps: (AllowedPortDep | InstallConfigFat)[], +): AllowedPortDep[] { + return deps.map( + (dep: any) => { + { + const res = allowedPortDep.safeParse(dep); + if (res.success) return res.data; + } + const inst = unwrapZodRes( + installConfigFat.safeParse(dep), + dep, + "invalid allowed dep object, provide either InstallConfigFat or AllowedPortDep objects", + ); + const out: AllowedPortDep = { + manifest: inst.port, + defaultInst: thinInstallConfig(inst), + }; + return allowedPortDep.parse(out); + }, + ); +} + +export function getPortRef(manifest: PortManifest) { + return `${manifest.name}@${manifest.version}`; +} + +export function thinInstallConfig(fat: InstallConfigFat) { + const { port, ...lite } = fat; + return { + portRef: getPortRef(port), + ...lite, + }; +} + +export function getInstallHash(install: InstallConfigResolvedX) { + const fullHashHex = objectHash(JSON.parse(JSON.stringify(install))); + return `${install.portRef}!${fullHashHex}`; +} diff --git a/src/sys_deno/ports/utils.ts b/src/sys_deno/ports/utils.ts deleted file mode 100644 index 88d2e9b..0000000 --- a/src/sys_deno/ports/utils.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { std_path } from "./deps.ts"; -import logger from "../../deno_utils/logger.ts"; -import { $ } from "../../deno_utils/mod.ts"; -import { GhjkCtx } from "../types.ts"; -import { AllowedPortDep } from "./mod.ts"; -import { - doDownloadStage, - doInstallStage, - getDepConfig, - getShimmedDepArts, - resolveAndInstall, - SyncCtx, - syncCtxFromGhjk, -} from "./sync.ts"; -import { InstallConfigResolvedX, PortManifestX } from "./types.ts"; - -export async function updateInstall( - gcx: GhjkCtx, - installId: string, - newVersion: string, - allowedDeps: Record, -) { - await using scx = await syncCtxFromGhjk(gcx); - - const db = scx.db.val; - - const install = await db.get(installId); - - if (!install) { - throw new Error("InstallSetId not found in InstallsDb", { - cause: { - installId, - }, - }); - } - - const config = install.conf; - - if (config.version === newVersion) { - logger().info("Skipping update. Install is already uptodate"); - return; - } - - // it's a user specified install, so skip - if (config.specifiedVersion) { - logger().info(`Skipping Version Specified Install: ${installId}`); - return; - } - - config.version = newVersion; - logger().info(`Updating installId ${installId} to version ${newVersion}...`); - await doInstall(installId, scx, install.manifest, allowedDeps, config); - logger().info( - `Successfully updated installId ${installId} to version ${newVersion}`, - ); -} - -async function doInstall( - installId: string, - scx: SyncCtx, - manifest: PortManifestX, - allowedDeps: Record, - config: InstallConfigResolvedX, -) { - const depShimsRootPath = await Deno.makeTempDir({ - dir: scx.tmpPath, - prefix: `shims_${installId}`, - }); - - // readies all the exports of the port's deps including - // shims for their exports - const totalDepArts = await getShimmedDepArts( - scx, - depShimsRootPath, - await Promise.all( - manifest.buildDeps?.map( - async (dep) => { - const depConfig = getDepConfig(allowedDeps, manifest, config, dep); - // we not only resolve but install the dep here - const { installId } = await resolveAndInstall( - scx, - allowedDeps, - depConfig.manifest, - depConfig.config, - ); - return [installId, dep.name]; - }, - ) ?? [], - ), - ); - - const stageArgs = { - installId, - installPath: std_path.resolve(scx.installsPath, installId), - downloadPath: std_path.resolve(scx.downloadsPath, installId), - tmpPath: scx.tmpPath, - config: config, - manifest, - depArts: totalDepArts, - }; - - const dbRow = { - installId, - conf: config, - manifest, - }; - let downloadArts; - - try { - downloadArts = await doDownloadStage({ - ...stageArgs, - }); - } catch (err) { - throw new Error(`error downloading ${installId}`, { cause: err }); - } - await scx.db.val.set(installId, { - ...dbRow, - progress: "downloaded", - downloadArts, - }); - - let installArtifacts; - try { - installArtifacts = await doInstallStage( - { - ...stageArgs, - ...downloadArts, - }, - ); - } catch (err) { - throw new Error(`error installing ${installId}`, { cause: err }); - } - await scx.db.val.set(installId, { - ...dbRow, - progress: "installed", - downloadArts, - installArts: installArtifacts, - }); - await $.removeIfExists(depShimsRootPath); -} diff --git a/tests/ports.ts b/tests/ports.ts index c8a0208..ef066b3 100644 --- a/tests/ports.ts +++ b/tests/ports.ts @@ -12,8 +12,6 @@ type CustomE2eTestCase = Omit & { secureConf?: FileArgs; }; -// FIXME: where did the asdf test go? - // order tests by download size to make failed runs less expensive const cases: CustomE2eTestCase[] = [ // 0 megs diff --git a/tests/portsOutdated.ts b/tests/portsOutdated.ts deleted file mode 100644 index 8473680..0000000 --- a/tests/portsOutdated.ts +++ /dev/null @@ -1,59 +0,0 @@ -import "../src/deno_utils/setup_logger.ts"; -import { E2eTestCase, genTsGhjkFile, harness } from "./utils.ts"; -import * as ports from "../ports/mod.ts"; -import type { InstallConfigFat } from "../src/sys_deno/ports/types.ts"; -import type { FileArgs } from "../src/ghjk_ts/mod.ts"; - -type CustomE2eTestCase = Omit & { - ePoint: string; - installConf: InstallConfigFat | InstallConfigFat[]; - secureConf?: FileArgs; -}; - -// FIXME: -const cases: CustomE2eTestCase[] = [ - { - name: "command", - installConf: [ - ports.jq_ghrel({ version: "jq-1.7" }), - ], - ePoint: `ghjk p outdated`, - secureConf: { - enableRuntimes: true, - }, - }, - { - name: "update_all", - installConf: [ - ports.jq_ghrel({ version: "jq-1.7" }), - ports.protoc({ version: "v28.2" }), - ], - ePoint: `ghjk p outdated --update-all`, - secureConf: { - enableRuntimes: true, - }, - }, -]; - -harness(cases.map((testCase) => ({ - ...testCase, - fs: { - "ghjk.ts": genTsGhjkFile( - { - secureConf: { - ...testCase.secureConf, - installs: Array.isArray(testCase.installConf) - ? testCase.installConf - : [testCase.installConf], - }, - }, - ), - }, - ePoints: [ - ...["bash -c", "fish -c", "zsh -c"].map((sh) => ({ - cmd: [...`env ${sh}`.split(" "), `"${testCase.ePoint}"`], - })), - ], - name: `portsOutdated/${testCase.name}`, - timeout_ms: 10 * 60 * 1000, -}))); diff --git a/tools/check.ts b/tools/check.ts index ddc8409..7062f84 100755 --- a/tools/check.ts +++ b/tools/check.ts @@ -21,4 +21,4 @@ const files = (await Array.fromAsync( ), )).map((ref) => ref.path.toString()); -await $`${Deno.env.get("DENO_EXEC_PATH") ?? "deno"} check ${files}`; +await $`bash -c "xargs deno check"`.stdinText(files.join(" "));