Add general CLI utilities and expand tests (#2)

* Add general CLI utilities and expand tests

* remove outdated todo

* prefer forEach for test iteration

Co-authored-by: Philippe Rivière <[email protected]>

* check perl version

---------

Co-authored-by: Philippe Rivière <[email protected]>

Dockerfile

@@ -18,6 +18,7 @@ RUN --mount=type=cache,target=/var/cache/apt,id=framework-runtime-node \
     apt update \
     && apt install -y --no-install-recommends nodejs \
     && npm install --global yarn
+RUN npm install --global svgo
 
 # == python ======================
 FROM base AS python
@@ -59,6 +60,7 @@ RUN cd $(mktemp -d); \
 # == rust ========================
 FROM base AS rust
 # Based on https://github.com/rust-lang/docker-rust/blob/c8d1e4f5c563dacb16b2aadf827f1be3ff3ac25b/1.77.0/bookworm/Dockerfile
+# Install rustup, and then use that to install Rust
 RUN set -eux; \
     dpkgArch="$(dpkg --print-architecture)"; \
     case "${dpkgArch##*-}" in \
@@ -73,10 +75,45 @@ RUN set -eux; \
     ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
     rm rustup-init; \
     chmod -R a+w $RUSTUP_HOME $CARGO_HOME;
-RUN cargo install rust-script
+# Install cargo-binstall, which looks for precompiled binaries of libraries instead of building them here
+RUN set -eux; \
+    dpkgArch="$(dpkg --print-architecture)"; \
+    case "${dpkgArch##*-}" in \
+        amd64) binstallArch='x86_64-unknown-linux-gnu' ;; \
+        arm64) binstallArch='aarch64-unknown-linux-gnu' ;; \
+        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
+    esac; \
+    url="https://github.com/cargo-bins/cargo-binstall/releases/download/v1.6.4/cargo-binstall-${binstallArch}.tgz"; \
+    curl -L --proto '=https' --tlsv1.2 -sSf "$url" | tar -xvzf -; \
+    ./cargo-binstall -y --force cargo-binstall
+# rust-script is what Framework uses to run Rust data loaders
+RUN cargo binstall -y --force rust-script
+# all the apache arrow-tools
+RUN cargo binstall -y --force csv2arrow csv2parquet json2arrow json2parquet 
+
+# == general-cli =================
+FROM base AS general-cli
+RUN --mount=type=cache,target=/var/cache/apt,id=framework-runtime-general-cli \
+    set -eux; \
+    apt update; \
+    apt install -y --no-install-recommends \
+        bind9-dnsutils \
+        csvkit \
+        iputils-ping \
+        iputils-tracepath \
+        jq \
+        nano \
+        netcat-openbsd \
+        openssl \
+        optipng \
+        ripgrep \
+        silversearcher-ag \
+        vim \
+        zstd
 
 # == runtime =====================
 FROM base AS runtime
+COPY --from=general-cli . .
 COPY --from=node . .
 COPY --from=python . .
 COPY --from=r . .

bin/test.ts

@@ -0,0 +1,42 @@
+import { spawn } from "node:child_process";
+import { glob } from "glob";
+import { IMAGE_TAG, StringStream } from "../tests/index.ts";
+import { dirname } from "node:path";
+import { run as runTests } from "node:test";
+import { spec } from "node:test/reporters";
+
+export async function buildTestImage() {
+  console.log("building image...");
+  let stdio = new StringStream();
+  let process = spawn("docker", ["buildx", "build", "-t", IMAGE_TAG, ".."], {
+    cwd: import.meta.dirname,
+    stdio: "pipe",
+  });
+  process.stdout.pipe(stdio);
+  process.stderr.pipe(stdio);
+
+  await new Promise((resolve, reject) => {
+    process.on("close", (code) => {
+      if (code !== 0)
+        reject(
+          new Error(
+            `docker buildx build failed with code ${code}\n${stdio.string}`
+          )
+        );
+      resolve(undefined);
+    });
+    process.on("error", reject);
+  });
+}
+
+const files = await glob(["tests/**/*.test.ts"], {
+  cwd: dirname(import.meta.dirname),
+});
+
+await buildTestImage();
+runTests({ files, concurrency: true })
+  .on("test:fail", () => {
+    process.exitCode = 1;
+  })
+  .compose(new spec())
+  .pipe(process.stdout);

package.json

@@ -6,12 +6,14 @@
   "type": "module",
   "scripts": {
     "build": "docker build . -t observablehq/framework-runtime:latest",
-    "test": "node --import tsx/esm --test ./test.ts"
+    "test": "yarn tsx bin/test.ts"
   },
   "packageManager": "[email protected]",
   "devDependencies": {
+    "@types/dockerode": "^3.3.28",
     "@types/node": "^20",
     "dockerode": "^4.0.2",
+    "glob": "^10.3.12",
     "semver": "^7.6.0",
     "tsx": "^4.7.1",
     "typescript": "^5.4.3"

tests/archives.test.ts

@@ -0,0 +1,11 @@
+import { binaryOnPathTest } from "./index.ts";
+
+const archiveTools = [
+  { binary: "bzip2" },
+  { binary: "gzip" },
+  { binary: "tar" },
+  { binary: "zip" },
+  { binary: "zstd" },
+];
+
+archiveTools.forEach(binaryOnPathTest);

tests/data-manip.test.ts

@@ -0,0 +1,15 @@
+import { binaryOnPathTest, binaryVersionTest } from "./index.ts";
+
+const dataManipTools = [
+  { binary: "jq" },
+  { binary: "in2csv", name: "csvkit" },
+  { binary: "csv2parquet" },
+];
+
+dataManipTools.forEach(binaryOnPathTest);
+
+binaryVersionTest({
+  binary: "duckdb",
+  semver: "^0.10.1",
+  extract: /^v(.*) [0-9a-f]*$/,
+});

tests/dataloader-languages.test.ts

@@ -0,0 +1,35 @@
+import { binaryVersionTest } from "./index.ts";
+
+const dataLoaderLanguages = [
+  { binary: "node", semver: "^20" },
+  { binary: "npm", semver: "^10.5" },
+  { binary: "yarn", semver: "^1.22" },
+  {
+    binary: "python3",
+    semver: "^3.11",
+    prefix: "Python",
+  },
+  {
+    binary: "Rscript",
+    semver: "^4.3",
+    extract: /^Rscript \(R\) version ([^\s]+)/,
+  },
+  {
+    name: "Rust",
+    binary: "cargo",
+    semver: "^1.77",
+    extract: /^cargo ([\d.]+)/,
+  },
+  {
+    binary: "rust-script",
+    semver: "^0.34",
+    prefix: "rust-script",
+  },
+  {
+    binary: "perl",
+    semver: "^5.36",
+    extract: /^This is perl 5,[^(]* \(([^)]+)\)/,
+  },
+];
+
+dataLoaderLanguages.forEach(binaryVersionTest);

tests/general-cli.test.ts

@@ -0,0 +1,13 @@
+import { binaryOnPathTest } from "./index.ts";
+
+const generalCliTools: { binary: string }[] = [
+  { binary: "md5sum" },
+  { binary: "sha256sum" },
+  { binary: "sha512sum" },
+  { binary: "shasum" },
+  { binary: "top" },
+  { binary: "uptime" },
+  { binary: "vmstat" },
+];
+
+generalCliTools.forEach(binaryOnPathTest);

tests/images.test.ts

@@ -0,0 +1,9 @@
+import { binaryOnPathTest } from "./index.ts";
+
+const imageTools = [
+  { binary: "svgo" },
+  { binary: "optipng" },
+  { binary: "convert", name: "imagemagick" },
+];
+
+imageTools.forEach(binaryOnPathTest);