index pkgs (#11)

.github/scripts/gen-index.html.ts

@@ -0,0 +1,37 @@
+#!/usr/bin/env -S pkgx deno run --allow-read --allow-write=./out
+
+console.log('hio')
+
+interface Pkg {
+  name?: string
+  project: string
+  description: string
+}
+
+const pkgs: Pkg[] = JSON.parse(Deno.readTextFileSync("./out/index.json"));
+
+if (pkgs.length <= 0) {
+  throw new Error("Empty pkgs!")
+}
+
+for (const pkg of pkgs) {
+  Deno.mkdirSync(`./out/${pkg.project}`, {recursive: true});
+
+  const title = `${pkg.name || pkg.project} — pkgx`;
+
+  let txt = Deno.readTextFileSync('./out/index.html');
+  txt = replace(txt, 'title', title);
+  txt = replace(txt, 'description', pkg.description);
+  txt = replace(txt, 'image',  `https://gui.tea.xyz/prod/${pkg.project}/1024x1024.webp`);
+  txt = replace(txt, 'url', `https://pkgx.dev/pkgs/${pkg.project}/`);
+
+  txt = txt.replace(/<title>.*<\/title>/, `<title>${title}</title>`);
+
+  Deno.writeTextFileSync(`./out/${pkg.project}/index.html`, txt);
+
+  console.log(`./out/${pkg.project}/index.html`)
+}
+
+function replace(txt: string, attr: string, value: string) {
+  return txt.replace(new RegExp(`<meta property="og:${attr}" content=".*">`), `<meta property="og:${attr}" content="${value}">`)
+}

.github/scripts/gen-index.json.ts

@@ -0,0 +1,134 @@
+#!/usr/bin/env -S pkgx deno run -A
+
+import * as fs from "node:fs"
+
+interface Package {
+  project: string,
+  birthtime: Date,
+  name?: string
+  description?: string
+  labels?: string[]
+}
+
+console.error('HI')
+
+export async function getKettleRemoteMetadata() {
+  const headers = { Authorization: 'public' }
+  const rsp = await fetch(`https://app.pkgx.dev/v1/packages/`, {headers})
+  const foo = await rsp.json() as {project: string, short_description: string}[]
+  return foo.reduce((acc, {project, short_description}) => {
+    acc[project] = short_description
+    return acc
+  }, {} as Record<string, string>)
+}
+
+const descriptions = await getKettleRemoteMetadata();
+
+async function getPackageYmlCreationDates(): Promise<Package[]> {
+  const cmdString = "git log --pretty=format:'%H %aI' --name-only --diff-filter=A -- 'projects/**/package.yml'";
+  const process = Deno.run({
+    cmd: ["bash", "-c", cmdString],
+    stdout: "piped",
+  });
+
+  const output = new TextDecoder().decode(await process.output());
+  await process.status();
+  process.close();
+
+  const lines = output.trim().split('\n');
+  const rv: Package[] = []
+  let currentCommitDate: string | null = null;
+
+  for (const line of lines) {
+    if (line.includes(' ')) {  // Detect lines with commit hash and date
+      currentCommitDate = line.split(' ')[1];
+    } else if (line.endsWith('package.yml')) {
+      const project = line.slice(9, -12)
+
+      if (!fs.existsSync(line)) {
+        // the file used to exist but has been deleted
+        console.warn("skipping yanked: ", project)
+        continue
+      }
+
+      const birthtime = new Date(currentCommitDate!)
+      const name = await get_name(line, project)
+
+      let description: string | undefined = descriptions[project]?.trim()
+      if (!description) description = undefined
+
+      let labels: string[] | undefined = [...await get_labels(line)]
+      if (labels.length == 0) labels = undefined
+
+      rv.push({ project, birthtime, name, description, labels })
+    }
+  }
+
+  return rv;
+}
+
+const pkgs = await getPackageYmlCreationDates();
+
+console.error({pkgs})
+
+// sort by birthtime
+pkgs.sort((a, b) => b.birthtime.getTime() - a.birthtime.getTime());
+
+console.log(JSON.stringify(pkgs, null, 2));
+
+//////////////////////////////////////////////////////
+import { parse } from "https://deno.land/[email protected]/yaml/mod.ts";
+import { isArray } from "https://deno.land/x/[email protected]/src/index.ts";
+import get_pkg_name from "../../src/utils/pkg-name.ts";
+
+async function get_name(path: string, project: string): Promise<string | undefined> {
+  const txt = await Deno.readTextFileSync(path)
+  const yml = await parse(txt) as Record<string, any>
+  if (yml['display-name']) {
+    return yml['display-name']
+  } else if (isArray(yml.provides) && yml.provides.length == 1) {
+    return yml.provides[0].slice(4)
+  } else {
+    return get_pkg_name(project)
+  }
+}
+
+import { parse_pkgs_node } from "https://deno.land/x/[email protected]/src/hooks/usePantry.ts"
+
+async function get_labels(path: string) {
+  const txt = await Deno.readTextFileSync(path)
+  const yml = await parse(txt) as Record<string, any>
+  const deps = parse_pkgs_node(yml.dependencies) //NOTE will ignore other platforms than linux, but should be fine
+
+  const labels = new Set<string>(deps.compact(({ project }) => {
+    switch (project) {
+    case 'nodejs.org':
+    case 'npmjs.com':
+      return 'node'
+    case 'python.org':
+    case 'pip.pypa.io':
+      return 'python'
+    case 'ruby-lang.org':
+    case 'rubygems.org':
+      return 'ruby'
+    case 'rust-lang.org':
+    case 'rust-lang.org/cargo':
+      return 'rust'
+    }
+  }))
+
+  console.error(path)
+
+  if (yml.build?.dependencies) for (const dep of parse_pkgs_node(yml.build.dependencies)) {
+    switch (dep.project) {
+    case 'rust-lang.org':
+    case 'rust-lang.org/cargo':
+      labels.add('rust')
+      break
+    case 'go.dev':
+      labels.add('go')
+    }
+  }
+
+  return labels
+}

.github/workflows/cd.yml

@@ -110,3 +110,8 @@ jobs:
       - uses: pkgxdev/setup@v1
       - run: pkgx hugo
       - run: pkgx hugo deploy
+
+  indexer:
+    needs: publish
+    uses: ./.github/workflows/indexer.yml
+    secrets: inherit

.github/workflows/indexer.yml

@@ -0,0 +1,62 @@
+name: indexer
+
+# we have to generate an index for each pkg so that the index.html that is
+# served has the correct metadata
+
+#TODO a lambda would be better because we copy the index.html from the bucket
+# root and modify it then push that back so it could easily get out of sync
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  schedule:
+    - cron: "13,26,39,52 3,9,15,21 * * *"
+  pull_request:
+    branches: main
+    paths:
+      - .github/scripts/indexer.ts
+      - .github/workflows/indexer.yml
+
+concurrency:
+  group: indexer/${{ github.ref || 'main' }}
+  cancel-in-progress: true
+
+jobs:
+  indexer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/checkout@v4
+        with:
+          repository: pkgxdev/pantry
+          path: pantry
+          fetch-depth: 0  # needed to get git metadata for ordering purposes
+
+      - uses: pkgxdev/setup@v2
+
+      - uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.WWW_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.WWW_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+
+      - run: aws s3 sync s3://www.pkgx.dev/pkgs/ ./out
+      - run: aws s3 cp s3://www.pkgx.dev/index.html ./out/index.html
+
+      - run: ../.github/scripts/gen-index.json.ts > ../out/index.json
+        working-directory: pantry
+
+      - run: cat ./out/index.html
+      - run: .github/scripts/gen-index.html.ts
+
+      - run: aws s3 sync
+              out/ s3://www.pkgx.dev/pkgs/
+              --size-only
+              --metadata-directive REPLACE
+              --cache-control no-cache,must-revalidate
+
+      - run: aws cloudfront
+               create-invalidation
+               --distribution-id E15VQ3SI584CSG
+               --paths /pkgs /pkgs/ /pkgs/*

.gitignore

@@ -1,2 +1,3 @@
 node_modules
-/dist
+/dist
+/out

index.html

@@ -2,6 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="utf-8" />
+    <title>pkgx — Run Anything</title>
     <link rel="icon" type="image/svg+xml" href="/favicon.svg" purpose="any maskable" />
     <link rel="icon" type="image/png" sizes="1024x1024" href="/favicon-1024.png" />
     <link rel="icon" type="image/png" sizes="180x180" href="/favicon-180.png" />
@@ -11,11 +12,11 @@
     <link rel="preconnect" href="https://fonts.googleapis.com" />
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
     <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;600;700&display=swap" />
-    <title>pkgx — Run Anything</title>
+    <meta property="og:type" content="website" />
     <meta property="og:title" content="Run Anything">
     <meta property="og:description" content="pkgx is a blazingly fast, standalone, cross‐platform binary that runs anything from the creator of Homebrew">
     <meta property="og:image" content="/og.jpg">
-    <meta property="og:url" content="https://pkgx.sh">
+    <meta property="og:url" content="https://pkgx.%VITE_PKGX_TLD%">
   </head>
   <body>
     <div id="root"></div>

src/pkgx.dev/PackageShowcase.tsx

@@ -5,7 +5,7 @@ import { Alert, Skeleton } from "@mui/material";
 
 export default function showcase() {
   const {loading, error, value: pkgs} = useAsync(async () => {
-    const rsp = await fetch('https://pkgxdev.github.io/pantry/pkgs.json')
+    const rsp = await fetch('https://pkgx.dev/pkgs/index.json')
     return await rsp.json()
   })