diff --git a/.github/scripts/gen-index.html.ts b/.github/scripts/gen-index.html.ts new file mode 100755 index 0000000..75db11d --- /dev/null +++ b/.github/scripts/gen-index.html.ts @@ -0,0 +1,37 @@ +#!/usr/bin/env -S pkgx deno run --allow-read --allow-write=./out + +console.log('hio') + +interface Pkg { + name?: string + project: string + description: string +} + +const pkgs: Pkg[] = JSON.parse(Deno.readTextFileSync("./out/index.json")); + +if (pkgs.length <= 0) { + throw new Error("Empty pkgs!") +} + +for (const pkg of pkgs) { + Deno.mkdirSync(`./out/${pkg.project}`, {recursive: true}); + + const title = `${pkg.name || pkg.project} — pkgx`; + + let txt = Deno.readTextFileSync('./out/index.html'); + txt = replace(txt, 'title', title); + txt = replace(txt, 'description', pkg.description); + txt = replace(txt, 'image', `https://gui.tea.xyz/prod/${pkg.project}/1024x1024.webp`); + txt = replace(txt, 'url', `https://pkgx.dev/pkgs/${pkg.project}/`); + + txt = txt.replace(/.*<\/title>/, `<title>${title}`); + + Deno.writeTextFileSync(`./out/${pkg.project}/index.html`, txt); + + console.log(`./out/${pkg.project}/index.html`) +} + +function replace(txt: string, attr: string, value: string) { + return txt.replace(new RegExp(``), ``) +} diff --git a/.github/scripts/gen-index.json.ts b/.github/scripts/gen-index.json.ts new file mode 100755 index 0000000..7e74149 --- /dev/null +++ b/.github/scripts/gen-index.json.ts @@ -0,0 +1,134 @@ +#!/usr/bin/env -S pkgx deno run -A + +import * as fs from "node:fs" + +interface Package { + project: string, + birthtime: Date, + name?: string + description?: string + labels?: string[] +} + +console.error('HI') + +export async function getKettleRemoteMetadata() { + const headers = { Authorization: 'public' } + const rsp = await fetch(`https://app.pkgx.dev/v1/packages/`, {headers}) + const foo = await rsp.json() as {project: string, short_description: string}[] + return foo.reduce((acc, {project, short_description}) => { + acc[project] = short_description + return acc + }, {} as Record) +} + +const descriptions = await getKettleRemoteMetadata(); + +async function getPackageYmlCreationDates(): Promise { + const cmdString = "git log --pretty=format:'%H %aI' --name-only --diff-filter=A -- 'projects/**/package.yml'"; + const process = Deno.run({ + cmd: ["bash", "-c", cmdString], + stdout: "piped", + }); + + const output = new TextDecoder().decode(await process.output()); + await process.status(); + process.close(); + + const lines = output.trim().split('\n'); + const rv: Package[] = [] + let currentCommitDate: string | null = null; + + for (const line of lines) { + if (line.includes(' ')) { // Detect lines with commit hash and date + currentCommitDate = line.split(' ')[1]; + } else if (line.endsWith('package.yml')) { + const project = line.slice(9, -12) + + if (!fs.existsSync(line)) { + // the file used to exist but has been deleted + console.warn("skipping yanked: ", project) + continue + } + + const birthtime = new Date(currentCommitDate!) + const name = await get_name(line, project) + + let description: string | undefined = descriptions[project]?.trim() + if (!description) description = undefined + + let labels: string[] | undefined = [...await get_labels(line)] + if (labels.length == 0) labels = undefined + + rv.push({ project, birthtime, name, description, labels }) + } + } + + return rv; +} + +const pkgs = await getPackageYmlCreationDates(); + +console.error({pkgs}) + +// sort by birthtime +pkgs.sort((a, b) => b.birthtime.getTime() - a.birthtime.getTime()); + +console.log(JSON.stringify(pkgs, null, 2)); + +////////////////////////////////////////////////////// +import { parse } from "https://deno.land/std@0.204.0/yaml/mod.ts"; +import { isArray } from "https://deno.land/x/is_what@v4.1.15/src/index.ts"; +import get_pkg_name from "../../src/utils/pkg-name.ts"; + +async function get_name(path: string, project: string): Promise { + const txt = await Deno.readTextFileSync(path) + const yml = await parse(txt) as Record + if (yml['display-name']) { + return yml['display-name'] + } else if (isArray(yml.provides) && yml.provides.length == 1) { + return yml.provides[0].slice(4) + } else { + return get_pkg_name(project) + } +} + +import { parse_pkgs_node } from "https://deno.land/x/libpkgx@v0.15.1/src/hooks/usePantry.ts" + +async function get_labels(path: string) { + const txt = await Deno.readTextFileSync(path) + const yml = await parse(txt) as Record + const deps = parse_pkgs_node(yml.dependencies) //NOTE will ignore other platforms than linux, but should be fine + + const labels = new Set(deps.compact(({ project }) => { + switch (project) { + case 'nodejs.org': + case 'npmjs.com': + return 'node' + case 'python.org': + case 'pip.pypa.io': + return 'python' + case 'ruby-lang.org': + case 'rubygems.org': + return 'ruby' + case 'rust-lang.org': + case 'rust-lang.org/cargo': + return 'rust' + } + })) + + console.error(path) + + if (yml.build?.dependencies) for (const dep of parse_pkgs_node(yml.build.dependencies)) { + switch (dep.project) { + case 'rust-lang.org': + case 'rust-lang.org/cargo': + labels.add('rust') + break + case 'go.dev': + labels.add('go') + } + } + + return labels +} diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 9962663..5ee6232 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -110,3 +110,8 @@ jobs: - uses: pkgxdev/setup@v1 - run: pkgx hugo - run: pkgx hugo deploy + + indexer: + needs: publish + uses: ./.github/workflows/indexer.yml + secrets: inherit diff --git a/.github/workflows/indexer.yml b/.github/workflows/indexer.yml new file mode 100644 index 0000000..1469252 --- /dev/null +++ b/.github/workflows/indexer.yml @@ -0,0 +1,62 @@ +name: indexer + +# we have to generate an index for each pkg so that the index.html that is +# served has the correct metadata + +#TODO a lambda would be better because we copy the index.html from the bucket +# root and modify it then push that back so it could easily get out of sync + +on: + workflow_call: + workflow_dispatch: + schedule: + - cron: "13,26,39,52 3,9,15,21 * * *" + pull_request: + branches: main + paths: + - .github/scripts/indexer.ts + - .github/workflows/indexer.yml + +concurrency: + group: indexer/${{ github.ref || 'main' }} + cancel-in-progress: true + +jobs: + indexer: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/checkout@v4 + with: + repository: pkgxdev/pantry + path: pantry + fetch-depth: 0 # needed to get git metadata for ordering purposes + + - uses: pkgxdev/setup@v2 + + - uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.WWW_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.WWW_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + + - run: aws s3 sync s3://www.pkgx.dev/pkgs/ ./out + - run: aws s3 cp s3://www.pkgx.dev/index.html ./out/index.html + + - run: ../.github/scripts/gen-index.json.ts > ../out/index.json + working-directory: pantry + + - run: cat ./out/index.html + - run: .github/scripts/gen-index.html.ts + + - run: aws s3 sync + out/ s3://www.pkgx.dev/pkgs/ + --size-only + --metadata-directive REPLACE + --cache-control no-cache,must-revalidate + + - run: aws cloudfront + create-invalidation + --distribution-id E15VQ3SI584CSG + --paths /pkgs /pkgs/ /pkgs/* diff --git a/.gitignore b/.gitignore index 0771922..72637b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules -/dist \ No newline at end of file +/dist +/out diff --git a/index.html b/index.html index 66bb00c..a948afc 100644 --- a/index.html +++ b/index.html @@ -1,21 +1,21 @@ + pkgx — Run Anything + + + + + + - - - pkgx — Run Anything - - - -
diff --git a/src/pkgx.dev/PackageShowcase.tsx b/src/pkgx.dev/PackageShowcase.tsx index 1fd7741..4df1094 100644 --- a/src/pkgx.dev/PackageShowcase.tsx +++ b/src/pkgx.dev/PackageShowcase.tsx @@ -5,7 +5,7 @@ import { Alert, Skeleton } from "@mui/material"; export default function showcase() { const {loading, error, value: pkgs} = useAsync(async () => { - const rsp = await fetch('https://pkgxdev.github.io/pantry/pkgs.json') + const rsp = await fetch('https://pkgx.dev/pkgs/index.json') return await rsp.json() })