topics: implement Topic Update Manifest data generation and uploading

.github/workflows/known_hosts

@@ -0,0 +1,2 @@
+# repo.aosc.io:22 SSH-2.0-OpenSSH_9.8
+repo.aosc.io ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAyJWDN3wR1Sa+2jvlafyZirKoQ+TiE0CaymGn2L7fi5

.github/workflows/tum-ci.yml

@@ -0,0 +1,60 @@
+name: Generate Update Manifests
+
+on:
+  push:
+    paths:
+    - "topics/*.toml"
+    - ".github/workflows/tum-*"
+  pull_request:
+    paths:
+    - "topics/*.toml"
+    - ".github/workflows/tum-*"
+  workflow_dispatch: {}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Install npm dependencies
+      run: npm install toml @cfworker/json-schema
+    - name: Generate manifests
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const {generateTopicUpdateData} = require('${{ github.workspace }}/.github/workflows/tum-process.js');
+          const topic = context.ref?.replace('refs/heads/', '');
+          generateTopicUpdateData(require, topic, '/tmp/dists/');
+    - name: Setup SSH private key
+      env:
+        KEY: ${{ secrets.KEY }}
+      run: |
+        mkdir -p ~/.ssh/
+        chmod 0700 ~/.ssh/
+        echo "$KEY" > ~/.ssh/id_ed25519
+        cp .github/workflows/known_hosts ~/.ssh/known_hosts
+        chmod 0600 ~/.ssh/id_ed25519 ~/.ssh/known_hosts
+    - name: Upload topic manifests
+      shell: bash
+      run: |
+        rsync \
+            -vr \
+            -e "ssh \
+                -o IdentityFile=$HOME/.ssh/id_ed25519 \
+                -o UserKnownHostsFile=$HOME/.ssh/known_hosts" \
+            /tmp/dists/ \
+            ${USER}@repo.aosc.io:/var/cache/p-vector/extra-dists/
+        ssh \
+            -v \
+            -o IdentityFile=~/.ssh/id_ed25519 \
+            -o UserKnownHostsFile=~/.ssh/known_hosts \
+            ${USER}@repo.aosc.io \
+            touch /mirror/.updated
+      env:
+        USER: ${{ secrets.USER }}
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: manifests
+        path: /tmp/dists/

.github/workflows/tum-process.js

@@ -0,0 +1,97 @@
+const fs = require("node:fs");
+
+/**
+ * @typedef TopicConfig
+ * @type { { name: { [key: string]: string }, security: boolean, caution?: { [key: string]: string }, topics?: string[], packages?: { [key: string]: string | boolean | null } } }
+ */
+
+/**
+ * Translates a TOML topic configuration into a structured JSON object.
+ * @param {import("toml")} toml
+ * @param {import("@cfworker/json-schema").Validator} schemaValidator
+ * @param {string} content
+ * @returns
+ */
+function translateTopic(toml, schemaValidator, filePath) {
+  /**
+   * @type { TopicConfig }
+   */
+  console.info(`Reading ${filePath}`);
+  const content = fs.readFileSync(filePath, "utf8");
+  let topic;
+
+  try {
+    topic = toml.parse(content);
+  } catch (error) {
+    console.error(`Error parsing TOML file at ${filePath}:`, error);
+    console.log(`::error file=${filePath}::Error parsing TOML file::${error}`);
+    throw error;
+  }
+
+  if (schemaValidator.validate(topic).valid) {
+    console.log("TOML file parsed successfully.");
+  } else {
+    const errors = schemaValidator.validate(topic).errors;
+    console.error("Invalid TOML:", errors);
+    console.log(
+      `::error file=${filePath}::${errors.map((e) => e.error).join(" ")}`
+    );
+    throw new Error("Invalid TOML");
+  }
+  // rewrite package versions
+  if (topic.packages) {
+    Object.keys(topic.packages).forEach((pkg) => {
+      if (!topic.packages[pkg]) {
+        topic.packages[pkg] = null;
+      }
+    });
+  }
+
+  topic.type = topic.packages ? "conventional" : "cumulative";
+  return topic;
+}
+
+/**
+ *
+ * @param {(name: string) => import(name)} require
+ * @param {string | null} topic
+ * @param {string} outputPath
+ */
+function generateTopicUpdateData(require, topic, outputPath) {
+  const toml = require("toml");
+  const validator = require("@cfworker/json-schema");
+  const schema = require("./topics/tum.schema.json");
+  const schemaValidator = new validator.Validator(schema);
+
+  /**
+   * @type { { [key: string]: { type: "conventional" | "cumulative", name: { [key: string]: string }, security: boolean, caution?: { [key: string]: string }, topics?: string[], packages?: { [key: string]: string | null } } } }
+   */
+  let result = {};
+  if (!topic) {
+    console.error("No topic specified. Use 'stable' or a topic name.");
+    return;
+  }
+  console.info(`Generating updates for topic "${topic}"`);
+  if (topic === "stable") {
+    fs.readdirSync("topics").forEach((file) => {
+      if (!file.endsWith(".toml")) {
+        return;
+      }
+      const filePath = `topics/${file}`;
+      const topicName = file.replace(/\.toml$/, "");
+      result[topicName] = translateTopic(toml, schemaValidator, filePath);
+    });
+  } else {
+    result[topic] = translateTopic(
+      toml,
+      schemaValidator,
+      `topics/${topic}.toml`
+    );
+  }
+
+  const outputDirPath = `${outputPath}/${topic}`;
+  fs.mkdirSync(outputDirPath, { recursive: true });
+  fs.writeFileSync(`${outputDirPath}/updates.json`, JSON.stringify(result));
+}
+
+module.exports = { generateTopicUpdateData };

.gitignore

@@ -4,3 +4,4 @@ repo-spec/last_db_update
 .*.swp
 *.save
 *.acbs-ckpt
+/node_modules

topics/tum.schema.json

@@ -0,0 +1,68 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/AOSC-Dev/aosc-os-abbs/blob/stable/topics/tum.schema.json",
+  "title": "",
+  "type": "object",
+  "properties": {
+    "security": {
+      "type": "boolean",
+      "description": "This topic contains security updates"
+    },
+    "name": {
+      "type": "object",
+      "properties": {
+        "default": {
+          "type": "string",
+          "description": "The name of the topic in English"
+        }
+      },
+      "patternProperties": {
+        "^[a-z]+_[A-Z]+": {
+          "type": "string",
+          "description": "The name of the topic in the language specified by the key"
+        }
+      },
+      "required": ["default"]
+    },
+    "caution": {
+      "type": "object",
+      "properties": {
+        "default": {
+          "type": "string",
+          "description": "PSA message to alert users about potential issues in English"
+        }
+      },
+      "patternProperties": {
+        "^[a-z]+_[A-Z]+": {
+          "type": "string",
+          "description": "PSA message to alert users about potential issues in the language specified by the key"
+        }
+      },
+      "required": ["default"]
+    },
+    "packages": {
+      "type": "object",
+      "patternProperties": {
+        "^[a-z0-9][a-z0-9+-.]+": {
+          "type": ["string", "boolean"],
+          "description": "Package version to update to or false to note that the package will be removed"
+        }
+      },
+      "minProperties": 1
+    },
+    "topics": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "description": "The name of the topic to be included in this cummulative update"
+      },
+      "minItems": 1
+    }
+  },
+  "oneOf": [
+    {"required": ["packages"]},
+    {"required": ["topics"]}
+  ],
+  "additionalProperties": false,
+  "required": ["security", "name"]
+}