osbuild: add aleph file stage

This is a set of patches from a currently a WIP PR from upstream
in osbuild/osbuild#1475

build.sh

@@ -184,6 +184,9 @@ patch_osbuild() {
         /usr/lib/coreos-assembler/0003-tools-osbuild-mpp-support-defining-multiple-image-la.patch \
         /usr/lib/coreos-assembler/0004-tools-osbuild-mpp-add-sector-size-support-for-image-.patch \
         /usr/lib/coreos-assembler/0005-tools-osbuild-mpp-set-part-ID-from-name-if-missing.patch   \
+        /usr/lib/coreos-assembler/0001-osbuild-util-ostree-convert-cli-to-return-the-comple.patch \
+        /usr/lib/coreos-assembler/0002-osbuild-util-ostree-optimize-deployment_path.patch         \
+        /usr/lib/coreos-assembler/0003-create-org.osbuild.ostree.aleph-stage.patch                \
             | patch -d /usr/lib/osbuild -p1
 
     # And then move the files back; supermin appliance creation will need it back

src/0001-osbuild-util-ostree-convert-cli-to-return-the-comple.patch

@@ -0,0 +1,39 @@
+From 57f13570023653f85d0920ea64ae3433216c68fc Mon Sep 17 00:00:00 2001
+From: Dusty Mabe <[email protected]>
+Date: Mon, 27 Nov 2023 16:14:43 -0500
+Subject: [PATCH 1/3] osbuild/util/ostree: convert cli to return the completed
+ process object
+
+And also set stdout=subprocess.PIPE. This will allow for callers to
+parse and use the output of the command, but has the side effect of
+meaning less gets printed to the screen during run.
+
+Co-authored-by: Luke Yang <[email protected]>
+---
+ osbuild/util/ostree.py | 10 +++++-----
+ 1 file changed, 5 insertions(+), 5 deletions(-)
+
+diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py
+index 43dd99c5..09c648fb 100644
+--- a/osbuild/util/ostree.py
++++ b/osbuild/util/ostree.py
+@@ -199,11 +199,11 @@ def cli(*args, _input=None, **kwargs):
+     """Thin wrapper for running the ostree CLI"""
+     args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()]
+     print("ostree " + " ".join(args), file=sys.stderr)
+-    subprocess.run(["ostree"] + args,
+-                   encoding="utf8",
+-                   stdout=sys.stderr,
+-                   input=_input,
+-                   check=True)
++    return subprocess.run(["ostree"] + args,
++                          encoding="utf8",
++                          stdout=subprocess.PIPE,
++                          input=_input,
++                          check=True)
+
+
+ def parse_input_commits(commits):
+-- 
+2.41.0
+

src/0002-osbuild-util-ostree-optimize-deployment_path.patch

@@ -0,0 +1,43 @@
+From 009813573024d4d42f37e5c1d3752e0c0e07bb06 Mon Sep 17 00:00:00 2001
+From: Dusty Mabe <[email protected]>
+Date: Mon, 27 Nov 2023 16:24:29 -0500
+Subject: [PATCH 2/3] osbuild/util/ostree: optimize deployment_path()
+
+When provisioning disk images there should really not be more than
+one deployment on a system. We don't really need any of the parameters
+here so let's make them optional and just find the deployment anyway.
+
+Co-authored-by: Luke Yang <[email protected]>
+---
+ osbuild/util/ostree.py | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py
+index 09c648fb..90821d5b 100644
+--- a/osbuild/util/ostree.py
++++ b/osbuild/util/ostree.py
+@@ -1,5 +1,6 @@
+ import collections
+ import contextlib
++import glob
+ import json
+ import os
+ import subprocess
+@@ -217,6 +218,14 @@ def parse_input_commits(commits):
+ def deployment_path(root: PathLike, osname: str, ref: str, serial: int):
+     """Return the path to a deployment given the parameters"""
+
++    if osname == "" and ref == "" and serial is None:
++        filenames = glob.glob(root + '/ostree/deploy/*/deploy/*.0', recursive=True)
++        if len(filenames) < 1:
++            raise ValueError("Could not find deployment")
++        if len(filenames) > 1:
++            raise ValueError("More than one deployment found")
++        return filenames[0]
++
+     base = os.path.join(root, "ostree")
+
+     repo = os.path.join(base, "repo")
+-- 
+2.41.0
+

src/0003-create-org.osbuild.ostree.aleph-stage.patch

@@ -0,0 +1,233 @@
+From f25a1718033dc6b2e51766c77febded06b5be978 Mon Sep 17 00:00:00 2001
+From: Luke Yang <[email protected]>
+Date: Wed, 25 Oct 2023 13:47:56 -0400
+Subject: [PATCH 3/3] create org.osbuild.ostree.aleph stage
+
+Similar to the aleph file created for builds of FCOS based on ostree
+commit inputs, this adds an aleph file that contains information about
+the initial deployment of data when the disk image was built
+
+A new stage is preferred here as both the org.osbuild.ostree.deploy
+and org.osbuild.ostree.deploy.container stages need an aleph file and
+use of the aleph file may depend on the project/product. For example,
+right now CoreOS is the only project that uses an aleph file, but others
+may want it in the future.
+---
+ osbuild/util/ostree.py          |  28 ++++++
+ stages/org.osbuild.ostree.aleph | 165 ++++++++++++++++++++++++++++++++
+ 2 files changed, 193 insertions(+)
+ create mode 100755 stages/org.osbuild.ostree.aleph
+
+diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py
+index 90821d5b..170c28c3 100644
+--- a/osbuild/util/ostree.py
++++ b/osbuild/util/ostree.py
+@@ -237,6 +237,34 @@ def deployment_path(root: PathLike, osname: str, ref: str, serial: int):
+     return sysroot
+
+
++def parse_origin(origin: PathLike):
++    """Parse the origin file and return the deployment type and imgref
++
++    Example container case: container-image-reference=ostree-remote-image:fedora:docker://quay.io/fedora/fedora-coreos:stable
++    Example ostree commit case: refspec=fedora:fedora/x86_64/coreos/stable
++    """
++    deploy_type = ""
++    imgref = ""
++    with open(origin, "r", encoding="utf8") as f:
++        for line in f:
++            separated_line = line.split("=")
++            if separated_line[0] == "container-image-reference":
++                deploy_type = "container"
++                imgref = separated_line[1].rstrip()
++                break
++            if separated_line[0] == "refspec":
++                deploy_type = "ostree_commit"
++                imgref = separated_line[1].rstrip()
++                break
++
++    if deploy_type == "":
++        raise ValueError("Could not find 'container-image-reference' or 'refspec' in origin file")
++    if imgref == "":
++        raise ValueError("Could not find imgref in origin file")
++
++    return deploy_type, imgref
++
++
+ class PasswdLike:
+     """Representation of a file with structure like /etc/passwd
+
+diff --git a/stages/org.osbuild.ostree.aleph b/stages/org.osbuild.ostree.aleph
+new file mode 100755
+index 00000000..b152a822
+--- /dev/null
++++ b/stages/org.osbuild.ostree.aleph
+@@ -0,0 +1,165 @@
++#!/usr/bin/python3
++"""
++Create aleph version file for the deployment.
++"""
++
++
++import json
++import os
++import sys
++
++import osbuild.api
++from osbuild.util import ostree
++
++CAPABILITIES = ["CAP_MAC_ADMIN"]
++ALEPH_FILENAME = ".aleph-version.json"
++COREOS_ALEPH_FILENAME = ".coreos-aleph-version.json"
++
++
++SCHEMA_2 = """
++"options": {
++  "additionalProperties": false,
++  "properties": {
++    "coreos_compat": {
++      "description": "boolean to allow for CoreOS aleph version backwards compatibility",
++      "type": "boolean"
++    },
++    "deployment": {
++      "additionalProperties": false,
++      "required": ["osname", "ref"],
++      "properties": {
++        "osname": {
++          "description": "Name of the stateroot to be used in the deployment",
++          "type": "string"
++        },
++        "ref": {
++          "description": "OStree ref to create and use for deployment",
++          "type": "string"
++        },
++        "serial": {
++          "description": "The deployment serial (usually '0')",
++          "type": "number",
++          "default": 0
++        }
++      }
++    }
++  }
++}
++"""
++
++
++def aleph_commit(tree, imgref):
++    extra_args = []
++    extra_args.append("--print-metadata-key=version")
++
++    aleph_version = ostree.cli("show", f"--repo={tree}/ostree/repo", imgref, *extra_args).stdout.rstrip().strip('\'')
++    aleph_ref = imgref
++    # get the commit by parsing the revision of the deployment
++    aleph_ostree_commit = ostree.rev_parse(tree + "/ostree/repo", imgref)
++
++    aleph_version_data = {
++        "osbuild-version": osbuild.__version__,
++        "version": aleph_version,
++        "ref": aleph_ref,
++        "ostree-commit": aleph_ostree_commit
++    }
++
++    return aleph_version_data
++
++
++def aleph_container(tree, imgref):
++    # extract the image name from the imgref
++    imgref_list = imgref.split(':')
++    if imgref_list[0] in ["ostree-remote-registry", "ostree-remote-image"]:
++        img_name = ':'.join(imgref_list[2:])
++    elif imgref_list[0] in ["ostree-image-signed", "ostree-unverified-registry"]:
++        img_name = ':'.join(imgref_list[1:])
++    else:
++        raise ValueError(f"Image ref {imgref} has unsupported type (supported: 'ostree-remote-registry', \
++                         'ostree-remote-image', 'ostree-image-signed', or 'ostree-unverified-registry')")
++
++    img_name = img_name.removeprefix('docker://')
++
++    extra_args = []
++    extra_args.append(f"--repo={tree}/ostree/repo")
++    extra_args.append(f"registry:{img_name}")
++
++    container_data_json = ostree.cli("container", "image", "metadata", *extra_args).stdout.rstrip()
++    container_data = json.loads(container_data_json)
++
++    extra_args.append("--config")
++    container_data_config_json = ostree.cli("container", "image", "metadata", *extra_args).stdout.rstrip()
++    container_data_config = json.loads(container_data_config_json)
++
++    aleph_digest = container_data['config']['digest']
++    aleph_ref = f"docker://{imgref}"
++    aleph_version = container_data_config['config']['Labels']['org.opencontainers.image.version']
++    aleph_container_image = container_data_config['config']['Labels']
++
++    aleph_version_data = {
++        "osbuild-version": osbuild.__version__,
++        "ref": aleph_ref,
++        "version": aleph_version,
++        "container-image": {
++            "image-name": img_name,
++            "image-digest": aleph_digest,
++            "image-labels": aleph_container_image
++        }
++    }
++
++    # the 'ostree.commit' label will be optional in the future so
++    # prevent hard failing if key is not found
++    aleph_ostree_commit = container_data_config['config']['Labels'].get('ostree.commit')
++    if aleph_ostree_commit is not None:
++        aleph_version_data["ostree-commit"] = aleph_ostree_commit
++
++    return aleph_version_data
++
++
++def construct_aleph_json(tree, origin):
++    deploy_type, imgref = ostree.parse_origin(origin)
++    data = {}
++    # null deploy_type and imgref error is caught in the parse_origin() function
++    if deploy_type == "container":
++        data = aleph_container(tree, imgref)
++    elif deploy_type == "ostree_commit":
++        data = aleph_commit(tree, imgref)
++    else:
++        raise ValueError("Unknown deployment type")
++
++    return data
++
++
++def main(tree, options):
++    coreos_compat = options.get("coreos_compat", False)
++    dep = options.get("deployment", None)
++    osname = ""
++    ref = ""
++    serial = None
++    origin = ""
++
++    # if deployment is specified, use it to find the origin file.
++    # otherwise, autodetect the only deployment found in the tree.
++    if dep is not None:
++        osname = dep.get("osname", "")
++        ref = dep.get("ref", "")
++        serial = dep.get("serial", 0)
++
++    origin = ostree.deployment_path(tree, osname, ref, serial) + ".origin"
++    data = construct_aleph_json(tree, origin)
++
++    # write the data out to the file
++    with open(os.path.join(tree, ALEPH_FILENAME), "w", encoding="utf8") as f:
++        json.dump(data, f, indent=4, sort_keys=True)
++        f.write("\n")
++
++    # create a symlink for backwards compatibility with CoreOS
++    if coreos_compat:
++        os.symlink(ALEPH_FILENAME, os.path.join(tree, COREOS_ALEPH_FILENAME))
++
++
++if __name__ == '__main__':
++    stage_args = osbuild.api.arguments()
++    r = main(stage_args["tree"],
++             stage_args["options"])
++    sys.exit(r)
+-- 
+2.41.0
+

src/coreos.osbuild.mpp.yaml

@@ -110,6 +110,9 @@ pipelines:
                 - ref: $ref
                   remote:
                     url: $repourl
+      - type: org.osbuild.ostree.aleph
+        options:
+          coreos_compat: true
       - type: org.osbuild.ostree.selinux
         options:
           deployment: