diff --git a/build.sh b/build.sh index e7e7572c7a..ae37ae7def 100755 --- a/build.sh +++ b/build.sh @@ -168,12 +168,35 @@ write_archive_info() { } patch_osbuild() { - # A few patches that either haven't made it into a release or + # Add a few patches that either haven't made it into a release or # that will be obsoleted with other work that will be done soon. + + # To make it easier to apply patches we'll move around the osbuild + # code on the system first: + rmdir /usr/lib/osbuild/osbuild + mv /usr/lib/python3.12/site-packages/osbuild /usr/lib/osbuild/ + mkdir /usr/lib/osbuild/tools + mv /usr/bin/osbuild-mpp /usr/lib/osbuild/tools/ + + # Now all the software is under the /usr/lib/osbuild dir and we can patch cat /usr/lib/coreos-assembler/0001-Mount-boot-from-host-in-host-builder-case.patch \ /usr/lib/coreos-assembler/0001-osbuild-util-fscache-calculate-actual-size-of-files.patch \ /usr/lib/coreos-assembler/0002-util-tweak-_calculate_size-to-_calculate_space.patch \ - | patch -p1 -d /usr/lib/python3.12/site-packages/ + /usr/lib/coreos-assembler/0001-devices-loopback-make-setting-sector_size-meaningful.patch \ + /usr/lib/coreos-assembler/0002-tools-osbuild-mpp-handle-corner-case-in-mpp-format-i.patch \ + /usr/lib/coreos-assembler/0003-tools-osbuild-mpp-support-defining-multiple-image-la.patch \ + /usr/lib/coreos-assembler/0004-tools-osbuild-mpp-add-sector-size-support-for-image-.patch \ + /usr/lib/coreos-assembler/0005-tools-osbuild-mpp-set-part-ID-from-name-if-missing.patch \ + /usr/lib/coreos-assembler/0001-osbuild-util-ostree-convert-cli-to-return-the-comple.patch \ + /usr/lib/coreos-assembler/0002-osbuild-util-ostree-optimize-deployment_path.patch \ + /usr/lib/coreos-assembler/0003-create-org.osbuild.ostree.aleph-stage.patch \ + | patch -d /usr/lib/osbuild -p1 + + # And then move the files back; supermin appliance creation will need it back + # in the places delivered by the RPM. + mv /usr/lib/osbuild/tools/osbuild-mpp /usr/bin/osbuild-mpp + mv /usr/lib/osbuild/osbuild /usr/lib/python3.12/site-packages/osbuild + mkdir /usr/lib/osbuild/osbuild } if [ $# -ne 0 ]; then diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 30bed3c856..01924648bf 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -174,7 +174,7 @@ StandardError=kmsg+console ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" ExecStart=/bin/sh -c "/usr/bin/udevadm settle" ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" -ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"] == \"%s\"' /mnt/.coreos-aleph-version.json" +ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"version\"] == \"%s\"' /mnt/.coreos-aleph-version.json" ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"ostree-commit\"] == \"%s\"' /mnt/.coreos-aleph-version.json" [Install] RequiredBy=coreos-installer.target diff --git a/src/0001-devices-loopback-make-setting-sector_size-meaningful.patch b/src/0001-devices-loopback-make-setting-sector_size-meaningful.patch new file mode 100644 index 0000000000..1c671e9f25 --- /dev/null +++ b/src/0001-devices-loopback-make-setting-sector_size-meaningful.patch @@ -0,0 +1,61 @@ +From 0770eb0090a7ad04bb2f5bca938be3453f71fe6e Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Tue, 21 Nov 2023 17:18:22 -0500 +Subject: [PATCH 1/5] devices/loopback: make setting sector_size meaningful + +For the org.osbuild.loopback the user can set the sector size, but +it had no effect on the underlying loopback device. Let's make it +meaningful by passing along the given value to the underlying code. +--- + devices/org.osbuild.loopback | 1 + + osbuild/loop.py | 8 ++++---- + 2 files changed, 5 insertions(+), 4 deletions(-) + +diff --git a/devices/org.osbuild.loopback b/devices/org.osbuild.loopback +index cc104391..0b6591d0 100755 +--- a/devices/org.osbuild.loopback ++++ b/devices/org.osbuild.loopback +@@ -82,6 +82,7 @@ class LoopbackService(devices.DeviceService): + setup=self.setup_loop, + offset=offset, + sizelimit=sizelimit, ++ blocksize=self.sector_size, + partscan=False, + autoclear=True) + +diff --git a/osbuild/loop.py b/osbuild/loop.py +index 675ea510..7f602191 100644 +--- a/osbuild/loop.py ++++ b/osbuild/loop.py +@@ -385,7 +385,7 @@ class Loop: + info = self._config_info(self.get_status(), offset, sizelimit, autoclear, partscan) + fcntl.ioctl(self.fd, self.LOOP_SET_STATUS64, info) + +- def configure(self, fd: int, offset=None, sizelimit=None, autoclear=None, partscan=None): ++ def configure(self, fd: int, offset=None, sizelimit=None, blocksize=0, autoclear=None, partscan=None): + """ + Configure the loopback device + Bind and configure in a single operation a file descriptor to the +@@ -427,6 +427,8 @@ class Loop: + sizelimit : int, optional + The max size in bytes to make the loopback device, or None + to leave unchanged (default is None) ++ blocksize : int, optional ++ Set the logical blocksize of the loopback device. Default is 0. + autoclear : bool, optional + Whether or not to enable autoclear, or None to leave unchanged + (default is None) +@@ -437,9 +439,7 @@ class Loop: + # pylint: disable=attribute-defined-outside-init + config = LoopConfig() + config.fd = fd +- # Previous implementation was not configuring the block size. +- # Keep same behavior here by setting the value to 0. +- config.block_size = 0 ++ config.block_size = int(blocksize) + config.info = self._config_info(LoopInfo(), offset, sizelimit, autoclear, partscan) + try: + fcntl.ioctl(self.fd, self.LOOP_CONFIGURE, config) +-- +2.41.0 + diff --git a/src/0001-osbuild-util-ostree-convert-cli-to-return-the-comple.patch b/src/0001-osbuild-util-ostree-convert-cli-to-return-the-comple.patch new file mode 100644 index 0000000000..bc6e3446f9 --- /dev/null +++ b/src/0001-osbuild-util-ostree-convert-cli-to-return-the-comple.patch @@ -0,0 +1,39 @@ +From 57f13570023653f85d0920ea64ae3433216c68fc Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Mon, 27 Nov 2023 16:14:43 -0500 +Subject: [PATCH 1/3] osbuild/util/ostree: convert cli to return the completed + process object + +And also set stdout=subprocess.PIPE. This will allow for callers to +parse and use the output of the command, but has the side effect of +meaning less gets printed to the screen during run. + +Co-authored-by: Luke Yang +--- + osbuild/util/ostree.py | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py +index 43dd99c5..09c648fb 100644 +--- a/osbuild/util/ostree.py ++++ b/osbuild/util/ostree.py +@@ -199,11 +199,11 @@ def cli(*args, _input=None, **kwargs): + """Thin wrapper for running the ostree CLI""" + args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()] + print("ostree " + " ".join(args), file=sys.stderr) +- subprocess.run(["ostree"] + args, +- encoding="utf8", +- stdout=sys.stderr, +- input=_input, +- check=True) ++ return subprocess.run(["ostree"] + args, ++ encoding="utf8", ++ stdout=subprocess.PIPE, ++ input=_input, ++ check=True) + + + def parse_input_commits(commits): +-- +2.41.0 + diff --git a/src/0002-osbuild-util-ostree-optimize-deployment_path.patch b/src/0002-osbuild-util-ostree-optimize-deployment_path.patch new file mode 100644 index 0000000000..7d24af7592 --- /dev/null +++ b/src/0002-osbuild-util-ostree-optimize-deployment_path.patch @@ -0,0 +1,43 @@ +From 009813573024d4d42f37e5c1d3752e0c0e07bb06 Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Mon, 27 Nov 2023 16:24:29 -0500 +Subject: [PATCH 2/3] osbuild/util/ostree: optimize deployment_path() + +When provisioning disk images there should really not be more than +one deployment on a system. We don't really need any of the parameters +here so let's make them optional and just find the deployment anyway. + +Co-authored-by: Luke Yang +--- + osbuild/util/ostree.py | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py +index 09c648fb..90821d5b 100644 +--- a/osbuild/util/ostree.py ++++ b/osbuild/util/ostree.py +@@ -1,5 +1,6 @@ + import collections + import contextlib ++import glob + import json + import os + import subprocess +@@ -217,6 +218,14 @@ def parse_input_commits(commits): + def deployment_path(root: PathLike, osname: str, ref: str, serial: int): + """Return the path to a deployment given the parameters""" + ++ if osname == "" and ref == "" and serial is None: ++ filenames = glob.glob(root + '/ostree/deploy/*/deploy/*.0', recursive=True) ++ if len(filenames) < 1: ++ raise ValueError("Could not find deployment") ++ if len(filenames) > 1: ++ raise ValueError("More than one deployment found") ++ return filenames[0] ++ + base = os.path.join(root, "ostree") + + repo = os.path.join(base, "repo") +-- +2.41.0 + diff --git a/src/0002-tools-osbuild-mpp-handle-corner-case-in-mpp-format-i.patch b/src/0002-tools-osbuild-mpp-handle-corner-case-in-mpp-format-i.patch new file mode 100644 index 0000000000..8e90b295b5 --- /dev/null +++ b/src/0002-tools-osbuild-mpp-handle-corner-case-in-mpp-format-i.patch @@ -0,0 +1,36 @@ +From 5207e92cab473627f4a5b5f2bea33c922cbac4d4 Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Tue, 21 Nov 2023 21:08:39 -0500 +Subject: [PATCH 2/5] tools/osbuild-mpp: handle corner case in mpp-format-int + +If you do math in mpp-format-int it could end up getting converted +to a float. Of course if you end up with a decimal value that isn't +`.0` that's a problem for an int, but if it is `.0` let's handle it +gracefully. + +For example, math like this could end up with a value with `.0`: + + mpp-format-int: "{bios_boot_size_mb * 1024 * 1024 / sector_size_bytes}" +--- + tools/osbuild-mpp | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/tools/osbuild-mpp b/tools/osbuild-mpp +index cce51a89..2bc33ac6 100755 +--- a/tools/osbuild-mpp ++++ b/tools/osbuild-mpp +@@ -1259,6 +1259,11 @@ class ManifestFile: + sys.exit(1) + + if res_type == "int": ++ # If the value ends with '.0' it could be because of ++ # some math that ended up converting the value to a ++ # float. Just trim it off in that case. ++ if res.endswith('.0'): ++ res = res.strip('.0') + res = int(res) + elif res_type == "json": + res = json.loads(res) +-- +2.41.0 + diff --git a/src/0003-create-org.osbuild.ostree.aleph-stage.patch b/src/0003-create-org.osbuild.ostree.aleph-stage.patch new file mode 100644 index 0000000000..84deee43f1 --- /dev/null +++ b/src/0003-create-org.osbuild.ostree.aleph-stage.patch @@ -0,0 +1,233 @@ +From f25a1718033dc6b2e51766c77febded06b5be978 Mon Sep 17 00:00:00 2001 +From: Luke Yang +Date: Wed, 25 Oct 2023 13:47:56 -0400 +Subject: [PATCH 3/3] create org.osbuild.ostree.aleph stage + +Similar to the aleph file created for builds of FCOS based on ostree +commit inputs, this adds an aleph file that contains information about +the initial deployment of data when the disk image was built + +A new stage is preferred here as both the org.osbuild.ostree.deploy +and org.osbuild.ostree.deploy.container stages need an aleph file and +use of the aleph file may depend on the project/product. For example, +right now CoreOS is the only project that uses an aleph file, but others +may want it in the future. +--- + osbuild/util/ostree.py | 28 ++++++ + stages/org.osbuild.ostree.aleph | 165 ++++++++++++++++++++++++++++++++ + 2 files changed, 193 insertions(+) + create mode 100755 stages/org.osbuild.ostree.aleph + +diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py +index 90821d5b..170c28c3 100644 +--- a/osbuild/util/ostree.py ++++ b/osbuild/util/ostree.py +@@ -237,6 +237,34 @@ def deployment_path(root: PathLike, osname: str, ref: str, serial: int): + return sysroot + + ++def parse_origin(origin: PathLike): ++ """Parse the origin file and return the deployment type and imgref ++ ++ Example container case: container-image-reference=ostree-remote-image:fedora:docker://quay.io/fedora/fedora-coreos:stable ++ Example ostree commit case: refspec=fedora:fedora/x86_64/coreos/stable ++ """ ++ deploy_type = "" ++ imgref = "" ++ with open(origin, "r", encoding="utf8") as f: ++ for line in f: ++ separated_line = line.split("=") ++ if separated_line[0] == "container-image-reference": ++ deploy_type = "container" ++ imgref = separated_line[1].rstrip() ++ break ++ if separated_line[0] == "refspec": ++ deploy_type = "ostree_commit" ++ imgref = separated_line[1].rstrip() ++ break ++ ++ if deploy_type == "": ++ raise ValueError("Could not find 'container-image-reference' or 'refspec' in origin file") ++ if imgref == "": ++ raise ValueError("Could not find imgref in origin file") ++ ++ return deploy_type, imgref ++ ++ + class PasswdLike: + """Representation of a file with structure like /etc/passwd + +diff --git a/stages/org.osbuild.ostree.aleph b/stages/org.osbuild.ostree.aleph +new file mode 100755 +index 00000000..b152a822 +--- /dev/null ++++ b/stages/org.osbuild.ostree.aleph +@@ -0,0 +1,165 @@ ++#!/usr/bin/python3 ++""" ++Create aleph version file for the deployment. ++""" ++ ++ ++import json ++import os ++import sys ++ ++import osbuild.api ++from osbuild.util import ostree ++ ++CAPABILITIES = ["CAP_MAC_ADMIN"] ++ALEPH_FILENAME = ".aleph-version.json" ++COREOS_ALEPH_FILENAME = ".coreos-aleph-version.json" ++ ++ ++SCHEMA_2 = """ ++"options": { ++ "additionalProperties": false, ++ "properties": { ++ "coreos_compat": { ++ "description": "boolean to allow for CoreOS aleph version backwards compatibility", ++ "type": "boolean" ++ }, ++ "deployment": { ++ "additionalProperties": false, ++ "required": ["osname", "ref"], ++ "properties": { ++ "osname": { ++ "description": "Name of the stateroot to be used in the deployment", ++ "type": "string" ++ }, ++ "ref": { ++ "description": "OStree ref to create and use for deployment", ++ "type": "string" ++ }, ++ "serial": { ++ "description": "The deployment serial (usually '0')", ++ "type": "number", ++ "default": 0 ++ } ++ } ++ } ++ } ++} ++""" ++ ++ ++def aleph_commit(tree, imgref): ++ extra_args = [] ++ extra_args.append("--print-metadata-key=version") ++ ++ aleph_version = ostree.cli("show", f"--repo={tree}/ostree/repo", imgref, *extra_args).stdout.rstrip().strip('\'') ++ aleph_ref = imgref ++ # get the commit by parsing the revision of the deployment ++ aleph_ostree_commit = ostree.rev_parse(tree + "/ostree/repo", imgref) ++ ++ aleph_version_data = { ++ "osbuild-version": osbuild.__version__, ++ "version": aleph_version, ++ "ref": aleph_ref, ++ "ostree-commit": aleph_ostree_commit ++ } ++ ++ return aleph_version_data ++ ++ ++def aleph_container(tree, imgref): ++ # extract the image name from the imgref ++ imgref_list = imgref.split(':') ++ if imgref_list[0] in ["ostree-remote-registry", "ostree-remote-image"]: ++ img_name = ':'.join(imgref_list[2:]) ++ elif imgref_list[0] in ["ostree-image-signed", "ostree-unverified-registry"]: ++ img_name = ':'.join(imgref_list[1:]) ++ else: ++ raise ValueError(f"Image ref {imgref} has unsupported type (supported: 'ostree-remote-registry', \ ++ 'ostree-remote-image', 'ostree-image-signed', or 'ostree-unverified-registry')") ++ ++ img_name = img_name.removeprefix('docker://') ++ ++ extra_args = [] ++ extra_args.append(f"--repo={tree}/ostree/repo") ++ extra_args.append(f"registry:{img_name}") ++ ++ container_data_json = ostree.cli("container", "image", "metadata", *extra_args).stdout.rstrip() ++ container_data = json.loads(container_data_json) ++ ++ extra_args.append("--config") ++ container_data_config_json = ostree.cli("container", "image", "metadata", *extra_args).stdout.rstrip() ++ container_data_config = json.loads(container_data_config_json) ++ ++ aleph_digest = container_data['config']['digest'] ++ aleph_ref = f"docker://{imgref}" ++ aleph_version = container_data_config['config']['Labels']['org.opencontainers.image.version'] ++ aleph_container_image = container_data_config['config']['Labels'] ++ ++ aleph_version_data = { ++ "osbuild-version": osbuild.__version__, ++ "ref": aleph_ref, ++ "version": aleph_version, ++ "container-image": { ++ "image-name": img_name, ++ "image-digest": aleph_digest, ++ "image-labels": aleph_container_image ++ } ++ } ++ ++ # the 'ostree.commit' label will be optional in the future so ++ # prevent hard failing if key is not found ++ aleph_ostree_commit = container_data_config['config']['Labels'].get('ostree.commit') ++ if aleph_ostree_commit is not None: ++ aleph_version_data["ostree-commit"] = aleph_ostree_commit ++ ++ return aleph_version_data ++ ++ ++def construct_aleph_json(tree, origin): ++ deploy_type, imgref = ostree.parse_origin(origin) ++ data = {} ++ # null deploy_type and imgref error is caught in the parse_origin() function ++ if deploy_type == "container": ++ data = aleph_container(tree, imgref) ++ elif deploy_type == "ostree_commit": ++ data = aleph_commit(tree, imgref) ++ else: ++ raise ValueError("Unknown deployment type") ++ ++ return data ++ ++ ++def main(tree, options): ++ coreos_compat = options.get("coreos_compat", False) ++ dep = options.get("deployment", None) ++ osname = "" ++ ref = "" ++ serial = None ++ origin = "" ++ ++ # if deployment is specified, use it to find the origin file. ++ # otherwise, autodetect the only deployment found in the tree. ++ if dep is not None: ++ osname = dep.get("osname", "") ++ ref = dep.get("ref", "") ++ serial = dep.get("serial", 0) ++ ++ origin = ostree.deployment_path(tree, osname, ref, serial) + ".origin" ++ data = construct_aleph_json(tree, origin) ++ ++ # write the data out to the file ++ with open(os.path.join(tree, ALEPH_FILENAME), "w", encoding="utf8") as f: ++ json.dump(data, f, indent=4, sort_keys=True) ++ f.write("\n") ++ ++ # create a symlink for backwards compatibility with CoreOS ++ if coreos_compat: ++ os.symlink(ALEPH_FILENAME, os.path.join(tree, COREOS_ALEPH_FILENAME)) ++ ++ ++if __name__ == '__main__': ++ stage_args = osbuild.api.arguments() ++ r = main(stage_args["tree"], ++ stage_args["options"]) ++ sys.exit(r) +-- +2.41.0 + diff --git a/src/0003-tools-osbuild-mpp-support-defining-multiple-image-la.patch b/src/0003-tools-osbuild-mpp-support-defining-multiple-image-la.patch new file mode 100644 index 0000000000..ace318252c --- /dev/null +++ b/src/0003-tools-osbuild-mpp-support-defining-multiple-image-la.patch @@ -0,0 +1,116 @@ +From 2e1f6e255356f9fa0e29484be5749ab954c1cba4 Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Tue, 21 Nov 2023 21:06:05 -0500 +Subject: [PATCH 3/5] tools/osbuild-mpp: support defining multiple image + layouts + +Right now you can only define a single image, lets add mpp-define-images +and accept a list. +--- + tools/osbuild-mpp | 66 ++++++++++++++++++++++++++++------------------- + 1 file changed, 39 insertions(+), 27 deletions(-) + +diff --git a/tools/osbuild-mpp b/tools/osbuild-mpp +index 2bc33ac6..b3165936 100755 +--- a/tools/osbuild-mpp ++++ b/tools/osbuild-mpp +@@ -263,8 +263,8 @@ Example: + + Defining partition layouts for disk images: + +-It is possbile to define a partition layout via `mpp-define-image`. The defined layout +-is actually written to a temporary sparse file and read back via `sfdisk`, so that all ++It is possbile to define partition layouts via `mpp-define-images`. The defined layouts ++are actually written to a temporary sparse file and read back via `sfdisk`, so that all + partition data like `size` and `start` include actual padding and such. The `image` + variable will be defined with `size` and `layout` keys, the latter containing the + partition layout data. It can be accessed via the "String expansion" explained above. +@@ -273,23 +273,28 @@ Example: + + ``` + ... +- "mpp-define-image": { +- "size": "10737418240", +- "table": { +- "uuid": "D209C89E-EA5E-4FBD-B161-B461CCE297E0", +- "label": "gpt", +- "partitions": [ +- { +- "id": "bios-boot", +- "start": 2048, +- "size": 2048, +- "type": "21686148-6449-6E6F-744E-656564454649", +- "bootable": true, +- "uuid": "FAC7F1FB-3E8D-4137-A512-961DE09A5549" +- "attrs": [ 60 ] +- }, +- ... +- } ++ "mpp-define-images": [ ++ { ++ "id": "image", ++ "size": "10737418240", ++ "table": { ++ "uuid": "D209C89E-EA5E-4FBD-B161-B461CCE297E0", ++ "label": "gpt", ++ "partitions": [ ++ { ++ "id": "bios-boot", ++ "start": 2048, ++ "size": 2048, ++ "type": "21686148-6449-6E6F-744E-656564454649", ++ "bootable": true, ++ "uuid": "FAC7F1FB-3E8D-4137-A512-961DE09A5549", ++ "attrs": [ 60 ] ++ }, ++ ... ++ ] ++ } ++ } ++ ] + ... + ``` + +@@ -1016,7 +1021,7 @@ class ManifestFile: + raise ValueError(f"Unknown manfest version {version}") + + m.process_imports() +- m.process_partition() ++ m.process_partitions() + + return m + +@@ -1303,16 +1308,23 @@ class ManifestFile: + def process_format(self): + self._process_format(self.root) + +- def process_partition(self): +- desc = self.get_mpp_node(self.root, "define-image") ++ def process_partitions(self): ++ images = self.get_mpp_node(self.root, "define-images") or [] + +- if not desc: +- return ++ # Backwards compat for mpp-define-image (no list) ++ image = self.get_mpp_node(self.root, "define-image") ++ if image: ++ if id not in image: ++ image['id'] = "image" ++ images.append(image) + +- self._process_format(desc) ++ if len(images) == 0: ++ return + +- name = desc.get("id", "image") +- self.vars[name] = Image.from_dict(desc) ++ for image in images: ++ self._process_format(image) ++ name = image["id"] ++ self.vars[name] = Image.from_dict(image) + + # pylint: disable=no-self-use + def get_pipeline_name(self, node): +-- +2.41.0 + diff --git a/src/0004-tools-osbuild-mpp-add-sector-size-support-for-image-.patch b/src/0004-tools-osbuild-mpp-add-sector-size-support-for-image-.patch new file mode 100644 index 0000000000..9f52012c23 --- /dev/null +++ b/src/0004-tools-osbuild-mpp-add-sector-size-support-for-image-.patch @@ -0,0 +1,45 @@ +From 28c2772d42159ccdf07b629509dbaf23ebb61f68 Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Tue, 21 Nov 2023 21:12:37 -0500 +Subject: [PATCH 4/5] tools/osbuild-mpp: add sector size support for image + layouts + +Now you can specify a sector_size in `mpp-define-images` to support +creating a 4k native disk image (sector_size=4096). + +This does use a loopback device, which means osbuild-mpp also needs +to run as root, when previously that wasn't necessary. +--- + tools/osbuild-mpp | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +diff --git a/tools/osbuild-mpp b/tools/osbuild-mpp +index b3165936..c2f440a7 100755 +--- a/tools/osbuild-mpp ++++ b/tools/osbuild-mpp +@@ -974,13 +974,19 @@ class Image: + def from_dict(cls, js): + size = js["size"] + data = js["table"] ++ sector_size = js.get('sector_size', 512) + + with tempfile.TemporaryDirectory() as tmp: + image = os.path.join(tmp, "disk.img") + subprocess.run(["truncate", "--size", size, image], check=True) +- +- table = PartitionTable.from_dict(data) +- table.write_to(image) ++ cp = subprocess.run(["losetup", "--find", "--show", f"--sector-size={sector_size}", image], ++ stdout=subprocess.PIPE, check=True) ++ loopimage = cp.stdout.rstrip() ++ try: ++ table = PartitionTable.from_dict(data) ++ table.write_to(loopimage) ++ finally: ++ subprocess.run(["losetup", "-d", loopimage]) + + return cls(size, table) + +-- +2.41.0 + diff --git a/src/0005-tools-osbuild-mpp-set-part-ID-from-name-if-missing.patch b/src/0005-tools-osbuild-mpp-set-part-ID-from-name-if-missing.patch new file mode 100644 index 0000000000..cc6294a462 --- /dev/null +++ b/src/0005-tools-osbuild-mpp-set-part-ID-from-name-if-missing.patch @@ -0,0 +1,57 @@ +From e191dc1d0dd8086067ac4a241d51501bd8f93060 Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Mon, 27 Nov 2023 15:53:13 -0500 +Subject: [PATCH 5/5] tools/osbuild-mpp: set part ID from name if missing + +For our Fedora CoreOS disk images we set the partition labels (name) +for the partitions. This is also supported using the primitives here +in OSBuild, but it wasn't obvious that I needed to set the name in +the mpp-define-images definition. Let's set the name there, but let's +also allow osbuild-mpp to set the `id`, which is what is used later +to access that partition from the `name` too if `id` isn't set. + +This means we allow something like: + + - name: BIOS-BOOT + type: 21686148-6449-6E6F-744E-656564454649 + bootable: true + uuid: FAC7F1FB-3E8D-4137-A512-961DE09A5549 + size: 100 + +rather than requiring something like: + + - id: BIOS-BOOT + name: BIOS-BOOT + type: 21686148-6449-6E6F-744E-656564454649 + bootable: true + uuid: FAC7F1FB-3E8D-4137-A512-961DE09A5549 + size: 100 +--- + tools/osbuild-mpp | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/tools/osbuild-mpp b/tools/osbuild-mpp +index c2f440a7..3214e647 100755 +--- a/tools/osbuild-mpp ++++ b/tools/osbuild-mpp +@@ -283,6 +283,7 @@ Example: + "partitions": [ + { + "id": "bios-boot", ++ "name": "BIOS-BOOT", + "start": 2048, + "size": 2048, + "type": "21686148-6449-6E6F-744E-656564454649", +@@ -820,7 +821,8 @@ class Partition: + name: str = None, + uuid: str = None, + attrs: List[int] = None): +- self.id = uid ++ # if no id provided, use the part-label (name) as the id ++ self.id = uid or name + self.type = pttype + self.start = start + self.size = size +-- +2.41.0 + diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 804314d552..3d77eda303 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -263,7 +263,7 @@ platforms_json="${workdir}/tmp/platforms.json" yaml2json "${configdir}/platforms.yaml" "${platforms_json}" # Currently we only support OSBuild for qemu and metal disk images -if [ "${image_type}" == "qemu" ] || [ "${image_type}" == "metal" ]; then +if [ "${image_type}" == "qemu" ] || [ "${image_type}" == "metal" ] || [ "${image_type}" == "metal4k" ]; then OSBUILD_SUPPORTED=1 fi diff --git a/src/coreos.osbuild.mpp.yaml b/src/coreos.osbuild.mpp.yaml index 1d92c1dbd6..7fb812757d 100644 --- a/src/coreos.osbuild.mpp.yaml +++ b/src/coreos.osbuild.mpp.yaml @@ -8,39 +8,68 @@ mpp-vars: # XXX: Dynamically set this size in the future root_size_mb: 2048 sector_size: 512 -mpp-define-image: - id: image - size: - mpp-format-string: "{disk_size_gb * 1024 * 1024 * 1024}" - table: - uuid: 00000000-0000-4000-a000-000000000001 - label: gpt - partitions: - - id: BIOS-BOOT - name: BIOS-BOOT - type: 21686148-6449-6E6F-744E-656564454649 - bootable: true - uuid: FAC7F1FB-3E8D-4137-A512-961DE09A5549 - size: - mpp-format-int: "{bios_boot_size_mb * 1024 * 1024 / sector_size}" - - id: EFI-SYSTEM - name: EFI-SYSTEM - type: C12A7328-F81F-11D2-BA4B-00A0C93EC93B - uuid: 68B2905B-DF3E-4FB3-80FA-49D1E773AA33 - size: - mpp-format-int: "{efi_system_size_mb * 1024 * 1024 / sector_size}" - - id: boot - name: boot - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - uuid: 61B2905B-DF3E-4FB3-80FA-49D1E773AA32 - size: - mpp-format-int: "{boot_size_mb * 1024 * 1024 / sector_size}" - - id: root - name: root - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - uuid: CA7D7CCB-63ED-4C53-861C-1742536059CC - size: - mpp-format-int: "{root_size_mb * 1024 * 1024 / sector_size}" + four_k_sector_size: 4096 +mpp-define-images: + - id: image + sector_size: + mpp-format-int: "{sector_size}" + size: + mpp-format-string: "{disk_size_gb * 1024 * 1024 * 1024}" + table: + uuid: 00000000-0000-4000-a000-000000000001 + label: gpt + partitions: + - name: BIOS-BOOT + type: 21686148-6449-6E6F-744E-656564454649 + bootable: true + uuid: FAC7F1FB-3E8D-4137-A512-961DE09A5549 + size: + mpp-format-int: "{bios_boot_size_mb * 1024 * 1024 / sector_size}" + - name: EFI-SYSTEM + type: C12A7328-F81F-11D2-BA4B-00A0C93EC93B + uuid: 68B2905B-DF3E-4FB3-80FA-49D1E773AA33 + size: + mpp-format-int: "{efi_system_size_mb * 1024 * 1024 / sector_size}" + - name: boot + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + uuid: 61B2905B-DF3E-4FB3-80FA-49D1E773AA32 + size: + mpp-format-int: "{boot_size_mb * 1024 * 1024 / sector_size}" + - name: root + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + uuid: CA7D7CCB-63ED-4C53-861C-1742536059CC + size: + mpp-format-int: "{root_size_mb * 1024 * 1024 / sector_size}" + - id: image4k + sector_size: + mpp-format-int: "{four_k_sector_size}" + size: + mpp-format-string: "{disk_size_gb * 1024 * 1024 * 1024}" + table: + uuid: 00000000-0000-4000-a000-000000000001 + label: gpt + partitions: + - name: BIOS-BOOT + type: 21686148-6449-6E6F-744E-656564454649 + bootable: true + uuid: FAC7F1FB-3E8D-4137-A512-961DE09A5549 + size: + mpp-format-int: "{bios_boot_size_mb * 1024 * 1024 / four_k_sector_size}" + - name: EFI-SYSTEM + type: C12A7328-F81F-11D2-BA4B-00A0C93EC93B + uuid: 68B2905B-DF3E-4FB3-80FA-49D1E773AA33 + size: + mpp-format-int: "{efi_system_size_mb * 1024 * 1024 / four_k_sector_size}" + - name: boot + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + uuid: 61B2905B-DF3E-4FB3-80FA-49D1E773AA32 + size: + mpp-format-int: "{boot_size_mb * 1024 * 1024 / four_k_sector_size}" + - name: root + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + uuid: CA7D7CCB-63ED-4C53-861C-1742536059CC + size: + mpp-format-int: "{root_size_mb * 1024 * 1024 / four_k_sector_size}" pipelines: - name: tree source-epoch: 1659397331 @@ -54,7 +83,7 @@ pipelines: repo: /ostree/repo config: sysroot: - readonly: false + readonly: true bootloader: none - type: org.osbuild.mkdir options: @@ -81,6 +110,9 @@ pipelines: - ref: $ref remote: url: $repourl + - type: org.osbuild.ostree.aleph + options: + coreos_compat: true - type: org.osbuild.ostree.selinux options: deployment: @@ -222,6 +254,126 @@ pipelines: number: mpp-format-int: '{image.layout[''boot''].index}' path: /grub2 + - name: raw-4k-image + stages: + - type: org.osbuild.truncate + options: + filename: disk.img + size: + mpp-format-string: '{image4k.size}' + - type: org.osbuild.sfdisk + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + sector-size: + mpp-format-int: "{four_k_sector_size}" + options: + mpp-format-json: '{image4k.layout}' + - type: org.osbuild.mkfs.fat + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image4k.layout[''EFI-SYSTEM''].start}' + size: + mpp-format-int: '{image4k.layout[''EFI-SYSTEM''].size}' + lock: true + sector-size: + mpp-format-int: "{four_k_sector_size}" + options: + label: EFI-SYSTEM + volid: 7B7795E7 + - type: org.osbuild.mkfs.ext4 + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image4k.layout[''boot''].start}' + size: + mpp-format-int: '{image4k.layout[''boot''].size}' + lock: true + sector-size: + mpp-format-int: "{four_k_sector_size}" + options: + uuid: 96d15588-3596-4b3c-adca-a2ff7279ea63 + label: boot + - type: org.osbuild.mkfs.xfs + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image4k.layout[''root''].start}' + size: + mpp-format-int: '{image4k.layout[''root''].size}' + lock: true + sector-size: + mpp-format-int: "{four_k_sector_size}" + options: + uuid: 910678ff-f77e-4a7d-8d53-86f2ac47a823 + label: root + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:tree + options: + paths: + - from: input://tree/ + to: mount://root/ + devices: + efi: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image4k.layout[''EFI-SYSTEM''].start}' + size: + mpp-format-int: '{image4k.layout[''EFI-SYSTEM''].size}' + sector-size: + mpp-format-int: "{four_k_sector_size}" + boot: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image4k.layout[''boot''].start}' + size: + mpp-format-int: '{image4k.layout[''boot''].size}' + sector-size: + mpp-format-int: "{four_k_sector_size}" + root: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image4k.layout[''root''].start}' + size: + mpp-format-int: '{image4k.layout[''root''].size}' + sector-size: + mpp-format-int: "{four_k_sector_size}" + mounts: + - name: root + type: org.osbuild.xfs + source: root + target: / + - name: boot + type: org.osbuild.ext4 + source: boot + target: /boot + - name: efi + type: org.osbuild.fat + source: efi + target: /boot/efi - name: raw-metal-image stages: - type: org.osbuild.copy @@ -254,6 +406,40 @@ pipelines: type: org.osbuild.ext4 source: boot target: / + - name: raw-metal4k-image + stages: + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-4k-image + options: + paths: + - from: input://tree/disk.img + to: tree:///disk.img + - type: org.osbuild.kernel-cmdline.bls-append + options: + bootpath: mount:/// + kernel_opts: + - ignition.platform.id=metal + devices: + boot: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image4k.layout[''boot''].start}' + size: + mpp-format-int: '{image4k.layout[''boot''].size}' + sector-size: + mpp-format-int: "{four_k_sector_size}" + mounts: + - name: boot + type: org.osbuild.ext4 + source: boot + target: / - name: raw-qemu-image stages: - type: org.osbuild.copy @@ -302,6 +488,20 @@ pipelines: - from: input://tree/disk.img to: mpp-format-string: 'tree:///{filename}' + - name: metal4k + stages: + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-metal4k-image + options: + paths: + - from: input://tree/disk.img + to: + mpp-format-string: 'tree:///{filename}' - name: qemu stages: - type: org.osbuild.qemu diff --git a/src/create_disk.sh b/src/create_disk.sh index 0924d51877..0d5d6f2d31 100755 --- a/src/create_disk.sh +++ b/src/create_disk.sh @@ -386,6 +386,11 @@ test -d "${deploy_root}" || (echo "failed to find $deploy_root"; exit 1) # # build: The coreos-assembler build ID; today we support # having the same ostree commit in different image builds. +# version: same value as build for now, even though version is +# not exactly the same as build ID, this will ease +# transition to using osbuild where it's defined +# appropriately and build: isn't there: +# https://github.com/osbuild/osbuild/pull/1475 # ref: The ostree ref used; useful for cross-checking. # ostree-commit: Similar to `ref`; one can derive this from looking # at the coreos-assembler builds, but it's very @@ -395,6 +400,7 @@ test -d "${deploy_root}" || (echo "failed to find $deploy_root"; exit 1) cat > $rootfs/.coreos-aleph-version.json << EOF { "build": "${buildid}", + "version": "${buildid}", "ref": "${ref}", "ostree-commit": "${commit}", "imgid": "${imgid}"