create-legacy-oscontainer: use runvm to build legacy oscontainer

This introduces a new command to create a oci-archive of the
legacy oscontainer that will be pushed with
`cosa push-container-manifest` by the pipeline.

(cherry picked from commit 1f4cf7f)

jlebon: Had to manually add to entrypoint.

docs/cosa.md

@@ -74,3 +74,4 @@ Those less commonly used commands are listed here:
 | [tag](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-tag) | Operate on the tags in `builds.json`
 | [test-coreos-installer](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-test-coreos-installer) | Automate an end-to-end run of coreos-installer with the metal image
 | [upload-oscontainer](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-upload-oscontainer) | Upload an oscontainer (historical wrapper for `cosa oscontainer`)
+| [create-legacy-oscontainer](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-create-legacy-oscontainer) | Create an oscontainer in legacy format (i.e. not OSTree-native)

src/cmd-create-legacy-oscontainer

@@ -0,0 +1,134 @@
+#!/usr/bin/python3 -u
+# Upload an oscontainer.  This is a wrapper for
+# `cosa oscontainer` that just for historical reasons
+# used to live downstream in the redhat-coreos pipeline.
+# In the future we should just have one `cosa oscontainer`
+# command.
+
+import argparse
+import json
+import yaml
+import os
+import shutil
+import subprocess
+import sys
+from cosalib.cmdlib import sha256sum_file
+from cosalib.utils import runcmd
+
+cosa_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, cosa_dir)
+
+from cosalib import cmdlib
+
+
+with open('builds/builds.json') as f:
+    builds = json.load(f)['builds']
+if len(builds) == 0:
+    cmdlib.fatal("No builds found")
+latest_build = builds[0]['id']
+arch = cmdlib.get_basearch()
+latest_build_path = f"builds/{latest_build}/{arch}"
+
+metapath = f"{latest_build_path}/meta.json"
+with open(metapath) as f:
+    meta = json.load(f)
+
+name = meta['name'] + '-' + meta['buildid'] + '-oscontainer.' + arch + '.ociarchive'
+parser = argparse.ArgumentParser()
+parser.add_argument("--arch-tag", help="append arch name to push tag",
+                    action='store_true')
+parser.add_argument("--name", help="oscontainer name",
+                    action='store', default=f'{name}')
+parser.add_argument("--from", help="Base image", default='scratch',
+                    dest='from_image')
+parser.add_argument("--format", help="Format to use for push")
+parser.add_argument("--add-directory", help="Copy in all content from referenced directory DIR",
+                    metavar='DIR', action='append', default=[])
+
+args = parser.parse_args()
+
+# for backcompat, we auto-build extensions if they're missing
+if os.path.exists('src/config/extensions.yaml'):
+    if 'extensions' not in meta:
+        runcmd(['coreos-assembler', 'buildextend-extensions'])
+        with open(metapath) as f:
+            meta = json.load(f)
+    assert 'extensions' in meta
+
+configdir = os.path.abspath('src/config')
+oscconfigpath = f'{configdir}/oscontainer.yaml'
+# XXX: fold oscontainer.yaml handling into oscontainer.py
+configyaml = {}
+if os.path.exists(oscconfigpath):
+    with open(oscconfigpath) as f:
+        configyaml = yaml.safe_load(f)
+
+if 'base' in configyaml:
+    args.from_image = configyaml['base']
+
+print("Preparing to upload oscontainer for build: {}".format(latest_build))
+ostree_commit = meta['ostree-commit']
+
+tmprepo = "{}/tmp/repo".format(os.getcwd())
+# if tmprepo is not a directory, but is unexpectedly a file,
+# just nuke it
+if not os.path.isdir(tmprepo) and os.path.exists(tmprepo):
+    os.remove(tmprepo)
+
+# if tmprepo is not a directory and not a file, recreate from
+# the tarfile
+if not os.path.exists(tmprepo):
+    os.makedirs(tmprepo, exist_ok=True)
+    ostree_commit_tar = meta['images']['ostree']['path']
+    subprocess.check_call(['tar', '-xf',
+                           f'{latest_build_path}/{ostree_commit_tar}',
+                           '-C', tmprepo])
+
+tmp_osreleasedir = 'tmp/usrlib-osrelease'
+subprocess.check_call(['rm', '-rf', tmp_osreleasedir])
+runcmd(['/usr/bin/ostree', 'checkout', '--repo', tmprepo,
+        '--user-mode', '--subpath=/usr/lib/os-release', ostree_commit,
+        tmp_osreleasedir])
+display_name = None
+with open(os.path.join(tmp_osreleasedir, "os-release")) as f:
+    display_name = subprocess.check_output(['/bin/sh', '-c', 'set -euo pipefail; . /proc/self/fd/0 && echo $NAME'], stdin=f, encoding='UTF-8').strip()
+if display_name == "":
+    raise SystemExit(f"Failed to find NAME= in /usr/lib/os-release in commit {ostree_commit}")
+shutil.rmtree(tmp_osreleasedir)
+
+osc_name_and_tag = f"{args.name}:{latest_build}"
+if args.arch_tag:
+    arch = meta.get("coreos-assembler.basearch", cmdlib.get_basearch)
+    osc_name_and_tag = f"{args.name}:{latest_build}-{arch}"
+
+# TODO: Use labels for the build hash and avoid pulling the oscontainer
+# every time we want to poll.
+# TODO: Remove --from
+digestfile = "tmp/oscontainer-digest"
+print("Entering vm to build oscontainer for build: {}".format(latest_build))
+
+cosa_argv = (['/usr/lib/coreos-assembler/create-legacy-oscontainer.sh', '--workdir=./tmp', 'build', f'--from={args.from_image}'])
+for d in args.add_directory:
+    cosa_argv.append(f'--add-directory="{d}"')
+cosa_argv.append(f'--display-name="{display_name}"')
+if 'labeled-packages' in configyaml:
+    pkgs = ' '.join(configyaml['labeled-packages'])
+    cosa_argv.append(f'--labeled-packages="{pkgs}"')
+if args.format is not None:
+    cosa_argv.append(f'--format={args.format}')
+subprocess.check_call(cosa_argv +
+    [f'--digestfile={digestfile}',
+        '--push', tmprepo,
+        meta['ostree-commit'],
+        osc_name_and_tag])
+
+# Inject the oscontainer with SHA256 into the build metadata
+oci_archive = f"{latest_build_path}/{args.name}"
+meta['images']['legacy-oscontainer'] = {'path': args.name,
+                                        'sha256': sha256sum_file(oci_archive),
+                                        'size': os.path.getsize(oci_archive),
+                                        "skip-compression": True}
+metapath_new = f"{metapath}.new"
+with open(metapath_new, 'w') as f:
+    json.dump(meta, f, sort_keys=True)
+shutil.move(metapath_new, metapath)

src/coreos-assembler

@@ -43,7 +43,7 @@ cmd=${1:-}
 # commands we'd expect to use in the local dev path
 build_commands="init fetch build run prune clean list"
 # commands more likely to be used in a prod pipeline only
-advanced_build_commands="buildprep buildupload oc-adm-release push-container-manifest upload-oscontainer"
+advanced_build_commands="buildprep buildupload oc-adm-release push-container-manifest upload-oscontainer create-legacy-oscontainer"
 buildextend_commands="aliyun aws azure digitalocean exoscale gcp ibmcloud live metal metal4k openstack qemu vmware vultr"
 utility_commands="aws-replicate compress generate-hashlist koji-upload kola remote-prune sign tag"
 other_commands="shell meta"