From 9c47b82858ce4582088c6499055b6f32e8e14e8e Mon Sep 17 00:00:00 2001 From: "Luke Browning [IBM]" <918375@nutanix.com> Date: Wed, 27 May 2020 11:30:28 -0700 Subject: [PATCH 1/5] Add support for Centos 7.7 builds Centos 7.3 used to be supported, but no longer works with EPEL 7, so the build process is being updated to support Centos 7.7 to resolve the EPEL 7 problem. Several changes to the build process have been made, including: - updating the minimal repository package list to explicitly include anaconda, anaconda-dracut, grub2, etc. These changes are ported from the upstream parent project open-power-host-os. - updating the yum repos to point at Centos 7.3.1611 Altarch - pylorax crash workaround of an extraneous reference to installroot/proc/modules - use old distro install tools (yum) instead of new tool (dnf). - minimize upstream yum caching as it seems to fail less often when downloading epel Signed-off-by: Luke Browning [IBM] <918375@nutanix.com> --- config.yaml | 13 +++++++++---- lib/iso_builder.py | 16 ++++++++++++++-- mock_configs/CentOS/7/CentOS-7-ppc64le.cfg | 7 ++++--- .../CentOS/7/build-iso-CentOS-7-ppc64le.cfg | 9 ++++++--- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/config.yaml b/config.yaml index da9b9915..9637bd39 100644 --- a/config.yaml +++ b/config.yaml @@ -3,7 +3,7 @@ common: result_dir: 'result' verbose: False log_size: 2097152 - packages_metadata_repo_url: "https://github.com/NXPower/versions.git" + packages_metadata_repo_url: "/home/nutanix/devel/versions" packages_metadata_repo_branch: '2.0-nutanix' nutanix_version_base_commit: '2.0-2017-03-31' nutanix_version: '20170331.${num_commits}' @@ -63,13 +63,18 @@ build_iso: - "debugging" - "nutanix-ahv-group" iso_root_fs_packages: + - "anaconda" + - "anaconda-dracut" + # packages needed by anaconda + - "redhat-upgrade-dracut" - "grub2" + - "yum-langpacks" mock_iso_repo_name: "host_os" mock_iso_repo_dir: "/host-os-repo" distro_repos_urls: - base: "http://vault.centos.org/altarch/7.3.1611/os/ppc64le/" - updates: "http://vault.centos.org/altarch/7.3.1611/updates/ppc64le/" - extras: "http://vault.centos.org/altarch/7.3.1611/extras/ppc64le/" + base: "http://vault.centos.org/altarch/7.7.1908/os/ppc64le/" + updates: "http://vault.centos.org/altarch/7.7.1908/updates/ppc64le/" + extras: "http://vault.centos.org/altarch/7.7.1908/extras/ppc64le/" epel: "http://download.fedoraproject.org/pub/epel/7/ppc64le/" distro_repo_args: - "--excludepkgs=kernel*,qemu*,libvirt,libvirt-client,libvirt-daemon*,SLOF,nmap-ncat" diff --git a/lib/iso_builder.py b/lib/iso_builder.py index 047b5fd6..4c18a5c6 100644 --- a/lib/iso_builder.py +++ b/lib/iso_builder.py @@ -66,7 +66,8 @@ def _setup(self): LOG.info("Initializing a chroot") self._run_mock_command("--init") - package_list = ["createrepo", "pungi"] + package_list = [ + "yum-plugin-priorities", "yum-utils", "createrepo", "pungi"] LOG.info("Installing %s inside the chroot" % " ".join(package_list)) self._run_mock_command("--install %s" % " ".join(package_list)) @@ -148,7 +149,18 @@ def _create_iso_kickstart(self): def _build(self): LOG.info("Building ISO") - build_cmd = ("pungi -c %s --nosource --nodebuginfo --name %s --ver %s" % + build_cmd = ("pungi -c %s --nosource --nodebuginfo --name %s --ver %s -G -C" % + (self.config.get('automated_install_file'), + self.distro, self.version)) + self._run_mock_command("--shell '%s'" % build_cmd) + + # pylorax opens/creates installroot/proc/modules across the call to dracut + # to minimize warnings in the latter, but installroot/proc doesn't exist so + # it fails. Interestingly, dracut only produces one warning anyway. + self._run_mock_command( + "--shell 'sed -i s?proc/modules?proc-modules? /usr/lib/python2.7/site-packages/pylorax/treebuilder.py'"); + + build_cmd = ("pungi -c %s --nosource --nodebuginfo --name %s --ver %s -B -I" % (self.config.get('automated_install_file'), self.distro, self.version)) self._run_mock_command("--shell '%s'" % build_cmd) diff --git a/mock_configs/CentOS/7/CentOS-7-ppc64le.cfg b/mock_configs/CentOS/7/CentOS-7-ppc64le.cfg index 49f15c8f..4a0a75df 100644 --- a/mock_configs/CentOS/7/CentOS-7-ppc64le.cfg +++ b/mock_configs/CentOS/7/CentOS-7-ppc64le.cfg @@ -19,6 +19,7 @@ config_opts['target_arch'] = 'ppc64le' config_opts['legal_host_arches'] = ('ppc64le',) config_opts['dist'] = 'el7' # only useful for --resultdir variable subst config_opts['releasever'] = '7' +config_opts['package_manager'] = 'yum' config_opts['yum.conf'] = """ [main] @@ -37,7 +38,7 @@ mdpolicy=group:primary # repos [base] name=BaseOS -baseurl=http://vault.centos.org/altarch/7.3.1611/os/ppc64le/ +baseurl=http://vault.centos.org/altarch/7.7.1908/os/ppc64le/ failovermethod=priority gpgkey=file:///etc/pki/mock/RPM-GPG-KEY-CentOS-7,file:///etc/pki/mock/RPM-GPG-KEY-CentOS-SIG-AltArch-7-ppc64le gpgcheck=0 @@ -45,7 +46,7 @@ gpgcheck=0 [updates] name=updates enabled=1 -baseurl=http://vault.centos.org/altarch/7.3.1611/updates/ppc64le/ +baseurl=http://vault.centos.org/altarch/7.7.1908/updates/ppc64le/ failovermethod=priority gpgkey=file:///etc/pki/mock/RPM-GPG-KEY-CentOS-7,file:///etc/pki/mock/RPM-GPG-KEY-CentOS-SIG-AltArch-7-ppc64le gpgcheck=0 @@ -59,7 +60,7 @@ gpgcheck=0 [extras] name=extras -baseurl=http://vault.centos.org/altarch/7.3.1611/extras/ppc64le/ +baseurl=http://vault.centos.org/altarch/7.7.1908/extras/ppc64le/ failovermethod=priority gpgkey=file:///etc/pki/mock/RPM-GPG-KEY-EPEL-7,file:///etc/pki/mock/RPM-GPG-KEY-CentOS-SIG-AltArch-7-ppc64le gpgcheck=0 diff --git a/mock_configs/CentOS/7/build-iso-CentOS-7-ppc64le.cfg b/mock_configs/CentOS/7/build-iso-CentOS-7-ppc64le.cfg index ba905652..38470068 100644 --- a/mock_configs/CentOS/7/build-iso-CentOS-7-ppc64le.cfg +++ b/mock_configs/CentOS/7/build-iso-CentOS-7-ppc64le.cfg @@ -15,11 +15,13 @@ config_opts["macros"]["%__gzip"] = "/usr/bin/pigz" config_opts["macros"]["%__bzip2"] = "/usr/bin/lbzip2" config_opts['nspawn_args'] = ["--bind=/dev", "--bind=/dev/pts", "--bind=/dev/shm"] config_opts['nspawn_args'] += ["--share-system"] +config_opts['nspawn_args'] += ["--capability=all"] config_opts['root'] = 'CentOS-7-ppc64le' config_opts['target_arch'] = 'ppc64le' config_opts['legal_host_arches'] = ('ppc64le',) config_opts['dist'] = 'el7' # only useful for --resultdir variable subst config_opts['releasever'] = '7' +config_opts['package_manager'] = 'yum' config_opts['yum.conf'] = """ [main] @@ -34,11 +36,12 @@ assumeyes=1 syslog_ident=mock syslog_device= mdpolicy=group:primary +http_caching=packages # repos [base] name=BaseOS -baseurl=http://vault.centos.org/altarch/7.3.1611/os/ppc64le/ +baseurl=http://vault.centos.org/altarch/7.7.1908/os/ppc64le/ failovermethod=priority gpgkey=file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-7,file:///usr/share/distribution-gpg-keys/centos//RPM-GPG-KEY-CentOS-SIG-AltArch-7-ppc64le gpgcheck=1 @@ -46,7 +49,7 @@ gpgcheck=1 [updates] name=updates enabled=1 -baseurl=http://vault.centos.org/altarch/7.3.1611/updates/ppc64le/ +baseurl=http://vault.centos.org/altarch/7.7.1908/updates/ppc64le/ failovermethod=priority gpgkey=file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-7,file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-SIG-AltArch-7-ppc64le gpgcheck=1 @@ -60,7 +63,7 @@ gpgcheck=1 [extras] name=extras -baseurl=http://vault.centos.org/altarch/7.3.1611/extras/ppc64le/ +baseurl=http://vault.centos.org/altarch/7.7.1908/extras/ppc64le/ failovermethod=priority gpgkey=file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-7,file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-SIG-AltArch-7-ppc64le gpgcheck=1 From 33a89e1903b600574d4eaf5083be6a262ae2ed68 Mon Sep 17 00:00:00 2001 From: lukebrowning Date: Wed, 27 May 2020 14:50:49 -0500 Subject: [PATCH 2/5] Update config.yaml Restore github.com NXPower URL for versions repo --- config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 9637bd39..4e49bfba 100644 --- a/config.yaml +++ b/config.yaml @@ -3,7 +3,7 @@ common: result_dir: 'result' verbose: False log_size: 2097152 - packages_metadata_repo_url: "/home/nutanix/devel/versions" + packages_metadata_repo_url: "https://github.com/NXPower/versions.git" packages_metadata_repo_branch: '2.0-nutanix' nutanix_version_base_commit: '2.0-2017-03-31' nutanix_version: '20170331.${num_commits}' From 8a4d7683bcb356ad9f0ad721fdb433ffde99ec5e Mon Sep 17 00:00:00 2001 From: lukebrowning Date: Thu, 28 May 2020 19:35:51 -0500 Subject: [PATCH 3/5] Update iso_builder.py Added second pylorax patch to increase tmpfs size to 4G for live dvd installer image used by foundation --- lib/iso_builder.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/iso_builder.py b/lib/iso_builder.py index 4c18a5c6..97f7278a 100644 --- a/lib/iso_builder.py +++ b/lib/iso_builder.py @@ -159,6 +159,13 @@ def _build(self): # it fails. Interestingly, dracut only produces one warning anyway. self._run_mock_command( "--shell 'sed -i s?proc/modules?proc-modules? /usr/lib/python2.7/site-packages/pylorax/treebuilder.py'"); + + # Increase the size of the temporary loop device used to build the live rootfs image + self._run_mock_command( + "--shell 'sed -i s?size=2\):?size=4\):? /usr/lib/python2.7/site-packages/pylorax/treebuilder.py'"); + + # TODO: This almost works, but the remembered word pattern \1\2 resolves to ^A^B instead of the matched pattern + #'--shell "sed -i -r \'s/(def[ \t]+create_runtime\()+(.*)+size=2\):/\1\2size=4\):/\' /usr/lib/python2.7/site-packages/pylorax/treebuilder.py"'); build_cmd = ("pungi -c %s --nosource --nodebuginfo --name %s --ver %s -B -I" % (self.config.get('automated_install_file'), From ecf26f815158f4b56ed2c9d93adbfc5dd2eefa8a Mon Sep 17 00:00:00 2001 From: lukebrowning Date: Fri, 29 May 2020 02:07:02 -0500 Subject: [PATCH 4/5] Update iso_builder.py --- lib/iso_builder.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/iso_builder.py b/lib/iso_builder.py index 97f7278a..0dd11402 100644 --- a/lib/iso_builder.py +++ b/lib/iso_builder.py @@ -162,10 +162,11 @@ def _build(self): # Increase the size of the temporary loop device used to build the live rootfs image self._run_mock_command( - "--shell 'sed -i s?size=2\):?size=4\):? /usr/lib/python2.7/site-packages/pylorax/treebuilder.py'"); + "--shell 'sed -i s?size=2,?size=4,? /usr/lib/python2.7/site-packages/pylorax/__init__.py'"); - # TODO: This almost works, but the remembered word pattern \1\2 resolves to ^A^B instead of the matched pattern - #'--shell "sed -i -r \'s/(def[ \t]+create_runtime\()+(.*)+size=2\):/\1\2size=4\):/\' /usr/lib/python2.7/site-packages/pylorax/treebuilder.py"'); + # Increase the size of the temporary loop device used to build the live rootfs image + self._run_mock_command( + "--shell 'sed -i s?size=2\):?size=4\):? /usr/lib/python2.7/site-packages/pylorax/treebuilder.py'"); build_cmd = ("pungi -c %s --nosource --nodebuginfo --name %s --ver %s -B -I" % (self.config.get('automated_install_file'), From 16f377cc4f8a91133492d3ba37800ffe739c1b56 Mon Sep 17 00:00:00 2001 From: lukebrowning Date: Sun, 31 May 2020 10:12:54 -0500 Subject: [PATCH 5/5] Update config.yaml --- config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 4e49bfba..4f1070f4 100644 --- a/config.yaml +++ b/config.yaml @@ -77,7 +77,7 @@ build_iso: extras: "http://vault.centos.org/altarch/7.7.1908/extras/ppc64le/" epel: "http://download.fedoraproject.org/pub/epel/7/ppc64le/" distro_repo_args: - - "--excludepkgs=kernel*,qemu*,libvirt,libvirt-client,libvirt-daemon*,SLOF,nmap-ncat" + - "--excludepkgs=kernel*,qemu*,libvirt,libvirt-client,libvirt-daemon*,nmap-ncat" update_metapackage: push_repo_url: push_repo_branch: "master"