From 50a10b6e4ebde4022510e415752e6ba2dcc576c5 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 21 Feb 2024 08:56:34 +0100 Subject: [PATCH 01/20] DAOS-15079 test: update packaging subsystem Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- packaging/Dockerfile.mockbuild | 20 +++++++--- packaging/Makefile_distro_vars.mk | 8 ++++ packaging/Makefile_packaging.mk | 9 ++++- packaging/ccache-stats.patch | 66 +++++++++++++++++++++++++++++++ packaging/get_base_branch | 22 +++++++++++ packaging/rpm_chrootbuild | 57 ++++++++++++++++---------- 6 files changed, 153 insertions(+), 29 deletions(-) create mode 100644 packaging/ccache-stats.patch create mode 100755 packaging/get_base_branch diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 0ca7003..76a6e94 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,11 +1,11 @@ # -# Copyright 2018-2023 Intel Corporation +# Copyright 2018-2024 Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image -ARG FVERSION=38 +ARG FVERSION=latest FROM fedora:$FVERSION # Needed for later use of FVERSION ARG FVERSION @@ -48,16 +48,24 @@ RUN dnf -y upgrade && \ # https://github.com/rpm-software-management/rpmlint/pull/795 in it # But make sure to patch after dnf upgrade so that an upgraded rpmlint # RPM doesn't wipe out our patch +# Ditto for the patch to zero and display ccache stats +# https://github.com/rpm-software-management/mock/pull/1299 ARG PACKAGINGDIR=packaging -COPY ${PACKAGINGDIR}/rpmlint--ignore-unused-rpmlintrc.patch . +COPY ${PACKAGINGDIR}/*.patch ./ RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ - if ! patch -p1; then \ + if ! patch -p1 < $OLDPWD/rpmlint--ignore-unused-rpmlintrc.patch; then \ exit 1; \ fi; \ rm -f rpmlint/__pycache__/{cli,lint}.*.pyc; \ - fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ - rm -f rpmlint--ignore-unused-rpmlintrc.patch + fi; \ + if ! grep _ccachePostBuildHook mockbuild/plugins/ccache.py; then \ + if ! patch -p3 < $OLDPWD/ccache-stats.patch; then \ + exit 1; \ + fi; \ + rm -f mockbuild/plugins/__pycache__/ccache.*.pyc; \ + fi); \ + rm -f rpmlint--ignore-unused-rpmlintrc.patch ccache-stats.patch # show the release that was built ARG CACHEBUST diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index 6a7f88b..4e8a09d 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -83,6 +83,14 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 15.4 SED_EXPR := 1p endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64) +VERSION_ID := 15.5 +DISTRO_ID := sl15.5 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.5 +SED_EXPR := 1p +endif endif ifeq ($(ID),centos) ID = el diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 2fdcaed..3201a22 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -54,7 +54,8 @@ RPM_BUILD_OPTIONS := $(BUILD_DEFINES) GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%') endif -COMMON_RPM_ARGS = --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) +FVERSION ?= latest +COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) DEB_RVERS := $(subst $(DOT),\$(DOT),$(VERSION)) @@ -363,12 +364,14 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) LOCAL_REPOS='$(LOCAL_REPOS)' \ ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ DISTRO_VERSION="$(DISTRO_VERSION)" \ + PACKAGE="$(NAME)" \ TARGET="$<" \ packaging/rpm_chrootbuild endif podman_chrootbuild: if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + --build-arg FVERSION=$(FVERSION) \ -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ -f packaging/Dockerfile.mockbuild .; then \ echo "Container build failed"; \ @@ -386,7 +389,9 @@ podman_chrootbuild: exit 1; \ fi; \ rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ - grep -v -e debuginfo -e debugsource -e src.rpm)' + grep -v -e debuginfo -e debugsource -e src.rpm)'; then \ + exit 1; \ + fi docker_chrootbuild: if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ diff --git a/packaging/ccache-stats.patch b/packaging/ccache-stats.patch new file mode 100644 index 0000000..26d5eeb --- /dev/null +++ b/packaging/ccache-stats.patch @@ -0,0 +1,66 @@ +From e87d916d7f49ea4949973adf0f09e9e5bf891e03 Mon Sep 17 00:00:00 2001 +From: "Brian J. Murrell" +Date: Tue, 30 Jan 2024 11:03:12 -0500 +Subject: [PATCH 1/2] Show ccache stats at the end of the build + +Zero the ccache stats at the beginning of the build and then display the +ccache stats at the end of the build to see how effective ccache was. + +Signed-off-by: Brian J. Murrell +--- + mock/py/mockbuild/plugins/ccache.py | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py +index 2666ad9fc..1080ffe68 100644 +--- a/mock/py/mockbuild/plugins/ccache.py ++++ b/mock/py/mockbuild/plugins/ccache.py +@@ -35,6 +35,7 @@ def __init__(self, plugins, conf, buildroot): + buildroot.preexisting_deps.append("ccache") + plugins.add_hook("prebuild", self._ccacheBuildHook) + plugins.add_hook("preinit", self._ccachePreInitHook) ++ plugins.add_hook("postbuild", self._ccachePostBuildHook) + buildroot.mounts.add( + BindMountPoint(srcpath=self.ccachePath, bindpath=buildroot.make_chroot_path("/var/tmp/ccache"))) + +@@ -47,6 +48,9 @@ def __init__(self, plugins, conf, buildroot): + @traceLog() + def _ccacheBuildHook(self): + self.buildroot.doChroot(["ccache", "-M", str(self.ccache_opts['max_cache_size'])], shell=False) ++ # zero ccache stats ++ getLog().info("Zero ccache stats:") ++ self.buildroot.doChroot(["ccache", "--zero-stats"], printOutput=True, shell=False) + + # set up the ccache dir. + # we also set a few variables used by ccache to find the shared cache. +@@ -61,3 +65,10 @@ def _ccachePreInitHook(self): + file_util.mkdirIfAbsent(self.buildroot.make_chroot_path('/var/tmp/ccache')) + file_util.mkdirIfAbsent(self.ccachePath) + self.buildroot.uid_manager.changeOwner(self.ccachePath, recursive=True) ++ ++ # get some cache stats ++ def _ccachePostBuildHook(self): ++ # show the cache hit stats ++ getLog().info("ccache stats:") ++ self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False) +++ + +From bfd3a7e1bb47d28ee60a94cb5985c1f66476475f Mon Sep 17 00:00:00 2001 +From: "Brian J. Murrell" +Date: Tue, 30 Jan 2024 11:17:48 -0500 +Subject: [PATCH 2/2] Remove extraneous line + +Signed-off-by: Brian J. Murrell +--- + mock/py/mockbuild/plugins/ccache.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py +index 1080ffe68..1a20846d3 100644 +--- a/mock/py/mockbuild/plugins/ccache.py ++++ b/mock/py/mockbuild/plugins/ccache.py +@@ -71,4 +71,3 @@ def _ccachePostBuildHook(self): + # show the cache hit stats + getLog().info("ccache stats:") + self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False) +-+ diff --git a/packaging/get_base_branch b/packaging/get_base_branch new file mode 100755 index 0000000..27515a7 --- /dev/null +++ b/packaging/get_base_branch @@ -0,0 +1,22 @@ +#!/bin/bash + +# find the base branch of the current branch + +set -eux -o pipefail +IFS=' ' read -r -a add_bases <<< "${1:-}" +origin=origin +mapfile -t all_bases < <(echo "master" + git branch -r | sed -ne "/^ $origin\\/release\\/[0-9]/s/^ $origin\\///p") +all_bases+=("${add_bases[@]}") +TARGET="master" +min_diff=-1 +for base in "${all_bases[@]}"; do + git rev-parse --verify "$origin/$base" &> /dev/null || continue + commits_ahead=$(git log --oneline "$origin/$base..HEAD" | wc -l) + if [ "$min_diff" -eq -1 ] || [ "$min_diff" -gt "$commits_ahead" ]; then + TARGET="$base" + min_diff=$commits_ahead + fi +done +echo "$TARGET" +exit 0 diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index d122e0e..4dcdaa4 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,10 +2,14 @@ set -uex -original_cfg_file="/etc/mock/$CHROOT_NAME.cfg" -cfg_file=mock.cfg +cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg + +# Enable mock ccache plugin +cat <> mock.cfg +config_opts['plugin_conf']['ccache_enable'] = True +config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" +EOF -cp "$original_cfg_file" "$cfg_file" if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then cat <> mock.cfg @@ -22,7 +26,7 @@ if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then fi # Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available -cat <> "$cfg_file" +cat <> mock.cfg config_opts['dnf.conf'] += """ [main] best=0 @@ -33,7 +37,7 @@ EOF repo_adds=() repo_dels=() -echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" +echo -e "config_opts['yum.conf'] += \"\"\"\n" >> mock.cfg if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then repo_dels+=("--disablerepo=\*") @@ -56,7 +60,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" fi fi - curl -sSf "${REPO_FILE_URL}daos_ci-$DISTRO"-mock-artifactory.repo >> "$cfg_file" + curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg repo_adds+=("--enablerepo *-artifactory") fi fi @@ -81,9 +85,9 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ name=$repo:${branch//[@\/]/_}:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ +baseurl=${ARTIFACTS_URL:-${JENKINS_URL:-https://build.hpdd.intel.com/}job/}daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ -gpgcheck=False\n" >> "$cfg_file" +gpgcheck=False\n" >> mock.cfg done for repo in $JOB_REPOS; do repo_name=${repo##*://} @@ -97,29 +101,40 @@ for repo in $JOB_REPOS; do echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ baseurl=${repo//\//%2F}\n\ -enabled=1\n" >> "$cfg_file" +enabled=1\n" >> mock.cfg done -echo "\"\"\"" >> "$cfg_file" +echo "\"\"\"" >> mock.cfg if [ -n "$DISTRO_VERSION" ]; then releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") fi bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap -if ls -l /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then - mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap" +if ls -l "$bs_dir"/root_cache/cache.tar.gz; then + mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap/" flock "$bs_dir" -c "cp -a $bs_dir/root_cache /var/cache/mock/${CHROOT_NAME}-bootstrap" fi +if ls -l "$bs_dir/ccache-$CHROOT_NAME-$PACKAGE".tar.gz; then + flock "$bs_dir" -c "tar -C / -xzf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz" +fi -# shellcheck disable=SC2086 -eval mock -r "$cfg_file" ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ - "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +rc=0 +# shellcheck disable=SC2086,SC2048 +if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ + --disablerepo=\*-debug* ${releasever_opt[*]} $MOCK_OPTIONS \ + $RPM_BUILD_OPTIONS "$TARGET"; then + rc=${PIPESTATUS[0]} +fi -date -if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz && - [ -d /scratch/ ]; then - mkdir -p /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/ - if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then - flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/" +# Save the ccache +if [ -d /scratch/ ]; then + mkdir -p "$bs_dir"/ + flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache" + if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then + if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then + flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/" + fi fi fi + +exit "$rc" From de677b23a64b36759f8492f474bd880224dc2631 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Thu, 25 Jan 2024 07:00:21 +0000 Subject: [PATCH 02/20] DAOS-15079 test: VMD Hot Plug - Support rpc.py in spdk-tools package We need to support rpc.py from spdk-tools package (RPM) so that we can use the script in CI. Make the changes in spdk.spec so that rpc.py is included in the RPM. After installing the RPM, we should be able to use the script by calling: /usr/share/spdk/scripts/rpc.py Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Makito Kano --- spdk.spec | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/spdk.spec b/spdk.spec index 199ba24..29fcc1d 100644 --- a/spdk.spec +++ b/spdk.spec @@ -11,7 +11,7 @@ Name: spdk Version: 22.01.2 -Release: 5%{?dist} +Release: 6%{?dist} Epoch: 0 Summary: Set of libraries and utilities for high performance user-mode storage @@ -198,6 +198,11 @@ cp build/examples/nvme_manage %{buildroot}/%{_bindir}/spdk_nvme_manage # Change /usr/bin/{env ,}bash to resolve env-script-interpreter rpmlint error. sed -i -e '1s/env //' %{install_datadir}/scripts/setup.sh +# Install rpc.py tool +cp scripts/rpc.py %{install_datadir}/scripts/ +mkdir -p %{install_datadir}/python/spdk +cp -r python/spdk/rpc %{install_datadir}/python/spdk/ + %if %{with doc} # Install doc mkdir -p %{install_docdir} @@ -226,7 +231,7 @@ rm -f %{buildroot}/%{_libdir}/*.a %files tools %{_datadir}/%{name}/include %{_datadir}/%{name}/scripts - +%{_datadir}/%{name}/python %if %{with doc} %files doc @@ -235,6 +240,9 @@ rm -f %{buildroot}/%{_libdir}/*.a %changelog +* Thu Jan 25 2024 Makito Kano - 0:22.01.2-6 +- Add rpc.py to spdk-tools package + * Mon Oct 16 2023 Brian J. Murrell - 0:22.01.2-5 - Change spdk-devel's R: dpdk-devel to dpdk-daos-devel to ensure we get the daos-targetted dpdk build From b2f241aa83643c628d5b82a7241e177933c89786 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Fri, 26 Jan 2024 00:24:09 +0000 Subject: [PATCH 03/20] DAOS-15079 test: Update comment Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Makito Kano --- spdk.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spdk.spec b/spdk.spec index 29fcc1d..0d81c07 100644 --- a/spdk.spec +++ b/spdk.spec @@ -241,7 +241,7 @@ rm -f %{buildroot}/%{_libdir}/*.a %changelog * Thu Jan 25 2024 Makito Kano - 0:22.01.2-6 -- Add rpc.py to spdk-tools package +- Add rpc.py to spdk-tools package. * Mon Oct 16 2023 Brian J. Murrell - 0:22.01.2-5 - Change spdk-devel's R: dpdk-devel to dpdk-daos-devel to ensure we get the From eb4f1789c95ab01b8086e14415239e2a0cbafc23 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 21 Feb 2024 14:38:33 +0100 Subject: [PATCH 04/20] DAOS-15079 test: install all rpc.py dependencies Required-githooks: true Signed-off-by: Tomasz Gromadzki --- spdk.spec | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spdk.spec b/spdk.spec index 0d81c07..6ff7da9 100644 --- a/spdk.spec +++ b/spdk.spec @@ -200,8 +200,8 @@ sed -i -e '1s/env //' %{install_datadir}/scripts/setup.sh # Install rpc.py tool cp scripts/rpc.py %{install_datadir}/scripts/ -mkdir -p %{install_datadir}/python/spdk -cp -r python/spdk/rpc %{install_datadir}/python/spdk/ +mkdir -p %{install_datadir}/scripts/rpc +cp scripts/rpc/*.py %{install_datadir}/scripts/rpc/ %if %{with doc} # Install doc @@ -231,7 +231,6 @@ rm -f %{buildroot}/%{_libdir}/*.a %files tools %{_datadir}/%{name}/include %{_datadir}/%{name}/scripts -%{_datadir}/%{name}/python %if %{with doc} %files doc From 9df51e41b79981c0a36062f7dd6b884505a0bff1 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 22 Feb 2024 18:00:27 +0100 Subject: [PATCH 05/20] DAOS-15079 test: skip Centos7 build Required-githooks: true Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b4916fe..2b1754a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,5 +41,5 @@ //@Library(value="pipeline-lib@your_branch") _ /* groovylint-disable-next-line CompileStatic */ -packageBuildingPipelineDAOSTest(['distros': ['centos7', 'el8', 'el9', 'leap15', 'ubuntu20.04'], +packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'], 'test-tag': 'pr']) From 25c89593321778ef0631e3a0ebfa28b88ee98669 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 29 Feb 2024 12:52:45 +0100 Subject: [PATCH 06/20] DAOS-15079 test: Fix rpmlint error Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 9 +++++++++ spdk.spec | 10 +++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/debian/changelog b/debian/changelog index fc15a2e..d4798b4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +spdk (22.01.2-6) unstable; urgency=medium + + [ Tomasz Gromadzki ] + * Add rpc.py to spdk-tools package. + * Synchronize Debian package with RPMs by adding examples to spdk-tools package. + + + -- Tomasz Gromadzki Tue, 05 Mar 2024 00:00:00 +0000 + spdk (22.01.2-3) unstable; urgency=medium [ Tom Nabarro ] diff --git a/spdk.spec b/spdk.spec index 6ff7da9..b126522 100644 --- a/spdk.spec +++ b/spdk.spec @@ -187,7 +187,7 @@ make -C doc # Install tools mkdir -p %{install_datadir}/scripts -# install the setup tool +# Install the setup tool cp scripts/{setup,common}.sh %{install_datadir}/scripts/ mkdir -p %{install_datadir}/include/%{name}/ cp include/%{name}/pci_ids.h %{install_datadir}/include/%{name}/ @@ -195,14 +195,14 @@ cp include/%{name}/pci_ids.h %{install_datadir}/include/%{name}/ cp build/examples/lsvmd %{buildroot}/%{_bindir}/spdk_nvme_lsvmd cp build/examples/nvme_manage %{buildroot}/%{_bindir}/spdk_nvme_manage -# Change /usr/bin/{env ,}bash to resolve env-script-interpreter rpmlint error. -sed -i -e '1s/env //' %{install_datadir}/scripts/setup.sh - # Install rpc.py tool cp scripts/rpc.py %{install_datadir}/scripts/ mkdir -p %{install_datadir}/scripts/rpc cp scripts/rpc/*.py %{install_datadir}/scripts/rpc/ +# Change /usr/bin/{env ,}bash/python to resolve env-script-interpreter rpmlint error. +sed -i -e '1s/env //' %{install_datadir}/scripts/{setup.sh,rpc.py} + %if %{with doc} # Install doc mkdir -p %{install_docdir} @@ -239,7 +239,7 @@ rm -f %{buildroot}/%{_libdir}/*.a %changelog -* Thu Jan 25 2024 Makito Kano - 0:22.01.2-6 +* Tue Mar 05 2024 Tomasz Gromadzki - 0:22.01.2-6 - Add rpc.py to spdk-tools package. * Mon Oct 16 2023 Brian J. Murrell - 0:22.01.2-5 From ff5daebce6488fdd361dbe57d9515cd236691f20 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 1 Mar 2024 15:05:00 +0100 Subject: [PATCH 07/20] DAOS-15079 test: update changelog Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index d4798b4..80068e5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,6 @@ spdk (22.01.2-6) unstable; urgency=medium * Add rpc.py to spdk-tools package. * Synchronize Debian package with RPMs by adding examples to spdk-tools package. - -- Tomasz Gromadzki Tue, 05 Mar 2024 00:00:00 +0000 spdk (22.01.2-3) unstable; urgency=medium From 61a0c87155ccd6dbe258333fe127677d61c09f8f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 5 Mar 2024 18:36:52 +0100 Subject: [PATCH 08/20] DAOS-15079 test: extend Debian package build procedure Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/rules | 1 + debian/spdk-tools.dirs | 2 ++ debian/spdk-tools.install | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 debian/spdk-tools.dirs diff --git a/debian/rules b/debian/rules index 6213553..d8c9e33 100755 --- a/debian/rules +++ b/debian/rules @@ -47,5 +47,6 @@ override_dh_auto_configure: override_dh_auto_install: dh_auto_install -- prefix=/usr libdir=/lib/x86_64-linux-gnu/ datadir=/usr/share + override_dh_missing: dh_missing --fail-missing diff --git a/debian/spdk-tools.dirs b/debian/spdk-tools.dirs new file mode 100644 index 0000000..2d832fe --- /dev/null +++ b/debian/spdk-tools.dirs @@ -0,0 +1,2 @@ +usr/share/spdk-tools/python +usr/share/spdk-tools/python/rpc diff --git a/debian/spdk-tools.install b/debian/spdk-tools.install index abb838e..bf052eb 100644 --- a/debian/spdk-tools.install +++ b/debian/spdk-tools.install @@ -1 +1,3 @@ -usr/bin/* \ No newline at end of file +usr/bin/* +scripts/rpc.py usr/share/spdk-tools/python +scripts/rpc/*.py usr/share/spdk-tools/python/rpc \ No newline at end of file From 57473fbe0db33bdf1f448b73d97e4ec96a060e06 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 7 Mar 2024 16:13:39 +0100 Subject: [PATCH 09/20] DAOS-15079 test: add examples installation for Debian to spdk-tools Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 2 +- debian/control | 2 +- debian/libspdk-dev.install | 2 +- debian/libspdk3.dirs | 1 + debian/libspdk3.install | 3 ++- debian/rules | 20 ++++++++++++++++---- debian/spdk-tools.dirs | 1 + debian/spdk-tools.install | 5 ++++- 8 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 debian/libspdk3.dirs diff --git a/debian/changelog b/debian/changelog index 80068e5..cd4daf1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,7 @@ spdk (22.01.2-6) unstable; urgency=medium [ Tomasz Gromadzki ] * Add rpc.py to spdk-tools package. * Synchronize Debian package with RPMs by adding examples to spdk-tools package. - + -- Tomasz Gromadzki Tue, 05 Mar 2024 00:00:00 +0000 spdk (22.01.2-3) unstable; urgency=medium diff --git a/debian/control b/debian/control index bc5e9ce..f913aa3 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Priority: optional Maintainer: daos-stack Build-Depends: debhelper, - python, + python3, libdpdk-dev (>= 21.11.2), librdmacm-dev, libibverbs-dev, diff --git a/debian/libspdk-dev.install b/debian/libspdk-dev.install index 08b9cb7..b893e83 100644 --- a/debian/libspdk-dev.install +++ b/debian/libspdk-dev.install @@ -1,4 +1,4 @@ usr/include/* lib/x86_64-linux-gnu/lib*.so lib/x86_64-linux-gnu/lib*.a -lib/x86_64-linux-gnu/pkgconfig/* \ No newline at end of file +lib/x86_64-linux-gnu/pkgconfig/* diff --git a/debian/libspdk3.dirs b/debian/libspdk3.dirs new file mode 100644 index 0000000..83e35c1 --- /dev/null +++ b/debian/libspdk3.dirs @@ -0,0 +1 @@ +usr/share/spdk/scripts diff --git a/debian/libspdk3.install b/debian/libspdk3.install index 0d82f92..d4e6c9d 100644 --- a/debian/libspdk3.install +++ b/debian/libspdk3.install @@ -1 +1,2 @@ -lib/x86_64-linux-gnu/lib*.so.* \ No newline at end of file +lib/x86_64-linux-gnu/lib*.so.* +scripts/{setup,common}.sh usr/share/spdk/scripts diff --git a/debian/rules b/debian/rules index d8c9e33..0a63090 100755 --- a/debian/rules +++ b/debian/rules @@ -6,6 +6,13 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+all # https://github.com/spdk/spdk/issues/2011 export DEB_CFLAGS_MAINT_APPEND = $(shell pkg-config --cflags libdpdk) +prefix := /usr + +cflags := -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 \ + -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong \ + -grecord-gcc-switches -m64 -mtune=generic -fasynchronous-unwind-tables \ + -fstack-clash-protection -fcf-protection + %: dh $@ @@ -41,12 +48,17 @@ override_dh_auto_configure: --without-vtune \ --with-shared -#override_dh_auto_build: -# make V=1 # -j 8 +override_dh_auto_build: + export CFLAGS='$(cflags)' + export CXXFLAGS='$(cflags)' + export FFLAGS='$(cflags)' + export LDFLAGS='-Wl,-z,relro -Wl,-z,now' + dh_auto_build -- CONFIG_ARCH=haswell V=1 override_dh_auto_install: - dh_auto_install -- prefix=/usr libdir=/lib/x86_64-linux-gnu/ datadir=/usr/share - + mv build/examples/lsvmd build/examples/spdk_lsvmd + mv build/examples/nvme_manage build/examples/spdk_nvme_manage + dh_auto_install -- prefix=$(prefix) libdir=/lib/x86_64-linux-gnu/ datadir=$(prefix)/share override_dh_missing: dh_missing --fail-missing diff --git a/debian/spdk-tools.dirs b/debian/spdk-tools.dirs index 2d832fe..116587c 100644 --- a/debian/spdk-tools.dirs +++ b/debian/spdk-tools.dirs @@ -1,2 +1,3 @@ +usr/include/spdk usr/share/spdk-tools/python usr/share/spdk-tools/python/rpc diff --git a/debian/spdk-tools.install b/debian/spdk-tools.install index bf052eb..26a06dd 100644 --- a/debian/spdk-tools.install +++ b/debian/spdk-tools.install @@ -1,3 +1,6 @@ usr/bin/* scripts/rpc.py usr/share/spdk-tools/python -scripts/rpc/*.py usr/share/spdk-tools/python/rpc \ No newline at end of file +scripts/rpc/*.py usr/share/spdk-tools/python/rpc +include/spdk/pci_ids.h usr/include/spdk +build/examples/spdk_lsvmd usr/bin/ +build/examples/spdk_nvme_manage usr/bin/ From 2c4375ab708ff73a505e5db08a3ceff1ac2a4a9f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 11 Mar 2024 21:09:36 +0100 Subject: [PATCH 10/20] DAOS-15079 test: install all scripts from spdk-tools Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/libspdk3.dirs | 1 - debian/spdk-tools.dirs | 1 + debian/spdk-tools.install | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 debian/libspdk3.dirs diff --git a/debian/libspdk3.dirs b/debian/libspdk3.dirs deleted file mode 100644 index 83e35c1..0000000 --- a/debian/libspdk3.dirs +++ /dev/null @@ -1 +0,0 @@ -usr/share/spdk/scripts diff --git a/debian/spdk-tools.dirs b/debian/spdk-tools.dirs index 116587c..3792b9d 100644 --- a/debian/spdk-tools.dirs +++ b/debian/spdk-tools.dirs @@ -1,3 +1,4 @@ +usr/share/spdk-tools/scripts usr/include/spdk usr/share/spdk-tools/python usr/share/spdk-tools/python/rpc diff --git a/debian/spdk-tools.install b/debian/spdk-tools.install index 26a06dd..81da758 100644 --- a/debian/spdk-tools.install +++ b/debian/spdk-tools.install @@ -1,6 +1,7 @@ usr/bin/* +scripts/{setup,common}.sh usr/share/spdk-tools/scripts +include/spdk/pci_ids.h usr/include/spdk scripts/rpc.py usr/share/spdk-tools/python scripts/rpc/*.py usr/share/spdk-tools/python/rpc -include/spdk/pci_ids.h usr/include/spdk build/examples/spdk_lsvmd usr/bin/ build/examples/spdk_nvme_manage usr/bin/ From ac97db13798fae44d8de7c82654ffb85ad5b2052 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 11 Mar 2024 21:22:22 +0100 Subject: [PATCH 11/20] DAOS-15079 test: remove redundant files Skip-test: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/libspdk3.install | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/libspdk3.install b/debian/libspdk3.install index d4e6c9d..0ea7251 100644 --- a/debian/libspdk3.install +++ b/debian/libspdk3.install @@ -1,2 +1 @@ lib/x86_64-linux-gnu/lib*.so.* -scripts/{setup,common}.sh usr/share/spdk/scripts From a67b2c095d01b4f7d0a412293c664eaa87a0c050 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 12 Mar 2024 18:14:33 +0100 Subject: [PATCH 12/20] DAOS-15079 test: update changelog Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 2 +- spdk.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index cd4daf1..cf4203b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,7 @@ spdk (22.01.2-6) unstable; urgency=medium * Add rpc.py to spdk-tools package. * Synchronize Debian package with RPMs by adding examples to spdk-tools package. - -- Tomasz Gromadzki Tue, 05 Mar 2024 00:00:00 +0000 + -- Tomasz Gromadzki Tue, 12 Mar 2024 00:00:00 +0000 spdk (22.01.2-3) unstable; urgency=medium diff --git a/spdk.spec b/spdk.spec index b126522..77e1401 100644 --- a/spdk.spec +++ b/spdk.spec @@ -239,7 +239,7 @@ rm -f %{buildroot}/%{_libdir}/*.a %changelog -* Tue Mar 05 2024 Tomasz Gromadzki - 0:22.01.2-6 +* Tue Mar 12 2024 Tomasz Gromadzki - 0:22.01.2-6 - Add rpc.py to spdk-tools package. * Mon Oct 16 2023 Brian J. Murrell - 0:22.01.2-5 From e052b865c179e280c81ac26cf166178fca24ab5c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 23 Apr 2024 22:25:28 +0200 Subject: [PATCH 13/20] Update changelog and test with proper tag on dedicated branch. Priority: 2 PR-repos: daos@PR-13602 Test-tag: pr,hw,medium,hot_plug Required-githooks: true Signed-off-by: Gromadzki, Tomasz --- debian/changelog | 2 +- spdk.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index cf4203b..0cab018 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,7 @@ spdk (22.01.2-6) unstable; urgency=medium * Add rpc.py to spdk-tools package. * Synchronize Debian package with RPMs by adding examples to spdk-tools package. - -- Tomasz Gromadzki Tue, 12 Mar 2024 00:00:00 +0000 + -- Tomasz Gromadzki Tue, 23 Apr 2024 00:00:00 +0000 spdk (22.01.2-3) unstable; urgency=medium diff --git a/spdk.spec b/spdk.spec index 77e1401..c6910ae 100644 --- a/spdk.spec +++ b/spdk.spec @@ -239,7 +239,7 @@ rm -f %{buildroot}/%{_libdir}/*.a %changelog -* Tue Mar 12 2024 Tomasz Gromadzki - 0:22.01.2-6 +* Tue Apr 23 2024 Tomasz Gromadzki - 0:22.01.2-6 - Add rpc.py to spdk-tools package. * Mon Oct 16 2023 Brian J. Murrell - 0:22.01.2-5 From 62bc83ade8551ec45cfcf44e36a0c821c8e034dd Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 23 Apr 2024 22:43:24 +0200 Subject: [PATCH 14/20] Update changelog and test with proper tag on dedicated branch. (fix PR reference) PR-repos: daos@PR-13620 Test-tag: pr,hw,medium,hot_plug Required-githooks: true Signed-off-by: Gromadzki, Tomasz From bab6f9500d8acb871ed421b92c92ec1e1d1e6fc2 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 Apr 2024 08:12:35 +0200 Subject: [PATCH 15/20] Rebuild with tests run on ci_vmd5 cluster PR-repos: daos@PR-13620:10 Test-tag: pr,hw,medium,hot_plug Required-githooks: true Skip-coverity-test: true Skip-fault-injection-test: true Skip-func-test-el8: true Skip-nlt: true Skip-python-bandit: true Skip-test-el-8.6-rpms: true Skip-unit-test-memcheck: true Skip-unit-tests: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Skip-func-hw-test-medium-md-on-ssd: false Skip-func-hw-test-medium-verbs-provider-md-on-ssd: false Skip-func-hw-test-medium-ucx-provider: true Skip-func-hw-test-large: true Signed-off-by: Gromadzki, Tomasz From bfa67a99d38847c1aab9c7e2ad197dd507672bba Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 Apr 2024 22:38:02 +0200 Subject: [PATCH 16/20] Test in typical (default) configuration. Required-githooks: true Signed-off-by: Gromadzki, Tomasz From b382b2442acc8d23311738c9e9670a4383fbeb29 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 9 Jul 2024 13:17:17 -0400 Subject: [PATCH 17/20] Just a skip list Skip-list-master: test_daos_rebuild_ec:DAOS-14982 test_daos_degraded_ec:DAOS-14982 Allow-unstable-test: true Skip-func-hw-test-large: false Priority: 2 Signed-off-by: Jan Michalski From bff247162978594cc77fc3a5c05368c9e8c1fb1f Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 9 Jul 2024 17:54:42 -0400 Subject: [PATCH 18/20] Update packaging Skip-list-master: test_daos_rebuild_ec:DAOS-14982 test_daos_degraded_ec:DAOS-14982 Allow-unstable-test: true Skip-func-hw-test-large: false Priority: 2 Signed-off-by: Jan Michalski --- packaging/Dockerfile.ubuntu.20.04 | 25 +++++++++++-------------- packaging/Makefile_packaging.mk | 6 ++++++ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index ec76bfd..4d05464 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -18,26 +18,23 @@ RUN if [ -n "$REPO_FILE_URL" ]; then \ true > ../sources.list && \ mv daos_ci-ubuntu20.04-artifactory.list.tmp \ daos_ci-ubuntu20.04-artifactory.list; \ + url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \ + else \ + url="https://downloads.linux.hpe.com/SDR/repo/ilorest/"; \ fi; \ cd -; \ - curl -f -O "$REPO_FILE_URL"esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import esad_repo.key; \ mkdir -p /usr/local/share/keyrings/; \ + curl -f -O "$url"GPG-KEY-hprest; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import GPG-KEY-hprest; \ gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/daos-stack-public.gpg; \ + --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ rm ./temp-keyring.gpg; \ - url_prefix=https://downloads.linux.hpe.com/SDR/; \ - for url in hpPublicKey2048.pub \ - hpPublicKey2048_key1.pub \ - hpePublicKey2048_key1.pub; do \ - curl -f -O "$url_prefix$url"; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import "$(basename $url)"; \ - done; \ + curl -f -O "$REPO_FILE_URL"esad_repo.key; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key; \ gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ - rm ./temp-keyring.gpg + --output /usr/local/share/keyrings/daos-stack-public.gpg # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 3201a22..75038c1 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -35,7 +35,11 @@ TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project REPO_NAME ?= $(NAME) +ifneq ($(CI_PR_REPOS),) +PR_REPOS ?= $(CI_PR_REPOS) +else PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos: *\(.*\)/\1/p') +endif LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') @@ -424,6 +428,8 @@ packaging_check: --exclude libfabric.spec \ --exclude Makefile \ --exclude README.md \ + --exclude SECURITY.md \ + --exclude LICENSE \ --exclude _topdir \ --exclude \*.tar.\* \ --exclude \*.code-workspace \ From c648e3591e17cdbc06653907398a9eaf83d6eff7 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Thu, 11 Jul 2024 11:52:09 -0400 Subject: [PATCH 19/20] Adjust skip lists Skip-list-master: test_daos_rebuild_ec:DAOS-14982 test_daos_degraded_ec:DAOS-14982 test_ms_failover:DAOS-16103 Skip-list-release/2.6: test_dfuse_daos_build_wb:DAOS-16168 test_osa_offline_reintegration_without_checksum:DAOS-15608 test_daos_extend_simple:DAOS-15640 Allow-unstable-test: true Skip-func-hw-test-large: false Priority: 2 Signed-off-by: Jan Michalski From 5ebd278a1c687ffe81429dd17b15ab7cdd264453 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Fri, 12 Jul 2024 14:55:05 -0400 Subject: [PATCH 20/20] Adjust skip lists Skip-list-master: test_daos_rebuild_ec:DAOS-14982 test_daos_degraded_ec:DAOS-14982 test_ms_failover:DAOS-16103 test_cart_rpc:DAOS-15989 Skip-list-release/2.6: test_dfuse_daos_build_wb:DAOS-16168 test_osa_offline_reintegration_without_checksum:DAOS-15608 test_daos_extend_simple:DAOS-15640 Allow-unstable-test: true Skip-func-hw-test-large: false Priority: 2 Signed-off-by: Jan Michalski