From c75654083d07916af7f8bf34927c8f7a0a915f2c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 5 Sep 2024 15:32:25 +1000 Subject: [PATCH 01/34] MDBF-787 Add OpenEuler --- constants.py | 2 ++ os_info.yaml | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/constants.py b/constants.py index 84441f5a..b1b8abb1 100644 --- a/constants.py +++ b/constants.py @@ -86,6 +86,7 @@ "aarch64-fedora-39", "aarch64-macos", "aarch64-macos-compile-only", + "aarch64-oe-2403", "aarch64-rhel-8", "aarch64-rhel-9", "aarch64-ubuntu-2004", @@ -102,6 +103,7 @@ "amd64-fedora-39", "amd64-fedora-40-valgrind", "amd64-freebsd-14", + "amd64-oe-2403", "amd64-rhel-7", "amd64-rhel-8", "amd64-rhel-9", diff --git a/os_info.yaml b/os_info.yaml index c5be56ea..77edcfa6 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -60,6 +60,12 @@ fedora-40: - amd64 - aarch64 type: rpm +oe-2403: + version_name: 24.03 + arch: + - amd64 + - aarch64 + type: rpm opensuse-15: version_name: 15 arch: From a23f3f38cee40cf8032ef717921eb3ce3a20b386 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 5 Sep 2024 15:32:44 +1000 Subject: [PATCH 02/34] MDBF-787: Add OpenEuler 24.03 as a rpm-autobake Add worker for openeuler 24.03. Rename container generation to mathc the same name. --- .github/workflows/build-centos.pip-based.yml | 2 +- constants.py | 4 ++-- master.cfg | 2 ++ os_info.yaml | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-centos.pip-based.yml b/.github/workflows/build-centos.pip-based.yml index 1dae2193..ad09c697 100644 --- a/.github/workflows/build-centos.pip-based.yml +++ b/.github/workflows/build-centos.pip-based.yml @@ -35,7 +35,7 @@ jobs: - image: docker.io/openeuler/openeuler:24.03-lts platforms: linux/amd64, linux/arm64/v8 - tag: oe2403 + tag: openeuler2403 nogalera: true uses: ./.github/workflows/bbw_build_container_template.yml diff --git a/constants.py b/constants.py index b1b8abb1..4e490ce5 100644 --- a/constants.py +++ b/constants.py @@ -86,7 +86,7 @@ "aarch64-fedora-39", "aarch64-macos", "aarch64-macos-compile-only", - "aarch64-oe-2403", + "aarch64-openeuler-2403", "aarch64-rhel-8", "aarch64-rhel-9", "aarch64-ubuntu-2004", @@ -103,7 +103,7 @@ "amd64-fedora-39", "amd64-fedora-40-valgrind", "amd64-freebsd-14", - "amd64-oe-2403", + "amd64-openeuler-2403", "amd64-rhel-7", "amd64-rhel-8", "amd64-rhel-9", diff --git a/master.cfg b/master.cfg index 532fd200..48ff4e09 100644 --- a/master.cfg +++ b/master.cfg @@ -379,6 +379,8 @@ for builder in master_config["builders"]: # From mariadb.org-tools/release/prep - under # Dirs for buildbot.mariadb.org if builder in [ + "aarch64-openeuler-2403", + "amd64-openeuler-2403", "ppc64le-debian-11", "s390x-ubuntu-2004", "s390x-rhel-8", diff --git a/os_info.yaml b/os_info.yaml index 77edcfa6..c6680415 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -60,7 +60,7 @@ fedora-40: - amd64 - aarch64 type: rpm -oe-2403: +openeuler-2403: version_name: 24.03 arch: - amd64 From 46c5899ff53788111d5f00acb9b0e115d518e817 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Fri, 20 Sep 2024 10:18:04 +0300 Subject: [PATCH 03/34] MDBF-796 - openeuler doesn't have compat rpm As per https://github.com/MariaDB/server/blob/8478a06cdbb7281e1cc2d507f18c985c736c2ffc/cmake/cpack_rpm.cmake#L374 MariaDB-compat won't be built for OpenEuler. --- utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index a54e472e..ba631a0d 100644 --- a/utils.py +++ b/utils.py @@ -475,7 +475,10 @@ def hasCompat(step): builderName = str(step.getProperty("buildername")) # For s390x and the listed distros there are no compat files - if any(builderName.find(sub) != -1 for sub in {"s390x", "fedora", "alma", "rocky"}): + if any( + builderName.find(sub) != -1 + for sub in {"s390x", "fedora", "alma", "rocky", "openeuler"} + ): return False if "rhel" in builderName or "centos" in builderName: return step.getProperty("rpm_type")[-1] in ["7", "8"] From 2e1ba1900b2bc7da4452f65ada412f431e2feb1b Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Thu, 19 Sep 2024 15:02:16 +0300 Subject: [PATCH 04/34] MDBF-794 - Link bintar binaries statically with libncurses Clients in our binary taballs are linked dynamically with libncurses, which causes a trouble on systems where it is no longer available As per MDBF-794 only centos7 and debian 10 bintar builders need to link statically with libncurses. - for debian 10: libncurses.a is already present in /scripts/local/lib/ and -DCURSES_NEED_NCURSES=1 flag is enough to link it statically. - for centos7: I've rebuilt the image and pushed manually a dev_ tag. libncurses.a was present on the system but not in the library path: /scripts/local/lib/ Tests available at: - debian10: https://buildbot.dev.mariadb.org/#/builders/187/builds/23 - centos7: https://buildbot.dev.mariadb.org/#/builders/172/builds/292 Reference CMAKE output line: -- Found Curses: /scripts/local/lib/libncurses.a --- master-docker-nonstandard/master.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 9284f096..14580539 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -1322,7 +1322,7 @@ f_bintar.addStep( "sh", "-c", util.Interpolate( - 'cmake . -DWITH_READLINE=1 -DBUILD_CONFIG=mysql_release -DCMAKE_C_FLAGS="%(kw:gnutls_no_signal)s" -DCMAKE_CXX_FLAGS="%(kw:gnutls_no_signal)s" -DWITH_SSL=bundled -DPLATFORM=linux-systemd && make -j%(kw:jobs)s package', + 'cmake . -DWITH_READLINE=1 -DCURSES_NEED_NCURSES=1 -DBUILD_CONFIG=mysql_release -DCMAKE_C_FLAGS="%(kw:gnutls_no_signal)s" -DCMAKE_CXX_FLAGS="%(kw:gnutls_no_signal)s" -DWITH_SSL=bundled -DPLATFORM=linux-systemd && make -j%(kw:jobs)s package', perf_schema=util.Property("perf_schema", default="YES"), build_type=util.Property("build_type", default="RelWithDebInfo"), jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"), From bce7a44f8548b38d1c8711a15fcf1e3504844dab Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 1 Aug 2024 13:27:29 +1000 Subject: [PATCH 05/34] MDBF-779 SLES/OpenSUSE 15.5 bb config Adjust images to 1505 notation as well. --- .github/workflows/build-opensuse.pip-based.yml | 2 +- .github/workflows/build-sles.pip-based.yml | 2 +- constants.py | 3 +++ master.cfg | 6 +++--- os_info.yaml | 14 ++++++++++++-- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-opensuse.pip-based.yml b/.github/workflows/build-opensuse.pip-based.yml index 87064da4..3627dbc8 100644 --- a/.github/workflows/build-opensuse.pip-based.yml +++ b/.github/workflows/build-opensuse.pip-based.yml @@ -29,7 +29,7 @@ jobs: include: - image: opensuse/leap:15.5 platforms: linux/amd64 - tag: opensuse15.5 + tag: opensuse1505 nogalera: true - image: opensuse/leap:15.6 diff --git a/.github/workflows/build-sles.pip-based.yml b/.github/workflows/build-sles.pip-based.yml index c6305d49..f470d5cb 100644 --- a/.github/workflows/build-sles.pip-based.yml +++ b/.github/workflows/build-sles.pip-based.yml @@ -30,7 +30,7 @@ jobs: include: - image: registry.suse.com/bci/bci-base:15.5 platforms: linux/amd64, linux/s390x - tag: sles15.5 + tag: sles1505 nogalera: true - image: registry.suse.com/bci/bci-base:15.6 diff --git a/constants.py b/constants.py index 4e490ce5..06801b3e 100644 --- a/constants.py +++ b/constants.py @@ -144,9 +144,12 @@ supportedPlatforms["10.6"] += [ "aarch64-ubuntu-2204", + "amd64-opensuse-1505", + "amd64-sles-1505", "amd64-ubuntu-2204", "ppc64le-ubuntu-2204", "s390x-ubuntu-2204", + "s390x-sles-1505", "x86-debian-12", ] diff --git a/master.cfg b/master.cfg index 48ff4e09..2563832d 100644 --- a/master.cfg +++ b/master.cfg @@ -393,9 +393,9 @@ for builder in master_config["builders"]: "amd64-debian-sid", "aarch64-debian-sid", "ppc64le-debian-sid", - "amd64-opensuse-15.5", - "amd64-sles-15.5", - "s390x-sles-15.5", + "amd64-opensuse-1505", + "amd64-sles-1505", + "s390x-sles-1505", ]: tags += ["release_packages"] c["builders"].append( diff --git a/os_info.yaml b/os_info.yaml index c6680415..e5c0ad55 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -65,9 +65,13 @@ openeuler-2403: arch: - amd64 - aarch64 +opensuse-1505: + version_name: 1505 + arch: + - amd64 type: rpm -opensuse-15: - version_name: 15 +opensuse-1506: + version_name: 1506 arch: - amd64 type: rpm @@ -109,6 +113,12 @@ sles-12: arch: - s390x type: rpm +sles-1505: + version_name: 15.5 + arch: + - amd64 + - s390x + type: rpm sles-15: version_name: 15 arch: From 0c23539f3302ddcfb665e702d46864ea8a256b91 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 18 Sep 2024 10:31:42 +1000 Subject: [PATCH 06/34] MDBF-779 Opensuse- rename 15 to 15.6 --- .github/workflows/build-opensuse.pip-based.yml | 2 +- constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-opensuse.pip-based.yml b/.github/workflows/build-opensuse.pip-based.yml index 3627dbc8..d599df3e 100644 --- a/.github/workflows/build-opensuse.pip-based.yml +++ b/.github/workflows/build-opensuse.pip-based.yml @@ -34,7 +34,7 @@ jobs: - image: opensuse/leap:15.6 platforms: linux/amd64 - tag: opensuse15 + tag: opensuse1506 nogalera: false uses: ./.github/workflows/bbw_build_container_template.yml diff --git a/constants.py b/constants.py index 06801b3e..f47e893c 100644 --- a/constants.py +++ b/constants.py @@ -167,7 +167,7 @@ "amd64-debian-12", "amd64-debian-12-debug-embedded", "amd64-fedora-40", - "amd64-opensuse-15", + "amd64-opensuse-1506", "amd64-sles-15", "amd64-ubuntu-2404", "ppc64le-ubuntu-2404", From 3a98dcce9f70f35bb6c376808a275b25d64b966b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 18 Sep 2024 10:32:38 +1000 Subject: [PATCH 07/34] Add OpenSUSE 15.6 to release builds --- master.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/master.cfg b/master.cfg index 2563832d..64aea4b4 100644 --- a/master.cfg +++ b/master.cfg @@ -394,6 +394,7 @@ for builder in master_config["builders"]: "aarch64-debian-sid", "ppc64le-debian-sid", "amd64-opensuse-1505", + "amd64-opensuse-1506", "amd64-sles-1505", "s390x-sles-1505", ]: From cfed57493a7f876a20c6a902510b9ac558784487 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 18 Sep 2024 10:36:20 +1000 Subject: [PATCH 08/34] MDBF-779 SLES-15.5 - temp disable on s390x - lack of hw --- os_info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_info.yaml b/os_info.yaml index e5c0ad55..f6b530c6 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -117,7 +117,7 @@ sles-1505: version_name: 15.5 arch: - amd64 - - s390x +# TEMP - currently short on hardware - s390x type: rpm sles-15: version_name: 15 From be6f672d8481b2ea32e10a755033dbc3a06a07a5 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 23 Sep 2024 16:59:11 +1000 Subject: [PATCH 09/34] MDBF-779 SLES-15 -> SLES-15.6 --- .github/workflows/build-sles.pip-based.yml | 2 +- os_info.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-sles.pip-based.yml b/.github/workflows/build-sles.pip-based.yml index f470d5cb..23b05985 100644 --- a/.github/workflows/build-sles.pip-based.yml +++ b/.github/workflows/build-sles.pip-based.yml @@ -35,7 +35,7 @@ jobs: - image: registry.suse.com/bci/bci-base:15.6 platforms: linux/amd64, linux/s390x - tag: sles15 + tag: sles1506 nogalera: false uses: ./.github/workflows/bbw_build_container_template.yml diff --git a/os_info.yaml b/os_info.yaml index f6b530c6..8a8e9341 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -119,8 +119,8 @@ sles-1505: - amd64 # TEMP - currently short on hardware - s390x type: rpm -sles-15: - version_name: 15 +sles-1506: + version_name: 15.6 arch: - amd64 - s390x From c9806579430196667add33366c7a1ed4cbbd2f2f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 23 Sep 2024 17:16:55 +1000 Subject: [PATCH 10/34] opensuse - 15.X name in os_info to match install/upgrade scripts --- os_info.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/os_info.yaml b/os_info.yaml index 8a8e9341..978288ce 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -66,12 +66,12 @@ openeuler-2403: - amd64 - aarch64 opensuse-1505: - version_name: 1505 + version_name: 15.5 arch: - amd64 type: rpm opensuse-1506: - version_name: 1506 + version_name: 15.6 arch: - amd64 type: rpm From 877fafc5bfc42fa5c12bacf9b3c7b271cb2d918e Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 23 Sep 2024 11:52:30 +0300 Subject: [PATCH 11/34] Fix type: rpm for openeuler after rebase --- os_info.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/os_info.yaml b/os_info.yaml index 978288ce..7ee574a8 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -65,6 +65,7 @@ openeuler-2403: arch: - amd64 - aarch64 + type: rpm opensuse-1505: version_name: 15.5 arch: From 6477da7e514d7bc6081840345fb907fa67017cff Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 23 Sep 2024 13:08:30 +0300 Subject: [PATCH 12/34] Fix for triggering sles 1506 on 10.11 --- constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/constants.py b/constants.py index f47e893c..828e020e 100644 --- a/constants.py +++ b/constants.py @@ -168,10 +168,10 @@ "amd64-debian-12-debug-embedded", "amd64-fedora-40", "amd64-opensuse-1506", - "amd64-sles-15", + "amd64-sles-1506", "amd64-ubuntu-2404", "ppc64le-ubuntu-2404", - "s390x-sles-15", + "s390x-sles-1506", "s390x-ubuntu-2404", "ppc64le-debian-12", ] From 64d5aa7d6c6b55b4628ea5fd1cc63e2bce3d28a2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 16 Sep 2024 12:46:10 +1000 Subject: [PATCH 13/34] MDBF-795: Add Fedora 41 (x86_64/Aarch64) Fedora 41 is including MariaDB 10.11 so use that as the minimum version. --- .github/workflows/build-fedora-based.yml | 4 ++++ constants.py | 2 ++ os_info.yaml | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/build-fedora-based.yml b/.github/workflows/build-fedora-based.yml index 48149692..61094275 100644 --- a/.github/workflows/build-fedora-based.yml +++ b/.github/workflows/build-fedora-based.yml @@ -31,6 +31,10 @@ jobs: platforms: linux/amd64, linux/arm64/v8 nogalera: false + - image: fedora:41 + platforms: linux/amd64, linux/arm64/v8 + nogalera: true + - image: fedora:40 platforms: linux/amd64 tag: fedora40-valgrind diff --git a/constants.py b/constants.py index 828e020e..b922e021 100644 --- a/constants.py +++ b/constants.py @@ -163,10 +163,12 @@ supportedPlatforms["10.11"] = [ "aarch64-debian-12", "aarch64-fedora-40", + "aarch64-fedora-41", "aarch64-ubuntu-2404", "amd64-debian-12", "amd64-debian-12-debug-embedded", "amd64-fedora-40", + "amd64-fedora-41", "amd64-opensuse-1506", "amd64-sles-1506", "amd64-ubuntu-2404", diff --git a/os_info.yaml b/os_info.yaml index 7ee574a8..1e825fed 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -60,6 +60,12 @@ fedora-40: - amd64 - aarch64 type: rpm +fedora-41: + version_name: 41 + arch: + - amd64 + - aarch64 + type: rpm openeuler-2403: version_name: 24.03 arch: From ffc4ab6ed55302a2ab1b3cbd9616bf149bacb5ca Mon Sep 17 00:00:00 2001 From: Faustin Lammler Date: Mon, 30 Sep 2024 11:14:21 +0200 Subject: [PATCH 14/34] Fix CS testing, rpm are not pulled --- scripts/rpm-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh index 841d51ab..7826168f 100755 --- a/scripts/rpm-install.sh +++ b/scripts/rpm-install.sh @@ -74,7 +74,7 @@ case "$systemdCapability" in esac sudo mariadb -e 'drop database if exists test; create database test; use test; create table t(a int primary key) engine=innodb; insert into t values (1); select * from t; drop table t;' -if find rpms/*.rpm | grep -qi columnstore; then +if echo "$pkg_list" | grep -qi columnstore; then sudo mariadb --verbose -e "create database cs; use cs; create table cs.t_columnstore (a int, b char(8)); insert into cs.t_columnstore select seq, concat('val',seq) from seq_1_to_10; select * from cs.t_columnstore" sudo systemctl restart mariadb sudo mariadb --verbose -e "select * from cs.t_columnstore; update cs.t_columnstore set b = 'updated'" From 97134c3f9f5df37cb480911a48d6fe7348cbb8a9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 30 Sep 2024 12:19:46 +1000 Subject: [PATCH 15/34] Add asio-devel to rpm based bb-worker images Centos based have asio-devel in EPEL. SLES is omitted as it requires a subpackage module repository to be enabled. Doing so would increase the package dependencies when using the built packages. https://packagehub.suse.com/packages/asio/ (SLE-Module-PackageHub-Subpackages-Module ) OpenSUSE also has asio-devel but adds a complicated path in with the openssl libraries and boost that we'd be influcting on users if they used a non-bundled version. On RHEL - asio-devel in ubi9 EPEL but not 8. --- ci_build_images/centos.Dockerfile | 1 + ci_build_images/fedora.Dockerfile | 1 + ci_build_images/rhel.Dockerfile | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci_build_images/centos.Dockerfile b/ci_build_images/centos.Dockerfile index 1a92eb6a..489799c4 100644 --- a/ci_build_images/centos.Dockerfile +++ b/ci_build_images/centos.Dockerfile @@ -59,6 +59,7 @@ RUN dnf -y install 'dnf-command(config-manager)' \ # not sure if needed \ # perl \ ${extra} \ + asio-devel \ bzip2 \ bzip2-devel \ ccache \ diff --git a/ci_build_images/fedora.Dockerfile b/ci_build_images/fedora.Dockerfile index d6387144..8431b304 100644 --- a/ci_build_images/fedora.Dockerfile +++ b/ci_build_images/fedora.Dockerfile @@ -19,6 +19,7 @@ RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf \ && dnf -y builddep mariadb-server \ && dnf -y install \ @development-tools \ + asio-devel \ buildbot-worker \ bzip2 \ bzip2-devel \ diff --git a/ci_build_images/rhel.Dockerfile b/ci_build_images/rhel.Dockerfile index f24562b8..d3415b21 100644 --- a/ci_build_images/rhel.Dockerfile +++ b/ci_build_images/rhel.Dockerfile @@ -21,7 +21,7 @@ RUN --mount=type=secret,id=rhel_orgid,target=/run/secrets/rhel_orgid \ && case $BASE_IMAGE in \ ubi9) \ v=9; \ - extra="fmt-devel buildbot-worker"; \ + extra="asio-devel buildbot-worker fmt-devel"; \ ;; \ ubi8) \ v=8; \ From 59c5bc5bd4f66e4a53a0fb1695d09be7e7451566 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 30 Sep 2024 16:16:51 +0300 Subject: [PATCH 16/34] OpenSuse / Sles 15.x - do not need MariaDB-compat --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index ba631a0d..69450764 100644 --- a/utils.py +++ b/utils.py @@ -477,7 +477,7 @@ def hasCompat(step): # For s390x and the listed distros there are no compat files if any( builderName.find(sub) != -1 - for sub in {"s390x", "fedora", "alma", "rocky", "openeuler"} + for sub in {"s390x", "fedora", "alma", "rocky", "openeuler", "suse15", "sles15"} ): return False if "rhel" in builderName or "centos" in builderName: From 5140e9cbdb6bb89d43074df29caead391d78df41 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 29 Sep 2024 20:20:52 +0300 Subject: [PATCH 17/34] MDBF-800 Extend minor upgrade tests to ensure transition from old bb Parts 1 and 2: Switching minor upgrade into "all" mode and extending it with additional checks (ldd, requirements, capabilities) --- master-libvirt/master.cfg | 2 +- scripts/bash_lib.sh | 142 +++++++++++++++++++++++++++++++++++++- scripts/deb-upgrade.sh | 45 ++---------- scripts/rpm-upgrade.sh | 29 ++++---- 4 files changed, 161 insertions(+), 57 deletions(-) diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index a1264a24..f8b2b78b 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -367,7 +367,7 @@ for builder_name in builders_install: "dist_name": os_name, "version_name": os_info[os_info_name]["version_name"], "arch": build_arch, - "test_mode": "server", + "test_mode": "all", "test_type": "minor", "BB_CI": True, "artifactsURL": artifactsURL, diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 3fafe9d7..f437ca33 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -488,9 +488,18 @@ control_mariadb_server() { } store_mariadb_server_info() { +# We need sudo here because the mariadb local root configured this way, not because we want special write permissions for the resulting file. +# pre-commit check has an issue with it, so instead of adding an exception before each line, we add a piped sort, which should be bogus, sudo mariadb --skip-column-names -e "select @@version" | awk -F'-' '{ print $1 }' >"/tmp/version.$1" - sudo mariadb --skip-column-names -e "select engine, support, transactions, savepoints from information_schema.engines" | sort >"/tmp/engines.$1" - sudo mariadb --skip-column-names -e "select plugin_name, plugin_status, plugin_type, plugin_library, plugin_license from information_schema.all_plugins" | sort >"/tmp/plugins.$1" + sudo mariadb --skip-column-names -e "select engine, support, transactions, savepoints from information_schema.engines order by engine" | sort > "./engines.$1" + sudo mariadb --skip-column-names -e "select plugin_name, plugin_status, plugin_type, plugin_library, plugin_license \ + from information_schema.all_plugins order by plugin_name" | sort > "./plugins.$1" + sudo mariadb --skip-column-names -e "select 'Stat' t, variable_name name, variable_value val + from information_schema.global_status where variable_name like '%have%' \ + union \ + select 'Vars' t, variable_name name, variable_value val \ + from information_schema.global_variables where variable_name like '%have%' \ + order by t, name" | sort > "./capabilities.$1" } check_upgraded_versions() { @@ -528,4 +537,133 @@ check_upgraded_versions() { bb_log_err "or if it is a development tree that is based on an old version" exit 1 fi + + res=0 + engines_disappeared_or_changed=$(comm -23 ./engines.old ./engines.new | wc -l) + if ((engines_disappeared_or_changed != 0)); then + bb_log_err "the lists of engines in the old and new installations differ" + diff -u ./engines.old ./engines.new + res=1 + fi + if [[ $test_type == "minor" ]]; then + + # We are using temporary files for further comparison, because we will need to make + # some adjustments to them to avoid false positives, but we want original files + # to be stored by buildbot + for f in ldd-main ldd-columnstore reqs-main reqs-columnstore plugins capabilities; do + if [ -e "./${f}.old" ] ; then + cp "./${f}.old" "./${f}.old.cmp" + cp "./${f}.new" "./${f}.new.cmp" + fi + done + + # Permanent adjustments + sed -i '/libstdc++.so.6(GLIBCXX/d;/libstdc++.so.6(CXXABI/d' ./reqs-*.cmp + # Don't compare subversions of libsystemd.so (e.g. libsystemd.so.0(LIBSYSTEMD_227)(64bit)) + sed -i '/libsystemd.so.[0-9]*(LIBSYSTEMD_/d' ./reqs-*.cmp + # Ignore shells, the number of which always changes + sed -i '/^\/bin\/sh$/d' ./reqs-*.cmp + + # Here is the place for temporary adjustments, + # when legitimate changes in dependencies happen between minor versions. + # The adjustments should be done to .cmp files, and removed after the release + # + # End of temporary adjustments + + bb_log_info "Comparing old and new ldd output for installed binaries" + # We are not currently comparing it for Columnstore, too much trouble for nothing + if ! diff -U1000 ./ldd-main.old.cmp ./ldd-main.new.cmp | (grep -E '^[-+]|^ =' || true) ; then + bb_log_error "Found changes in ldd output on installed binaries" + res=1 + else + bb_log_info "OK" + fi + + bb_log_info "Comparing old and new package requirements" + # We are not currently comparing it for Columnstore, but we may need to in the future + if ! diff -U150 ./reqs-main.old.cmp ./reqs-main.new.cmp | (grep -E '^ [^ ]|^[-+]' || true) ; then + bb_log_error "Found changes in package requirements" + res=1 + else + bb_log_info "OK" + fi + + bb_log_info "Comparing old and new server capabilities ('%have%' variables)" + if ! diff -u ./capabilities.old.cmp ./capabilities.new.cmp ; then + bb_log_error "ERROR: found changes in server capabilities" + res=1 + else + bb_log_info "OK" + fi + + echo "Comparing old and new plugins" + # Columnstore version changes often, we'll ignore it + grep -i columnstore ./plugins.old.cmp > /tmp/columnstore.old + grep -i columnstore ./plugins.new.cmp > /tmp/columnstore.new + if ! diff -u /tmp/columnstore.old /tmp/columnstore.new ; then + bb_log_warn "Columnstore version changed. Downgraded to a warning as it is usually intentional" + sed -i '/COLUMNSTORE/d;/Columnstore/d' ./plugins.*.cmp + fi + + if ! diff -u ./plugins.old.cmp ./plugins.new.cmp ; then + bb_log_error "Found changes in available plugins" + res=1 + else + bb_log_info "OK" + fi + fi + exit $res +} + +# Collect package requirements and ldd for all binaries included into packages. +# Expects "old" and "new" as an argument, and +# $package_list to be set by the time of execution +collect_dependencies() { + old_or_new=$1 + case "$package_list" in + *.rpm) + pkgtype=rpm + if rpm --noartifact ; then + # --noartifact option is needed to avoid build-id in rpm -lp output + NOARTIFACT="--noartifact" + fi + ;; + *.deb) + pkgtype=deb + ;; + *) + bb_log_err "unknown package type in the package list $package_list" + exit 1 + ;; + esac + + rm -f "./ldd-*.${old_or_new}" "./reqs-*.${old_or_new}" + bb_log_info "Collecting dependencies for the ${old_or_new} server" + + for p in ${package_list} ${spider_package_list} ; do + if [[ "$p" =~ columnstore ]] ; then + suffix="columnstore" + else + suffix="main" + fi + echo "-----------------" >> "./reqs-${suffix}.${old_or_new}" + echo "$p:" >> "./reqs-${suffix}.${old_or_new}" + if [ "$pkgtype" == "rpm" ] ; then + rpm -q -R "$p" | awk '{print $1}' | grep -vE '/usr/bin/env|/usr/bin/bash' >> "./reqs-${suffix}.${old_or_new}" + # NOARTIFACT is either an empty string or --noartifact option, it won't be globbed or word-splitted + # shellcheck disable=SC2086 + filelist=$(rpm $NOARTIFACT -ql "$p" | sort) + else + # We need sudo here for the apt-cache command, not for redirection + # shellcheck disable=SC2024 + sudo apt-cache depends "$p" --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances >> "./reqs-${suffix}.${old_or_new}" + filelist=$(dpkg-query -L "$p" | sort) + fi + echo "====== Package $p" >> "./ldd-${suffix}.${old_or_new}" + for f in $filelist ; do + sudo ldd "$f" > /dev/null 2>&1 || continue + echo "=== $f" >> "./ldd-${suffix}.${old_or_new}" + sudo ldd "$f" | sort | sed 's/(.*)//' >> "./ldd-${suffix}.${old_or_new}" + done + done } diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index a1cf8335..969162c8 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -146,42 +146,10 @@ fi # Check that the server is functioning and create some structures check_mariadb_server_and_create_structures -# Store information about server version and available plugins/engines before upgrade -if [[ $test_mode == "all" ]]; then - # Due to MDEV-14560, we have to restart the server to get the full list of engines - # MDEV-14560 is fixed in 10.2 - if [ "${prev_major_version}" != 10.2 ]; then - control_mariadb_server restart - fi -fi - +# Store information about the server before upgrade +collect_dependencies old store_mariadb_server_info old -# //TEMP todo, see below -# # Store dependency information for old binaries/libraries: -# # - names starting with "mysql*" in the directory where mysqld is located; -# # - names starting with "mysql*" in the directory where mysql is located; -# # - everything in the plugin directories installed by any MariaDB packages -# set +x -# for i in $(sudo which mysqld | sed -e 's/mysqld$/mysql\*/') $(which mysql | sed -e 's/mysql$/mysql\*/') $(dpkg-query -L $(dpkg -l | grep mariadb | awk '{print $2}' | xargs) | grep -v 'mysql-test' | grep -v '/debug/' | grep '/plugin/' | sed -e 's/[^\/]*$/\*/' | sort | uniq | xargs); do -# echo "=== $i" -# ldd "$i" | sort | sed 's/(.*)//' -# done >/buildbot/ldd.old -# set -x - -# # setup repository -# sudo sh -c "echo 'deb [trusted=yes] https://deb.mariadb.org/$master_branch/$dist_name $version_name main' >/etc/apt/sources.list.d/galera-test-repo.list" -# # Update galera-test-repo.list to point at either the galera-3 or galera-4 test repo -# //TEMP -# case "$branch" in -# *10.[1-3]*) -# sudo sed -i 's/repo/repo3/' /etc/apt/sources.list.d/galera-test-repo.list -# ;; -# *10.[4-9]*) -# sudo sed -i 's/repo/repo4/' /etc/apt/sources.list.d/galera-test-repo.list -# ;; -# esac - if [[ $test_mode == "deps" ]]; then # For the dependency check, only keep the local repo //TEMP what does this do??? sudo sh -c "grep -iE 'deb .*file|deb-src .*file' /etc/apt/sources.list.backup >/etc/apt/sources.list" @@ -192,10 +160,7 @@ else fi sudo rm /etc/apt/preferences.d/release -# We also need official mirror for dependencies not available in BB artifacts -# (Galera) //TEMP seems to not be necessary anymore (galera packages build in -# artifacts?) -# deb_setup_mariadb_mirror "$master_branch" +deb_setup_bb_galera_artifacts_mirror deb_setup_bb_artifacts_mirror apt_get_update @@ -249,8 +214,8 @@ fi # available check_mariadb_server_and_verify_structures -# Store information about server version and available plugins/engines after -# upgrade +# Store information about the server after upgrade +collect_dependencies new store_mariadb_server_info new # //TEMP what needs to be done here? diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh index c91ff20e..be1d294c 100755 --- a/scripts/rpm-upgrade.sh +++ b/scripts/rpm-upgrade.sh @@ -44,11 +44,11 @@ rpm_setup_mariadb_mirror "$prev_major_version" case $test_mode in all) # retrieve full package list from repo - pkg_list=$(rpm_repoquery) || + package_list=$(rpm_repoquery) || bb_log_err "unable to retrieve package list from repository" ;; server) - pkg_list="MariaDB-server MariaDB-client" + package_list="MariaDB-server MariaDB-client" ;; *) bb_log_err "unknown test mode ($test_mode)" @@ -59,18 +59,18 @@ esac # case $test_mode in # all | deps | columnstore) # if [[ $test_mode == "all" ]]; then -# if echo "$pkg_list" | grep -qi columnstore; then +# if echo "$package_list" | grep -qi columnstore; then # bb_log_warn "due to MCOL-4120 and other issues, Columnstore upgrade will be tested separately" # fi -# package_list=$(echo "$pkg_list" | grep MariaDB | +# package_list=$(echo "$package_list" | grep MariaDB | # grep -viE 'galera|columnstore' | sed -e 's///' | # sed -e 's/<\/name>//' | sort -u | xargs) # elif [[ $test_mode == "deps" ]]; then -# package_list=$(echo "$pkg_list" | +# package_list=$(echo "$package_list" | # grep -iE 'MariaDB-server|MariaDB-test|MariaDB-client|MariaDB-common|MariaDB-compat' | # sed -e 's///' | sed -e 's/<\/name>//' | sort -u | xargs) # elif [[ $test_mode == "columnstore" ]]; then -# if ! echo "$pkg_list" | grep -q columnstore; then +# if ! echo "$package_list" | grep -q columnstore; then # bb_log_warn "columnstore was not found in the released packages, the test will not be run" # exit # fi @@ -78,7 +78,7 @@ esac # fi # if [[ $arch == ppc* ]]; then -# package_list=$(echo "$pkg_list" | xargs -n1 | sed -e 's/MariaDB-compat//gi' | xargs) +# package_list=$(echo "$package_list" | xargs -n1 | sed -e 's/MariaDB-compat//gi' | xargs) # fi # ;; # server) @@ -90,7 +90,7 @@ esac # ;; # esac -bb_log_info "Package_list: $pkg_list" +bb_log_info "Package_list: $package_list" # # //TEMP this needs to be implemented once we have SLES VM in new BB # # Prepare yum/zypper configuration for installation of the last release @@ -149,7 +149,7 @@ bb_log_info "Package_list: $pkg_list" sudo "$pkg_cmd" clean all # Install previous release -echo "$pkg_list" | xargs sudo "$pkg_cmd" -y install || +echo "$package_list" | xargs sudo "$pkg_cmd" -y install || bb_log_err "installation of a previous release failed, see the output above" #fi @@ -179,8 +179,8 @@ fi check_mariadb_server_and_create_structures -# Store information about server version and available plugins/engines before -# upgrade +# Store information about the server before upgrade +collect_dependencies old store_mariadb_server_info old # If the tested branch has the same version as the public repository, @@ -252,10 +252,10 @@ rpm_setup_bb_galera_artifacts_mirror rpm_setup_bb_artifacts_mirror if [[ $test_type == "major" ]]; then # major upgrade (remove then install) - echo "$pkg_list" | xargs sudo "$pkg_cmd" -y install + echo "$package_list" | xargs sudo "$pkg_cmd" -y install else # minor upgrade (upgrade works) - echo "$pkg_list" | xargs sudo "$pkg_cmd" -y upgrade + echo "$package_list" | xargs sudo "$pkg_cmd" -y upgrade fi # set +e @@ -297,7 +297,8 @@ set +e # Check that the server is functioning and previously created structures are available check_mariadb_server_and_verify_structures -# Store information about server version and available plugins/engines after upgrade +# Store information about the server after upgrade +collect_dependencies new store_mariadb_server_info new # # Dependency information for new binaries/libraries From f4e68db512f96dcf52aede0853d9057873487186 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Mon, 30 Sep 2024 22:13:01 +0300 Subject: [PATCH 18/34] MDBF-800 Extend package minor upgrade tests to ensure transition Post-review fixes and cosmetic changes --- scripts/bash_lib.sh | 67 +++++++++++++++++++++--------------------- scripts/deb-upgrade.sh | 4 +-- scripts/rpm-upgrade.sh | 5 ++-- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index f437ca33..2b9e9751 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -221,7 +221,7 @@ rpm_repoquery() { wait_for_mariadb_upgrade() { res=1 for i in {1..20}; do - if pgrep -if 'mysql_upgrade|mysqlcheck|mysqlrepair|mysqlanalyze|mysqloptimize|mariadb-upgrade|mariadb-check'; then + if pgrep -ifa 'mysql_upgrade|mysqlcheck|mysqlrepair|mysqlanalyze|mysqloptimize|mariadb-upgrade|mariadb-check'; then bb_log_info "wait for mysql_upgrade to finish ($i)" sleep 5 else @@ -532,17 +532,20 @@ check_upgraded_versions() { fi fi if diff -u /tmp/version.old /tmp/version.new; then - bb_log_err "server version has not changed after upgrade" - bb_log_err "it can be a false positive if we forgot to bump version after release," + bb_log_err "Server version has not changed after upgrade." + bb_log_err "It can be a false positive if we forgot to bump version after release" bb_log_err "or if it is a development tree that is based on an old version" exit 1 fi res=0 + errors="" engines_disappeared_or_changed=$(comm -23 ./engines.old ./engines.new | wc -l) + set +x if ((engines_disappeared_or_changed != 0)); then - bb_log_err "the lists of engines in the old and new installations differ" + bb_log_err "Found changes in the list of engines:" diff -u ./engines.old ./engines.new + errors="$errors engines," res=1 fi if [[ $test_type == "minor" ]]; then @@ -570,30 +573,34 @@ check_upgraded_versions() { # # End of temporary adjustments + set -o pipefail bb_log_info "Comparing old and new ldd output for installed binaries" # We are not currently comparing it for Columnstore, too much trouble for nothing if ! diff -U1000 ./ldd-main.old.cmp ./ldd-main.new.cmp | (grep -E '^[-+]|^ =' || true) ; then - bb_log_error "Found changes in ldd output on installed binaries" + bb_log_err "Found changes in ldd output, see above" + errors="$errors ldd," res=1 else - bb_log_info "OK" + bb_log_info "ldd OK" fi bb_log_info "Comparing old and new package requirements" # We are not currently comparing it for Columnstore, but we may need to in the future if ! diff -U150 ./reqs-main.old.cmp ./reqs-main.new.cmp | (grep -E '^ [^ ]|^[-+]' || true) ; then - bb_log_error "Found changes in package requirements" + bb_log_err "Found changes in package requirements, see above" + errors="$errors requirements," res=1 else - bb_log_info "OK" + bb_log_info "Package requirements OK" fi bb_log_info "Comparing old and new server capabilities ('%have%' variables)" if ! diff -u ./capabilities.old.cmp ./capabilities.new.cmp ; then - bb_log_error "ERROR: found changes in server capabilities" + bb_log_err "Found changes in server capabilities, see above" + errors="$errors capabilities," res=1 else - bb_log_info "OK" + bb_log_info "Capabilities OK" fi echo "Comparing old and new plugins" @@ -606,40 +613,33 @@ check_upgraded_versions() { fi if ! diff -u ./plugins.old.cmp ./plugins.new.cmp ; then - bb_log_error "Found changes in available plugins" + bb_log_err "Found changes in available plugins, see above" + errors="$errors plugins," res=1 else - bb_log_info "OK" + bb_log_info "Plugins OK" fi fi + if [ -n "$errors" ] ; then + bb_log_err "Problems were found with:$errors see above output for details" + fi exit $res } # Collect package requirements and ldd for all binaries included into packages. -# Expects "old" and "new" as an argument, and -# $package_list to be set by the time of execution +# Expects "old" and "new" as the first argument, +# "deb" or "rpm" as the second argument, +# and $package_list to be set by the time of execution collect_dependencies() { + [[ $test_type == "minor" ]] || return old_or_new=$1 - case "$package_list" in - *.rpm) - pkgtype=rpm - if rpm --noartifact ; then - # --noartifact option is needed to avoid build-id in rpm -lp output - NOARTIFACT="--noartifact" - fi - ;; - *.deb) - pkgtype=deb - ;; - *) - bb_log_err "unknown package type in the package list $package_list" - exit 1 - ;; - esac - - rm -f "./ldd-*.${old_or_new}" "./reqs-*.${old_or_new}" + pkgtype=$2 + if rpm --noartifact ; then + # --noartifact option is needed to avoid build-id in rpm -lp output + NOARTIFACT="--noartifact" + fi bb_log_info "Collecting dependencies for the ${old_or_new} server" - + set +x for p in ${package_list} ${spider_package_list} ; do if [[ "$p" =~ columnstore ]] ; then suffix="columnstore" @@ -666,4 +666,5 @@ collect_dependencies() { sudo ldd "$f" | sort | sed 's/(.*)//' >> "./ldd-${suffix}.${old_or_new}" done done + set -x } diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index 969162c8..36fc3544 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -147,7 +147,7 @@ fi check_mariadb_server_and_create_structures # Store information about the server before upgrade -collect_dependencies old +collect_dependencies old deb store_mariadb_server_info old if [[ $test_mode == "deps" ]]; then @@ -215,7 +215,7 @@ fi check_mariadb_server_and_verify_structures # Store information about the server after upgrade -collect_dependencies new +collect_dependencies new deb store_mariadb_server_info new # //TEMP what needs to be done here? diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh index be1d294c..c34cb880 100755 --- a/scripts/rpm-upgrade.sh +++ b/scripts/rpm-upgrade.sh @@ -180,7 +180,7 @@ fi check_mariadb_server_and_create_structures # Store information about the server before upgrade -collect_dependencies old +collect_dependencies old rpm store_mariadb_server_info old # If the tested branch has the same version as the public repository, @@ -292,13 +292,12 @@ fi if [[ $major_version == "$development_branch" ]] || [[ $test_type == "major" ]]; then sudo mariadb-upgrade fi -set +e # Check that the server is functioning and previously created structures are available check_mariadb_server_and_verify_structures # Store information about the server after upgrade -collect_dependencies new +collect_dependencies new rpm store_mariadb_server_info new # # Dependency information for new binaries/libraries From 88a75b3258e60f782361eefde7f8b4610c046dec Mon Sep 17 00:00:00 2001 From: Faustin Lammler Date: Tue, 1 Oct 2024 09:27:29 +0200 Subject: [PATCH 19/34] Support for PPC on Debian 11 is EOL See: https://lists.debian.org/debian-announce/2024/msg00004.html --- .github/workflows/build-debian-based.yml | 2 +- constants.py | 1 - master.cfg | 1 - os_info.yaml | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-debian-based.yml b/.github/workflows/build-debian-based.yml index 8af41015..c0f14e03 100644 --- a/.github/workflows/build-debian-based.yml +++ b/.github/workflows/build-debian-based.yml @@ -25,7 +25,7 @@ jobs: matrix: include: - image: debian:11 - platforms: linux/amd64, linux/arm64/v8, linux/ppc64le + platforms: linux/amd64, linux/arm64/v8 branch: 10.11 nogalera: false diff --git a/constants.py b/constants.py index b922e021..fab85921 100644 --- a/constants.py +++ b/constants.py @@ -118,7 +118,6 @@ "amd64-windows-packages", "ppc64be-aix-71", "ppc64le-centos-stream9", - "ppc64le-debian-11", "ppc64le-rhel-8", "ppc64le-rhel-9", "ppc64le-ubuntu-2004", diff --git a/master.cfg b/master.cfg index 64aea4b4..770f1cab 100644 --- a/master.cfg +++ b/master.cfg @@ -381,7 +381,6 @@ for builder in master_config["builders"]: if builder in [ "aarch64-openeuler-2403", "amd64-openeuler-2403", - "ppc64le-debian-11", "s390x-ubuntu-2004", "s390x-rhel-8", "s390x-sles-12", diff --git a/os_info.yaml b/os_info.yaml index 1e825fed..6caba015 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -30,7 +30,6 @@ debian-11: arch: - amd64 - aarch64 - - ppc64le type: deb debian-12: version_name: bookworm From a1542ccd72d460b85550924dea417ef32bb5c3d3 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 30 Sep 2024 16:37:49 +0300 Subject: [PATCH 20/34] Prevent MariaDB hanging service on Windows During the "Compile step" - create_initial_db.cmake runs a bootstrap as a sanity check to avoid spending time on test runs. It can happen that some changes on the server may cause a loop during the bootstrap process. Buildbot will terminate the parent process (dojob) when it reaches a timeout, but the MariaDB service will remain up causing new builds to fail. https://buildbot.mariadb.org/#/builders/234/builds/32911 Modified the stop_processes step so handle this. --- master-nonlatent/master.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index a277b5c5..b206844a 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -167,7 +167,7 @@ f_windows = util.BuildFactory() f_windows.addStep( steps.ShellCommand( name="stop_processes", - command="taskkill /im mysqltest /f || ver>nul", + command="taskkill /im mariadb-test.exe /im mariadbd.exe /f || ver>nul", alwaysRun=True, ) ) @@ -340,7 +340,7 @@ f_windows_msi = util.BuildFactory() f_windows_msi.addStep( steps.ShellCommand( name="stop_processes", - command="taskkill /im mysqltest /f || ver>nul", + command="taskkill /im mariadb-test.exe /im mariadbd.exe /f || ver>nul", alwaysRun=True, ) ) From f8e5d7bb24b2a0b8760c1c6c87a83b153b33576b Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 7 Oct 2024 16:23:04 +0300 Subject: [PATCH 21/34] MDBF-802 - deb-upgrade.sh does not fetch Packages file from releases Use ( deb | archive ).mariadb.org to download the Packages file. This file is mandatory for both the 'All' and 'Columnstore' test types, exit with error if it is not found. Apply uniq to remove duplicate values in the package list and sort the list for readability. --- scripts/deb-upgrade.sh | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index 36fc3544..16a64e69 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -29,16 +29,31 @@ bb_log_info "Current test mode: $test_mode" set -x +# Prepare apt repository configuration for installation of the previous major +# version +deb_setup_mariadb_mirror "$prev_major_version" + +get_packages_file_mirror() { + set -u + if ! wget "$baseurl/$dist_name/dists/$version_name/main/binary-$(deb_arch)/Packages" + then + bb_log_err "Could not find the 'Packages' file for a previous version." + exit 1 + fi + set +u +} + # Define the list of packages to install/upgrade case $test_mode in all) + get_packages_file_mirror if grep -qi columnstore Packages; then bb_log_warn "due to MCOL-4120 (Columnstore leaves the server shut down) and other bugs Columnstore upgrade is tested separately" fi - package_list=$(grep "^Package:" Packages | grep -vE 'galera|spider|columnstore' | awk '{print $2}' | xargs) + package_list=$(grep "^Package:" Packages | grep -vE 'galera|spider|columnstore' | awk '{print $2}' | sort | uniq | xargs) if grep -qi spider Packages; then bb_log_warn "due to MDEV-14622 Spider will be installed separately after the server" - spider_package_list=$(grep "^Package:" Packages | grep 'spider' | awk '{print $2}' | xargs) + spider_package_list=$(grep "^Package:" Packages | grep 'spider' | awk '{print $2}' | sort | uniq | xargs) fi if grep -si tokudb Packages; then # For the sake of installing TokuDB, disable hugepages @@ -52,6 +67,7 @@ case $test_mode in package_list=mariadb-server ;; columnstore) + get_packages_file_mirror if ! grep columnstore Packages >/dev/null; then bb_log_warn "Columnstore was not found in packages, the test will not be run" exit @@ -59,7 +75,7 @@ case $test_mode in bb_log_warn "Columnstore isn't necessarily built on Sid, the test will be skipped" exit fi - package_list="mariadb-server "$(grep "^Package:" Packages | grep 'columnstore' | awk '{print $2}' | xargs) + package_list="mariadb-server "$(grep "^Package:" Packages | grep 'columnstore' | awk '{print $2}' | sort | uniq | xargs) ;; *) bb_log_err "unknown test mode: $test_mode" @@ -68,10 +84,6 @@ case $test_mode in esac bb_log_info "Package_list: $package_list" -# Prepare apt repository configuration for installation of the previous major -# version -deb_setup_mariadb_mirror "$prev_major_version" - # We need to pin directory to ensure that installation happens from MariaDB # repo rather than from the default distro repo if [[ $prev_major_version == "10.2" ]]; then From e340a430870d45242e2e832f21d44c28393c30cc Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 6 Oct 2024 22:22:43 +0300 Subject: [PATCH 22/34] MDBF-800 Extend package minor upgrade tests to ensure transition from old bb - .build-id files may also be present on old systems where --noartifact does not work - package dependencies need to be sorted for comparison - dependency collection hangs frequently in buildbot, enabled logging temporarily to get more output for the issue --- scripts/bash_lib.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 2b9e9751..8d40245c 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -634,12 +634,8 @@ collect_dependencies() { [[ $test_type == "minor" ]] || return old_or_new=$1 pkgtype=$2 - if rpm --noartifact ; then - # --noartifact option is needed to avoid build-id in rpm -lp output - NOARTIFACT="--noartifact" - fi bb_log_info "Collecting dependencies for the ${old_or_new} server" - set +x +# set +x for p in ${package_list} ${spider_package_list} ; do if [[ "$p" =~ columnstore ]] ; then suffix="columnstore" @@ -649,22 +645,23 @@ collect_dependencies() { echo "-----------------" >> "./reqs-${suffix}.${old_or_new}" echo "$p:" >> "./reqs-${suffix}.${old_or_new}" if [ "$pkgtype" == "rpm" ] ; then - rpm -q -R "$p" | awk '{print $1}' | grep -vE '/usr/bin/env|/usr/bin/bash' >> "./reqs-${suffix}.${old_or_new}" - # NOARTIFACT is either an empty string or --noartifact option, it won't be globbed or word-splitted - # shellcheck disable=SC2086 - filelist=$(rpm $NOARTIFACT -ql "$p" | sort) + rpm -q -R "$p" | awk '{print $1}' | sort | grep -vE '/usr/bin/env|/usr/bin/bash' >> "./reqs-${suffix}.${old_or_new}" + filelist=$(rpm -ql "$p" | sort) else # We need sudo here for the apt-cache command, not for redirection # shellcheck disable=SC2024 - sudo apt-cache depends "$p" --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances >> "./reqs-${suffix}.${old_or_new}" + sudo apt-cache depends "$p" --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances | sort >> "./reqs-${suffix}.${old_or_new}" filelist=$(dpkg-query -L "$p" | sort) fi echo "====== Package $p" >> "./ldd-${suffix}.${old_or_new}" for f in $filelist ; do + if [[ "$f" =~ "/.build-id/" ]] ; then + continue + fi sudo ldd "$f" > /dev/null 2>&1 || continue echo "=== $f" >> "./ldd-${suffix}.${old_or_new}" sudo ldd "$f" | sort | sed 's/(.*)//' >> "./ldd-${suffix}.${old_or_new}" done done - set -x +# set -x } From 5bd165ecf84e5c3aec140b0af90498eec54ca347 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 6 Oct 2024 22:34:13 +0300 Subject: [PATCH 23/34] MDBF-800 Extend package minor upgrade tests to ensure transition from old bb - .build-id files may also be present on old systems where --noartifact does not work - package dependencies need to be sorted for comparison - dependency collection hangs frequently in buildbot, enabled logging temporarily to get more output for the issue --- scripts/bash_lib.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 8d40245c..9b1a9061 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -655,6 +655,8 @@ collect_dependencies() { fi echo "====== Package $p" >> "./ldd-${suffix}.${old_or_new}" for f in $filelist ; do + # We do want to match literally here, not as regex + # shellcheck disable=SC2076 if [[ "$f" =~ "/.build-id/" ]] ; then continue fi From dc1b355200a52cc1cad85528a7bb4ab773d04ba8 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Tue, 8 Oct 2024 13:33:41 +0300 Subject: [PATCH 24/34] MDBF-806 - Exclude LDD for mariadb-test packages Improving build time by excluding the too many files of mariadb-test package from the ldd check when dependencies are collected. --- scripts/bash_lib.sh | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 9b1a9061..af46d2c2 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -642,28 +642,36 @@ collect_dependencies() { else suffix="main" fi + echo "-----------------" >> "./reqs-${suffix}.${old_or_new}" echo "$p:" >> "./reqs-${suffix}.${old_or_new}" if [ "$pkgtype" == "rpm" ] ; then rpm -q -R "$p" | awk '{print $1}' | sort | grep -vE '/usr/bin/env|/usr/bin/bash' >> "./reqs-${suffix}.${old_or_new}" - filelist=$(rpm -ql "$p" | sort) else # We need sudo here for the apt-cache command, not for redirection # shellcheck disable=SC2024 - sudo apt-cache depends "$p" --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances | sort >> "./reqs-${suffix}.${old_or_new}" - filelist=$(dpkg-query -L "$p" | sort) + sudo apt-cache depends "$p" --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances | sort >> "./reqs-${suffix}.${old_or_new}" fi - echo "====== Package $p" >> "./ldd-${suffix}.${old_or_new}" - for f in $filelist ; do - # We do want to match literally here, not as regex - # shellcheck disable=SC2076 - if [[ "$f" =~ "/.build-id/" ]] ; then - continue + + # Collect LDD output for files installed by the package on the system + if [[ ${p} != "mariadb-test"* ]]; then + if [ "$pkgtype" == "rpm" ] ; then + filelist=$(rpm -ql "$p" | sort) + else + filelist=$(dpkg-query -L "$p" | sort) fi - sudo ldd "$f" > /dev/null 2>&1 || continue - echo "=== $f" >> "./ldd-${suffix}.${old_or_new}" - sudo ldd "$f" | sort | sed 's/(.*)//' >> "./ldd-${suffix}.${old_or_new}" - done + echo "====== Package $p" >> "./ldd-${suffix}.${old_or_new}" + for f in $filelist ; do + # We do want to match literally here, not as regex + # shellcheck disable=SC2076 + if [[ "$f" =~ "/.build-id/" ]] ; then + continue + fi + sudo ldd "$f" > /dev/null 2>&1 || continue + echo "=== $f" >> "./ldd-${suffix}.${old_or_new}" + sudo ldd "$f" | sort | sed 's/(.*)//' >> "./ldd-${suffix}.${old_or_new}" + done + fi done # set -x } From dee303a8c612b73c0b7e5941050da02991caed1e Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Tue, 8 Oct 2024 13:38:31 +0300 Subject: [PATCH 25/34] Disable previously enabled command logging in MariaDB#582 --- scripts/bash_lib.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index af46d2c2..367cbaac 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -635,7 +635,7 @@ collect_dependencies() { old_or_new=$1 pkgtype=$2 bb_log_info "Collecting dependencies for the ${old_or_new} server" -# set +x + set +x for p in ${package_list} ${spider_package_list} ; do if [[ "$p" =~ columnstore ]] ; then suffix="columnstore" @@ -650,7 +650,7 @@ collect_dependencies() { else # We need sudo here for the apt-cache command, not for redirection # shellcheck disable=SC2024 - sudo apt-cache depends "$p" --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances | sort >> "./reqs-${suffix}.${old_or_new}" + sudo apt-cache depends "$p" --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances | sort >> "./reqs-${suffix}.${old_or_new}" fi # Collect LDD output for files installed by the package on the system @@ -673,5 +673,5 @@ collect_dependencies() { done fi done -# set -x + set -x } From 8a3ad076ed4f437fb49375dc16cea311f6a91a04 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Tue, 8 Oct 2024 13:56:15 +0300 Subject: [PATCH 26/34] Make package name lowercase in LDD comparison --- scripts/bash_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 367cbaac..56e6ba33 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -654,7 +654,7 @@ collect_dependencies() { fi # Collect LDD output for files installed by the package on the system - if [[ ${p} != "mariadb-test"* ]]; then + if [[ ${p,,} != "mariadb-test"* ]]; then if [ "$pkgtype" == "rpm" ] ; then filelist=$(rpm -ql "$p" | sort) else From 0d5a2269078fe76b263305b7db05eb1fdf81e440 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Wed, 9 Oct 2024 09:54:56 +0300 Subject: [PATCH 27/34] [MDBF-800][Post-Fix] - Extra restart needed due to MDEV-23061 Align code with old-bb and peform an extra restart after the previous version is installed so plugins are loaded. --- scripts/deb-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index 16a64e69..cb04f208 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -150,7 +150,7 @@ if [[ $systemdCapability == "yes" ]]; then fi fi -if [[ $test_mode == "all" ]] && [[ $branch == *"10."[5-9]* ]]; then +if [[ $test_mode == "all" ]]; then bb_log_warn "Due to MDEV-23061, an extra server restart is needed" control_mariadb_server restart fi From c213db3af554bbdb1f183a8f9bff2f4b786ce435 Mon Sep 17 00:00:00 2001 From: Vlad Bogolin Date: Tue, 1 Oct 2024 11:16:57 +0300 Subject: [PATCH 28/34] Decrease parallel for x86-debian-12-fulltest The build runs --big-test which puts a lot of memory pressure on the machine, so adjusting the parallel to mitigate this. --- master-docker-nonstandard-2/master.cfg | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index 0c61360a..7c174cd6 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -142,7 +142,7 @@ addWorker( 1, "-debian-12-32-bit", os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12-386", - jobs=20, + jobs=10, save_packages=False, shm_size="30G", ) @@ -152,7 +152,7 @@ addWorker( 2, "-debian-12-32-bit", os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12-386", - jobs=20, + jobs=10, save_packages=False, shm_size="30G", ) @@ -162,8 +162,9 @@ addWorker( 3, "-debian-12-32-bit", os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12-386", - jobs=20, + jobs=10, save_packages=False, + shm_size="30G", ) addWorker( From 6e0a15bb796caa8e6cd32d3984abd4e209ac006d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 9 Oct 2024 15:30:46 +1100 Subject: [PATCH 29/34] MDEV-31050 Add cursor protocol to full test builders --- constants.py | 2 ++ master-docker-nonstandard-2/master.cfg | 4 +++- master-docker-nonstandard/master.cfg | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/constants.py b/constants.py index fab85921..a315ae5c 100644 --- a/constants.py +++ b/constants.py @@ -213,6 +213,7 @@ # run `./mtr testname`, they need a special build test_type_to_mtr_arg = { "nm": "", + "cursor": "--cursor-protocol", "ps": "--ps-protocol", "connect": "--suite=connect", "emb": "--embedded", @@ -223,6 +224,7 @@ "ubsan": "", "valgrind": "", "debug": "", + "debug-cursor": "--cursor-protocol", "debug-ps": "--ps-protocol", "debug-emb": "--embedded", "debug-emb-ps": "--embedded --ps-protocol", diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index 7c174cd6..90c9191a 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -568,7 +568,9 @@ def add_test_steps(factory, test_types): # Custom parts of the command extra_args = "" - if test_type == "emb": + if test_type == "cursor": + extra_args = "--cursor-protocol" + elif test_type == "emb": extra_args = "--embedded-server" elif test_type == "ps": extra_args = "--ps-protocol" diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 14580539..098362f0 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -1045,6 +1045,7 @@ f_full_test.addStep( # MTR steps full_test_configs = { + "cursor": {}, "emb": {}, "nm": {}, "ps": {}, From c8775dda5b89793459924b1e04bfc26d63d7a7ae Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Wed, 9 Oct 2024 16:43:30 +0300 Subject: [PATCH 30/34] [Mdev 31050][Post-Fix] - Enable cursor test on x86-debian-12-fulltest --- master-docker-nonstandard-2/master.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index 90c9191a..de0eec96 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -679,6 +679,7 @@ f_full_test.addStep( # List of test configurations: (test_type, output_dir, suite) full_test_configs = [ + "cursor", "emb", "nm", "ps", From 272438427f25632f3de752e579125570e0e5feda Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Fri, 11 Oct 2024 11:01:57 +0300 Subject: [PATCH 31/34] [MDBF-800][Post-fix] - Filter RPM package_list in test_mode='ALL' Filter out galera | columnstore. Match old BB configuration. --- scripts/rpm-upgrade.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh index c34cb880..24988344 100755 --- a/scripts/rpm-upgrade.sh +++ b/scripts/rpm-upgrade.sh @@ -46,6 +46,8 @@ case $test_mode in # retrieve full package list from repo package_list=$(rpm_repoquery) || bb_log_err "unable to retrieve package list from repository" + package_list=$(echo "$package_list" | grep -viE 'galera|columnstore') + bb_log_warn "Due to MCOL-4120 and other issues, Columnstore upgrade will be tested separately" ;; server) package_list="MariaDB-server MariaDB-client" From 9992cf439011cedaa51414628f2287aa7a074e91 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 14 Oct 2024 09:38:47 +0300 Subject: [PATCH 32/34] CentOS based workflow is no longer present. Fix workflow dispatcher errors: https://github.com/MariaDB/buildbot/actions/runs/11322242668 --- .github/workflows/build-workflow-dispatcher.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/build-workflow-dispatcher.yml b/.github/workflows/build-workflow-dispatcher.yml index 6d1ba3a3..69a859e8 100644 --- a/.github/workflows/build-workflow-dispatcher.yml +++ b/.github/workflows/build-workflow-dispatcher.yml @@ -3,11 +3,6 @@ name: Dispatch all build container images workflows on: workflow_dispatch: inputs: - build-centos-based: - description: 'Run CentOS-based builds' - required: true - default: false - type: boolean build-centos7pip-based: description: 'Run CentOS 7 pip-based builds' required: true @@ -55,10 +50,6 @@ on: type: boolean jobs: - build-centos-based: - if: ${{ inputs.build-centos-based }} - uses: ./.github/workflows/build-centos-based.yml - secrets: inherit build-centos7pip-based: if: ${{ inputs.build-centos7pip-based }} uses: ./.github/workflows/build-centos7.pip-based.yml From 2fe6de3c453ab64482ca190a6e1b1a37c1e2fb20 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Wed, 9 Oct 2024 11:26:35 +0300 Subject: [PATCH 33/34] Dedicated builder for triggering Galera builds - easy to re-trigger all the builds. - when new platforms are introduced one can trigger a build by re-building "trigger-galera-builds" for a particular revision . No need to wait for upstream changes to build on new platforms. --- master-galera/master.cfg | 68 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/master-galera/master.cfg b/master-galera/master.cfg index ef5da46a..20ba3b52 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -28,6 +28,8 @@ c = BuildmasterConfig = {} config = {"private": {}} exec(open("../master-private.cfg").read(), config, {}) +FQDN = os.getenv("BUILDMASTER_WG_IP", default="100.64.100.1") + ####### PROJECT IDENTITY # the 'title' string will appear at the top of this buildbot installation's @@ -89,9 +91,14 @@ schedulerTrigger = schedulers.AnyBranchScheduler( repository="https://github.com/MariaDB/galera", branch_fn=upstream_branch_fn ), treeStableTimer=60, - builderNames=builders_galera, + builderNames=["trigger-galera-builds"], +) +schedulerGaleraBuilders = schedulers.Triggerable( + name="s_galera_builders", builderNames=builders_galera ) + c["schedulers"].append(schedulerTrigger) +c["schedulers"].append(schedulerGaleraBuilders) if os.getenv("ENVIRON") == "DEV": schedulerTrigger = schedulers.AnyBranchScheduler( @@ -101,7 +108,7 @@ if os.getenv("ENVIRON") == "DEV": branch_fn=upstream_branch_fn, ), treeStableTimer=60, - builderNames=builders_galera, + builderNames=["trigger-galera-builds"], ) c["schedulers"].append(schedulerTrigger) @@ -112,6 +119,38 @@ if os.getenv("ENVIRON") == "DEV": # worker name and password must be configured on the worker. c["workers"] = [] +c["workers"].append( + worker.DockerLatentWorker( + "hz-bbw1-docker-galera-trigger", + None, + docker_host=config["private"]["docker_workers"]["hz-bbw1-docker"], + image=os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12", + followStartupLogs=False, + autopull=True, + alwaysPull=True, + masterFQDN=FQDN, + hostconfig={"shm_size": "1G"}, + max_builds=1, + build_wait_timeout=0, + ) +) + +c["workers"].append( + worker.DockerLatentWorker( + "hz-bbw4-docker-galera-trigger", + None, + docker_host=config["private"]["docker_workers"]["hz-bbw4-docker"], + image=os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12", + followStartupLogs=False, + autopull=True, + alwaysPull=True, + masterFQDN=FQDN, + hostconfig={"shm_size": "1G"}, + max_builds=1, + build_wait_timeout=0, + ) +) + # Docker workers GALERA_PACKAGES = os.getenv( "GALERA_PACKAGES_DIR", default="/mnt/autofs/galera_packages" @@ -119,7 +158,6 @@ GALERA_PACKAGES = os.getenv( workers = {} - def addWorker( worker_name_prefix, worker_id, @@ -252,6 +290,16 @@ def rpmSave(): ####### FACTORY CODE +f_trigger_builds = util.BuildFactory() +f_trigger_builds.addStep( + steps.Trigger( + schedulerNames=["s_galera_builders"], + waitForFinish=False, + updateSourceStamp=False, + doStepIf=waitIfStaging, + ) +) + ## f_deb_build - create source tarball f_deb_build = util.BuildFactory() f_deb_build.addStep( @@ -387,6 +435,20 @@ f_rpm_build.addStep( ####### BUILDERS LIST c["builders"] = [] +c["builders"].append( + util.BuilderConfig( + name="trigger-galera-builds", + workernames=[ + "hz-bbw1-docker-galera-trigger", + "hz-bbw4-docker-galera-trigger", + ], + tags=["galera"], + collapseRequests=True, + nextBuild=nextBuild, + factory=f_trigger_builds, + ) +) + for os_i in os_info: if "install_only" in os_info[os_i] and os_info[os_i]["install_only"]: continue From b685b8043d90e3f17e3fc1daa0dd18518add2fad Mon Sep 17 00:00:00 2001 From: Tuukka Pasanen Date: Mon, 14 Oct 2024 14:23:47 +0300 Subject: [PATCH 34/34] MDBF-772: Test all past MariaDB version that they install Test with all current MariaDB versions that commit can be installed over them and it works after that --- master-libvirt/master.cfg | 21 +++++++++++++++++++++ scripts/deb-upgrade.sh | 8 +++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index f8b2b78b..68dd0734 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -14,6 +14,8 @@ import docker import os import sys +from constants import * + sys.setrecursionlimit(10000) sys.path.insert(0, "/srv/buildbot/master") @@ -212,6 +214,7 @@ def getDebInstallStep(): "systemdCapability", "test_mode", "test_type", + "prev_major_version", "version_name", ] ), @@ -228,6 +231,21 @@ def getMajorVersionStep(): ), ) +def getDebUpgradeVersionSteps(f_deb_upgrade_steps): + current_major_version = util.Interpolate(f"%(prop:major_version)s") + rtn_steps = util.BuildFactory() + for cur_branch in branches_main: + if current_major_version == cur_branch: + break + + f_deb_upgrade_steps.addStep( + steps.SetProperty( + property="prev_major_version", + value=cur_branch + ) + ) + + f_deb_upgrade_steps.addStep(getDebUpgradeStep()) # FACTORY @@ -243,6 +261,9 @@ f_deb_upgrade.addStep(getMajorVersionStep()) f_deb_upgrade.addStep(getScript("deb-upgrade.sh")) f_deb_upgrade.addStep(getDebUpgradeStep()) +## f_deb_upgrade_from_version +getDebUpgradeVersionSteps(f_deb_upgrade) + ## f_rpm_install f_rpm_install = util.BuildFactory() f_rpm_install.addStep(getScript("rpm-install.sh")) diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index cb04f208..7c8a6291 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -15,7 +15,13 @@ set -e manual_run_switch "$1" upgrade_type_mode -upgrade_test_type "$test_type" + +# If there is available previous major version +# then expect that it will be used as base of +# installation +if [ -z "${prev_major_version}" ]; then + upgrade_test_type "$test_type" +fi bb_print_env