diff --git a/common_factories.py b/common_factories.py index cbb252c6..c75fa95e 100644 --- a/common_factories.py +++ b/common_factories.py @@ -10,7 +10,7 @@ from buildbot.steps.mtrlogobserver import MTR # Local -from constants import MTR_ENV, test_type_to_mtr_arg +from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES, test_type_to_mtr_arg from utils import ( createVar, dockerfile, @@ -34,7 +34,6 @@ mtrEnv, mtrJobsMultiplier, printEnv, - savedPackageBranches, saveLogs, savePackageIfBranchMatch, ) @@ -143,7 +142,9 @@ def addPostTests(factory): steps.SetPropertyFromCommand( command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary", - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + ), ) ) factory.addStep( @@ -159,7 +160,9 @@ def addPostTests(factory): && sync /packages/%(prop:tarbuildnum)s """ ), - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + ), ) ) factory.addStep( @@ -176,7 +179,7 @@ def addPostTests(factory): "parentbuildername": Property("buildername"), }, doStepIf=( - lambda step: savePackageIfBranchMatch(step, savedPackageBranches) + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) and hasEco(step) ), ) @@ -753,7 +756,7 @@ def getRpmAutobakeFactory(mtrDbPool): ), doStepIf=( lambda step: hasFiles(step) - and savePackageIfBranchMatch(step, savedPackageBranches) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) ), descriptionDone=util.Interpolate( """ @@ -793,7 +796,7 @@ def getRpmAutobakeFactory(mtrDbPool): }, doStepIf=( lambda step: hasInstall(step) - and savePackageIfBranchMatch(step, savedPackageBranches) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) and hasFiles(step) ), ) @@ -812,7 +815,7 @@ def getRpmAutobakeFactory(mtrDbPool): }, doStepIf=( lambda step: hasUpgrade(step) - and savePackageIfBranchMatch(step, savedPackageBranches) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) and hasFiles(step) ), ) diff --git a/constants.py b/constants.py index 39af5e9a..71f9cba8 100644 --- a/constants.py +++ b/constants.py @@ -5,7 +5,7 @@ DEVELOPMENT_BRANCH = "11.3" # Used to trigger the appropriate main branch -branches_main = [ +BRANCHES_MAIN = [ "10.5", "10.6", "10.11", @@ -22,7 +22,7 @@ ] # Defines what builders report status to GitHub -github_status_builders = [ +GITHUB_STATUS_BUILDERS = [ "aarch64-macos-compile-only", "amd64-debian-12", "amd64-debian-12-debug-embedded", @@ -38,21 +38,21 @@ ] # Special builders triggering -builders_big = ["amd64-ubuntu-2004-bigtest"] -builders_eco = [ +BUILDERS_BIG = ["amd64-ubuntu-2004-bigtest"] +BUILDERS_ECO = [ "amd64-debian-10-eco-mysqljs", "amd64-debian-10-eco-pymysql", "amd64-ubuntu-2004-eco-php", ] if os.getenv("ENVIRON") == "DEV": - builders_wordpress = ["amd64-rhel9-wordpress"] - builders_dockerlibrary = ["amd64-rhel9-dockerlibrary"] + BUILDERS_WORDPRESS = ["amd64-rhel9-wordpress"] + BUILDERS_DOCKERLIBRARY = ["amd64-rhel9-dockerlibrary"] else: - builders_wordpress = ["amd64-rhel8-wordpress"] - builders_dockerlibrary = ["amd64-rhel8-dockerlibrary"] + BUILDERS_WORDPRESS = ["amd64-rhel8-wordpress"] + BUILDERS_DOCKERLIBRARY = ["amd64-rhel8-dockerlibrary"] -builders_galera_mtr = [ +BUILDERS_GALERA_MTR = [ "aarch64-debian-12", "amd64-fedora-39", "s390x-ubuntu-2004", @@ -61,13 +61,13 @@ "ppc64le-ubuntu-2204", "amd64-freebsd-14", ] -builders_s3_mtr = [ +BUILDERS_S3_MTR = [ "aarch64-ubuntu-2004-debug", "s390x-sles-1506", ] # Defines branches for which we save packages -savedPackageBranches = branches_main + [ +SAVED_PACKAGE_BRANCHES = BRANCHES_MAIN + [ "bb-*-release", "bb-10.2-compatibility", "preview-*", @@ -75,13 +75,13 @@ ] # The trees for which we save binary packages. -releaseBranches = ["bb-*-release", "preview-*"] +RELEASE_BRANCHES = ["bb-*-release", "preview-*"] # Note: # Maximum supported branch is the one where the default distro MariaDB package major version <= branch # For example, if Debian 10 has MariaDB 10.3 by default, we don't support MariaDB 10.2 on it. -supportedPlatforms = {} -supportedPlatforms["10.5"] = [ +SUPPORTED_PLATFORMS = {} +SUPPORTED_PLATFORMS["10.5"] = [ "aarch64-centos-stream9", "aarch64-debian-10-bintar", "aarch64-debian-11", @@ -130,17 +130,17 @@ "x86-debian-12-fulltest", ] -supportedPlatforms["10.6"] = supportedPlatforms["10.5"].copy() +SUPPORTED_PLATFORMS["10.6"] = SUPPORTED_PLATFORMS["10.5"].copy() # Add only 10.5 supported platforms -supportedPlatforms["10.5"] += [ +SUPPORTED_PLATFORMS["10.5"] += [ "amd64-kvm-centos-6-bintar", "amd64-kvm-ubuntu-1604-bintar", "x86-kvm-centos-6-bintar", "x86-kvm-ubuntu-1604-bintar", ] -supportedPlatforms["10.6"] += [ +SUPPORTED_PLATFORMS["10.6"] += [ "aarch64-ubuntu-2204", "amd64-opensuse-1505", "amd64-sles-1505", @@ -151,14 +151,14 @@ "x86-debian-12", ] -supportedPlatforms["10.9"] = supportedPlatforms["10.6"].copy() +SUPPORTED_PLATFORMS["10.9"] = SUPPORTED_PLATFORMS["10.6"].copy() -supportedPlatforms["10.10"] = [ +SUPPORTED_PLATFORMS["10.10"] = [ "amd64-debian-11-aocc", ] -supportedPlatforms["10.10"] += supportedPlatforms["10.9"] +SUPPORTED_PLATFORMS["10.10"] += SUPPORTED_PLATFORMS["10.9"] -supportedPlatforms["10.11"] = [ +SUPPORTED_PLATFORMS["10.11"] = [ "aarch64-debian-12", "aarch64-fedora-40", "aarch64-fedora-41", @@ -175,14 +175,14 @@ "s390x-ubuntu-2404", "ppc64le-debian-12", ] -supportedPlatforms["10.11"] += supportedPlatforms["10.10"] +SUPPORTED_PLATFORMS["10.11"] += SUPPORTED_PLATFORMS["10.10"] -supportedPlatforms["11.0"] = supportedPlatforms["10.11"].copy() -supportedPlatforms["11.1"] = supportedPlatforms["11.0"].copy() -supportedPlatforms["11.2"] = supportedPlatforms["11.1"].copy() -supportedPlatforms["11.3"] = supportedPlatforms["11.2"].copy() -supportedPlatforms["11.4"] = supportedPlatforms["11.3"].copy() -supportedPlatforms["11.4"] += [ +SUPPORTED_PLATFORMS["11.0"] = SUPPORTED_PLATFORMS["10.11"].copy() +SUPPORTED_PLATFORMS["11.1"] = SUPPORTED_PLATFORMS["11.0"].copy() +SUPPORTED_PLATFORMS["11.2"] = SUPPORTED_PLATFORMS["11.1"].copy() +SUPPORTED_PLATFORMS["11.3"] = SUPPORTED_PLATFORMS["11.2"].copy() +SUPPORTED_PLATFORMS["11.4"] = SUPPORTED_PLATFORMS["11.3"].copy() +SUPPORTED_PLATFORMS["11.4"] += [ "aarch64-debian-sid", "aarch64-ubuntu-2410", "amd64-debian-sid", @@ -190,11 +190,11 @@ "ppc64le-debian-sid", "x86-debian-sid", ] -supportedPlatforms["11.5"] = supportedPlatforms["11.4"].copy() -supportedPlatforms["11.6"] = supportedPlatforms["11.5"].copy() -supportedPlatforms["11.7"] = supportedPlatforms["11.6"].copy() -supportedPlatforms["11.8"] = supportedPlatforms["11.7"].copy() -supportedPlatforms["main"] = supportedPlatforms["11.8"].copy() +SUPPORTED_PLATFORMS["11.5"] = SUPPORTED_PLATFORMS["11.4"].copy() +SUPPORTED_PLATFORMS["11.6"] = SUPPORTED_PLATFORMS["11.5"].copy() +SUPPORTED_PLATFORMS["11.7"] = SUPPORTED_PLATFORMS["11.6"].copy() +SUPPORTED_PLATFORMS["11.8"] = SUPPORTED_PLATFORMS["11.7"].copy() +SUPPORTED_PLATFORMS["main"] = SUPPORTED_PLATFORMS["11.8"].copy() # Define environment variables for MTR step MTR_ENV = { @@ -238,30 +238,30 @@ # Edit with care with open("/srv/buildbot/master/os_info.yaml") as f: - os_info = yaml.safe_load(f) + OS_INFO = yaml.safe_load(f) # Generate install builders based on the os_info data -builders_install = [] -builders_upgrade = [] -builders_autobake = [] -all_platforms = set() -for os_i in os_info: - for arch in os_info[os_i]["arch"]: +BUILDERS_INSTALL = [] +BUILDERS_UPGRADE = [] +BUILDERS_AUTOBAKE = [] +ALL_PLATFORMS = set() +for os_i in OS_INFO: + for arch in OS_INFO[os_i]["arch"]: builder_name_autobake = ( - arch + "-" + os_i + "-" + os_info[os_i]["type"] + "-autobake" + arch + "-" + os_i + "-" + OS_INFO[os_i]["type"] + "-autobake" ) - if not ("install_only" in os_info[os_i] and os_info[os_i]["install_only"]): - all_platforms.add(arch) - builders_autobake.append(builder_name_autobake) + if not ("install_only" in OS_INFO[os_i] and OS_INFO[os_i]["install_only"]): + ALL_PLATFORMS.add(arch) + BUILDERS_AUTOBAKE.append(builder_name_autobake) # Currently there are no VMs for x86 and s390x and OpenSUSE and SLES if arch not in ["s390x", "x86"] and "sles" not in os_i: - builders_install.append(builder_name_autobake + "-install") - builders_upgrade.append(builder_name_autobake + "-minor-upgrade-all") - builders_upgrade.append( + BUILDERS_INSTALL.append(builder_name_autobake + "-install") + BUILDERS_UPGRADE.append(builder_name_autobake + "-minor-upgrade-all") + BUILDERS_UPGRADE.append( builder_name_autobake + "-minor-upgrade-columnstore" ) - builders_upgrade.append(builder_name_autobake + "-major-upgrade") + BUILDERS_UPGRADE.append(builder_name_autobake + "-major-upgrade") -builders_galera = list( - map(lambda x: "gal-" + "-".join(x.split("-")[:3]), builders_autobake) +BUILDERS_GALERA = list( + map(lambda x: "gal-" + "-".join(x.split("-")[:3]), BUILDERS_AUTOBAKE) ) diff --git a/define_masters.py b/define_masters.py index d5b8065e..ed90ccec 100755 --- a/define_masters.py +++ b/define_masters.py @@ -10,14 +10,14 @@ exec(open("master-private.cfg").read(), config, {}) with open("os_info.yaml", encoding="utf-8") as file: - os_info = yaml.safe_load(file) + OS_INFO = yaml.safe_load(file) platforms = {} -for os_name in os_info: - if "install_only" in os_info[os_name] and os_info[os_name]["install_only"]: +for os_name in OS_INFO: + if "install_only" in OS_INFO[os_name] and OS_INFO[os_name]["install_only"]: continue - for arch in os_info[os_name]["arch"]: + for arch in OS_INFO[os_name]["arch"]: builder_name = arch + "-" + os_name if arch not in platforms: platforms[arch] = [] diff --git a/locks.py b/locks.py index 9f916227..f744f4db 100644 --- a/locks.py +++ b/locks.py @@ -5,7 +5,7 @@ from buildbot.plugins import util # Local -from constants import builders_install, builders_upgrade, github_status_builders +from constants import BUILDERS_INSTALL, BUILDERS_UPGRADE, GITHUB_STATUS_BUILDERS LOCKS: dict[str, util.MasterLock] = {} # worker_locks.yaml currently is in the same folder as locks.py. @@ -30,9 +30,9 @@ def getLocks(props): assert builder_name is not None if ( - builder_name in github_status_builders - or builder_name in builders_install - or builder_name in builders_upgrade + builder_name in GITHUB_STATUS_BUILDERS + or builder_name in BUILDERS_INSTALL + or builder_name in BUILDERS_UPGRADE ): return [] diff --git a/master-bintars/master.cfg b/master-bintars/master.cfg index 4ef8d3ef..53401f6f 100644 --- a/master-bintars/master.cfg +++ b/master-bintars/master.cfg @@ -50,10 +50,17 @@ gs = reporters.GitHubStatusPush( startDescription="Build started.", endDescription="Build done.", verbose=True, - builders=github_status_builders, + builders=GITHUB_STATUS_BUILDERS, ) c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))] +c["secretsProviders"] = [ + secrets.SecretInAFile( + dirname=os.getenv( + "MASTER_CREDENTIALS_DIR", + default="/srv/buildbot/master/master-credential-provider", + ) + ) +] ####### PROJECT IDENTITY @@ -124,7 +131,9 @@ default_workers_bintar = """ } } """ -workers_bintar = json.loads(os.getenv('MASTER_NONLATENT_BINTARS_WORKERS', default=default_workers_bintar)) +workers_bintar = json.loads( + os.getenv("MASTER_NONLATENT_BINTARS_WORKERS", default=default_workers_bintar) +) c["workers"] = [] @@ -132,11 +141,12 @@ for k, v in workers_bintar.items(): c["workers"].append( mkWorker( f"{k}", - max_builds=v['max_builds'], - properties={"jobs": v['jobs']}, + max_builds=v["max_builds"], + properties={"jobs": v["jobs"]}, ) ) + ####### FACTORY CODE def memory_def(): # TODO Dummy implementation, replace with actual logic to define memory @@ -161,7 +171,7 @@ def getMTR(**kwargs): return steps.MTR( dbpool=mtrDbPool, warningPattern="MTR's internal check of the test case '.*' failed|Failing test\(s\):.*|\*\*\*Warnings generated in error logs.*|Test warning:.*", - **kwargs + **kwargs, ) @@ -265,7 +275,9 @@ def getBintarFactory( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + ), ) ) @@ -344,7 +356,7 @@ builder_definitions = { "centos-6": "centos6", } -current_port = int(os.getenv('MASTER_NONLATENT_BINTARS_VM_PORT', default=10000)) +current_port = int(os.getenv("MASTER_NONLATENT_BINTARS_VM_PORT", default=10000)) for b in builder_definitions: for arch in ["amd64", "i386"]: diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index aee720d0..c0ed10eb 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -49,10 +49,17 @@ gs = reporters.GitHubStatusPush( startDescription="Build started.", endDescription="Build done.", verbose=True, - builders=github_status_builders, + builders=GITHUB_STATUS_BUILDERS, ) c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))] +c["secretsProviders"] = [ + secrets.SecretInAFile( + dirname=os.getenv( + "MASTER_CREDENTIALS_DIR", + default="/srv/buildbot/master/master-credential-provider", + ) + ) +] ####### PROJECT IDENTITY # the 'title' string will appear at the top of this buildbot installation's @@ -141,7 +148,8 @@ addWorker( "amd-bbw", 1, "-debian-12-32-bit", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12-386", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian12-386", jobs=10, save_packages=False, shm_size="30G", @@ -151,7 +159,8 @@ addWorker( "amd-bbw", 2, "-debian-12-32-bit", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12-386", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian12-386", jobs=10, save_packages=False, shm_size="30G", @@ -161,7 +170,8 @@ addWorker( "apexis-bbw", 3, "-debian-12-32-bit", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12-386", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian12-386", jobs=10, save_packages=False, shm_size="30G", @@ -171,7 +181,8 @@ addWorker( "apexis-bbw", 3, "-msan-clang-debian-11", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian11-msan", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian11-msan", jobs=20, save_packages=False, ) @@ -180,7 +191,8 @@ addWorker( "apexis-bbw", 3, "-ubuntu-2204-jepsen-mariadb", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu22.04-jepsen-mariadb", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "ubuntu22.04-jepsen-mariadb", jobs=5, save_packages=False, ) @@ -756,8 +768,7 @@ f_without_server.addStep( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=lambda step: savePackageIfBranchMatch(step, - savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES), ) ) f_without_server.addStep( @@ -1044,8 +1055,7 @@ f_bintar.addStep( steps.SetPropertyFromCommand( command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary", - doStepIf=lambda step: savePackageIfBranchMatch(step, - savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES), ) ) f_bintar.addStep( @@ -1069,8 +1079,7 @@ f_bintar.addStep( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=lambda step: savePackageIfBranchMatch(step, - savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES), ) ) f_bintar.addStep( @@ -1087,9 +1096,10 @@ f_bintar.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=(lambda step: savePackageIfBranchMatch(step, - savedPackageBranches) - and hasEco(step)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + and hasEco(step) + ), ) ) f_bintar.addStep( diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 513a9a36..478bc9c3 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -49,10 +49,17 @@ gs = reporters.GitHubStatusPush( startDescription="Build started.", endDescription="Build done.", verbose=True, - builders=github_status_builders, + builders=GITHUB_STATUS_BUILDERS, ) c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))] +c["secretsProviders"] = [ + secrets.SecretInAFile( + dirname=os.getenv( + "MASTER_CREDENTIALS_DIR", + default="/srv/buildbot/master/master-credential-provider", + ) + ) +] ####### PROJECT IDENTITY # the 'title' string will appear at the top of this buildbot installation's @@ -212,7 +219,10 @@ c["workers"].append( "bm-bbw1-docker-ubuntu-2004", None, docker_host=config["private"]["docker_workers"]["bm-bbw1-docker"], - image=os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + image=os.getenv( + "CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:" + ) + + "ubuntu20.04", followStartupLogs=False, autopull=True, alwaysPull=True, @@ -233,7 +243,8 @@ addWorker( "hz-bbw", 6, "-bigtest-ubuntu-2004", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "ubuntu20.04", jobs=20, save_packages=False, ) @@ -244,7 +255,10 @@ for w_name in ["ppc64le-osuosl-bbw"]: w_name, 1, "-ubuntu-2004", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv( + "CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:" + ) + + "ubuntu20.04", jobs=7, save_packages=True, shm_size="20G", @@ -253,7 +267,10 @@ for w_name in ["ppc64le-osuosl-bbw"]: w_name, 1, "-ubuntu-2004-debug", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv( + "CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:" + ) + + "ubuntu20.04", jobs=30, save_packages=True, ) @@ -270,7 +287,11 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "-aocc-debian-11", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian11-aocc", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "debian11-aocc", jobs=jobs, save_packages=False, ) @@ -278,7 +299,11 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "-asan-ubuntu-2404", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu24.04", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "ubuntu24.04", jobs=jobs, save_packages=False, ) @@ -286,7 +311,11 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "-icc-ubuntu-2204", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu22.04-icc", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "ubuntu22.04-icc", jobs=jobs, save_packages=False, ) @@ -294,7 +323,11 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "-ubuntu-2004", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "ubuntu20.04", jobs=jobs, save_packages=True, ) @@ -304,7 +337,8 @@ addWorker( "amd-bbw", 1, "-valgrind-fedora-40", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "fedora40-valgrind", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -312,7 +346,8 @@ addWorker( "amd-bbw", 2, "-valgrind-fedora-40", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "fedora40-valgrind", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -320,7 +355,8 @@ addWorker( "hz-bbw", 6, "-valgrind-fedora-40", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "fedora40-valgrind", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -329,7 +365,8 @@ addWorker( "hz-bbw", 1, "-msan-clang-16-debian-11", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian11-msan-clang-16", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian11-msan-clang-16", jobs=20, save_packages=False, ) @@ -337,7 +374,8 @@ addWorker( "hz-bbw", 4, "-msan-clang-16-debian-11", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian11-msan-clang-16", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian11-msan-clang-16", jobs=20, save_packages=False, ) @@ -345,7 +383,8 @@ addWorker( "hz-bbw", 5, "-msan-clang-16-debian-11", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian11-msan-clang-16", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian11-msan-clang-16", jobs=30, save_packages=False, ) @@ -354,7 +393,8 @@ addWorker( "hz-bbw", 2, "-debian-12", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian12", jobs=20, save_packages=False, ) @@ -362,7 +402,8 @@ addWorker( "hz-bbw", 5, "-debian-12", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian12", jobs=20, save_packages=False, ) @@ -371,7 +412,8 @@ addWorker( "aarch64-bbw", 6, "-ubuntu-2004-debug", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "ubuntu20.04", jobs=10, save_packages=True, ) @@ -379,7 +421,8 @@ addWorker( "aarch64-bbw", 6, "-debian-10-bintar", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian10-bintar", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "debian10-bintar", jobs=10, save_packages=True, ) @@ -388,7 +431,8 @@ addWorker( "hz-bbw", 5, "-centos-7-bintar", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "centos7-bintar", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "centos7-bintar", jobs=10, save_packages=True, ) @@ -397,7 +441,8 @@ addWorker( "s390x-bbw", 1, "-ubuntu-2004-debug", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -406,7 +451,8 @@ addWorker( "s390x-bbw", 2, "-ubuntu-2004-debug", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -415,7 +461,8 @@ addWorker( "s390x-bbw", 3, "-ubuntu-2004-debug", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -426,7 +473,10 @@ c["workers"].append( "release-prep-docker", None, docker_host=config["private"]["docker_workers"]["release-prep-docker"], - image=os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12-release", + image=os.getenv( + "CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:" + ) + + "debian12-release", followStartupLogs=False, autopull=True, alwaysPull=True, @@ -1133,7 +1183,7 @@ f_without_server.addStep( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=(lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)), ) ) f_without_server.addStep( @@ -1421,7 +1471,7 @@ f_bintar.addStep( steps.SetPropertyFromCommand( command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary", - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=(lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)), ) ) f_bintar.addStep( @@ -1445,7 +1495,7 @@ f_bintar.addStep( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=(lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)), ) ) f_bintar.addStep( @@ -1460,9 +1510,8 @@ f_bintar.addStep( "mariadb_binary": Property("mariadb_binary"), "mariadb_version": Property("mariadb_version"), "master_branch": Property("master_branch"), - "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches) and hasEco(step), + doStepIf=(lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)), ) ) f_bintar.addStep( diff --git a/master-galera/master.cfg b/master-galera/master.cfg index c14ea5c1..8c676adc 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -65,8 +65,8 @@ c["buildbotNetUsageData"] = None # Configure the Schedulers, which decide how to react to incoming changes. -branches_main = ["mariadb-3.x", "mariadb-4.x", "bb-*"] -savedPackageBranches = ["mariadb-3.x", "mariadb-4.x", "bb-*"] +BRANCHES_MAIN = ["mariadb-3.x", "mariadb-4.x", "bb-*"] +SAVED_PACKAGE_BRANCHES_GALERA = ["mariadb-3.x", "mariadb-4.x", "bb-*"] # git branch filter using fnmatch import fnmatch @@ -74,7 +74,7 @@ import fnmatch def upstream_branch_fn(branch): return ( - branch in branches_main + branch in BRANCHES_MAIN or fnmatch.fnmatch(branch, "mariadb-3.x") or fnmatch.fnmatch(branch, "mariadb-4.x") or fnmatch.fnmatch(branch, "bb-*") @@ -94,7 +94,7 @@ schedulerTrigger = schedulers.AnyBranchScheduler( builderNames=["trigger-galera-builds"], ) schedulerGaleraBuilders = schedulers.Triggerable( - name="s_galera_builders", builderNames=builders_galera + name="s_galera_builders", builderNames=BUILDERS_GALERA ) c["schedulers"].append(schedulerTrigger) @@ -191,7 +191,7 @@ def addWorker( c["workers"].append(instance) -for platform in all_platforms: +for platform in ALL_PLATFORMS: jobs = None if platform == "amd64": machines = ["hz-bbw"] @@ -219,13 +219,13 @@ for platform in all_platforms: for w_name in machines: for i in worker_ids: - for os_str in os_info: + for os_str in OS_INFO: if ( - "install_only" in os_info[os_str] - and os_info[os_str]["install_only"] + "install_only" in OS_INFO[os_str] + and OS_INFO[os_str]["install_only"] ): continue - if platform in os_info[os_str]["arch"]: + if platform in OS_INFO[os_str]["arch"]: quay_name = os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "".join( os_str.split("-") ) @@ -265,7 +265,9 @@ def dpkgDeb(): """ ), ], - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=(lambda step: + savePackageIfBranchMatch(step, + SAVED_PACKAGE_BRANCHES_GALERA)), ) @@ -284,7 +286,9 @@ def rpmSave(): """ ), ], - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=(lambda step: + savePackageIfBranchMatch(step, + SAVED_PACKAGE_BRANCHES_GALERA)), ) @@ -366,7 +370,9 @@ EOF """, url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), ), - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=(lambda step: + savePackageIfBranchMatch(step, + SAVED_PACKAGE_BRANCHES_GALERA)), ) ) f_deb_build.addStep( @@ -423,7 +429,9 @@ EOF """, url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), ), - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=(lambda step: + savePackageIfBranchMatch(step, + SAVED_PACKAGE_BRANCHES_GALERA)), ) ) f_rpm_build.addStep( @@ -449,13 +457,13 @@ c["builders"].append( ) ) -for os_i in os_info: - if "install_only" in os_info[os_i] and os_info[os_i]["install_only"]: +for os_i in OS_INFO: + if "install_only" in OS_INFO[os_i] and OS_INFO[os_i]["install_only"]: continue - for arch in os_info[os_i]["arch"]: + for arch in OS_INFO[os_i]["arch"]: builder_name = "gal-" + arch + "-" + os_i - assert builder_name in builders_galera + assert builder_name in BUILDERS_GALERA worker_name = arch + "-bbw-docker-" + os_i if arch == "amd64": @@ -463,7 +471,7 @@ for os_i in os_info: if arch == "x86": worker_name = "x64-bbw-docker-" + os_i + "-i386" - if os_info[os_i]["type"] == "rpm": + if OS_INFO[os_i]["type"] == "rpm": factory = f_rpm_build else: factory = f_deb_build diff --git a/master-libvirt/get_ssh_cnx_num.py b/master-libvirt/get_ssh_cnx_num.py index c78796da..daa65386 100644 --- a/master-libvirt/get_ssh_cnx_num.py +++ b/master-libvirt/get_ssh_cnx_num.py @@ -4,11 +4,11 @@ import yaml with open("../os_info.yaml") as f: - os_info = yaml.safe_load(f) + OS_INFO = yaml.safe_load(f) SSH_CONNECTIONS = 0 -for os in os_info: - for arch in os_info[os]["arch"]: +for os in OS_INFO: + for arch in OS_INFO[os]["arch"]: if arch not in ["s390x", "x86"] and "sles" not in os: SSH_CONNECTIONS += 1 diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index e35b064a..71fcd851 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -19,7 +19,7 @@ sys.setrecursionlimit(10000) sys.path.insert(0, "/srv/buildbot/master") from utils import * -from constants import os_info +from constants import OS_INFO # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. @@ -236,7 +236,7 @@ c["workers"] = [] c["builders"] = [] # Add the workers and builds based on the configured install builders (see constants.py) -for builder_name in builders_install: +for builder_name in BUILDERS_INSTALL: # Parse builder name platform, os_name, os_version, builder_type = builder_name.split("-")[:4] @@ -274,13 +274,13 @@ for builder_name in builders_install: elif builder_type == "rpm": factory_install = f_rpm_install factory_upgrade = f_rpm_upgrade - build_arch = os_name + str(os_info[os_info_name]["version_name"]) + "-" + platform + build_arch = os_name + str(OS_INFO[os_info_name]["version_name"]) + "-" + platform # FIXME - all RPM's should follow the same conventions! - if os_name == "centos" and os_info[os_info_name]["version_name"] >= 9: + if os_name == "centos" and OS_INFO[os_info_name]["version_name"] >= 9: if platform == "amd64": platform = "x86_64" - build_arch = f"centos/{os_info[os_info_name]['version_name']}/{platform}" + build_arch = f"centos/{OS_INFO[os_info_name]['version_name']}/{platform}" c["builders"].append( @@ -295,7 +295,7 @@ for builder_name in builders_install: "systemdCapability": "yes", "needsGalera": "yes", "dist_name": os_name, - "version_name": os_info[os_info_name]["version_name"], + "version_name": OS_INFO[os_info_name]["version_name"], "arch": build_arch, "BB_CI": True, "artifactsURL": artifactsURL, @@ -319,7 +319,7 @@ for builder_name in builders_install: "systemdCapability": "yes", "needsGalera": "yes", "dist_name": os_name, - "version_name": os_info[os_info_name]["version_name"], + "version_name": OS_INFO[os_info_name]["version_name"], "arch": build_arch, "test_mode": "server", "test_type": "major", @@ -345,7 +345,7 @@ for builder_name in builders_install: "systemdCapability": "yes", "needsGalera": "yes", "dist_name": os_name, - "version_name": os_info[os_info_name]["version_name"], + "version_name": OS_INFO[os_info_name]["version_name"], "arch": build_arch, "test_mode": "all", "test_type": "minor", @@ -369,7 +369,7 @@ for builder_name in builders_install: "systemdCapability": "yes", "needsGalera": "no", "dist_name": os_name, - "version_name": os_info[os_info_name]["version_name"], + "version_name": OS_INFO[os_info_name]["version_name"], "arch": build_arch, "test_mode": "columnstore", "test_type": "minor", diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index d1989c1f..76fce0c6 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -82,10 +82,17 @@ gs = reporters.GitHubStatusPush( startDescription="Build started.", endDescription="Build done.", verbose=True, - builders=github_status_builders, + builders=GITHUB_STATUS_BUILDERS, ) c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))] +c["secretsProviders"] = [ + secrets.SecretInAFile( + dirname=os.getenv( + "MASTER_CREDENTIALS_DIR", + default="/srv/buildbot/master/master-credential-provider", + ) + ) +] ####### Builder priority c["prioritizeBuilders"] = prioritizeBuilders @@ -268,23 +275,22 @@ f_windows.addStep( ) windows_tests = { - "nm": { - "create_scripts": True - }, + "nm": {"create_scripts": True}, "connect": { "suites": ["connect"], }, } for typ in windows_tests: - addWinTests(f_windows, - mtr_test_type=typ, - mtr_env=f_windows_env, - mtr_additional_args=util.Property("mtr_additional_args", default=""), - mtr_step_db_pool=mtrDbPool, - mtr_suites=windows_tests[typ].get('suites',["default"]), - create_scripts=windows_tests[typ].get('create_scripts',False) - ) + addWinTests( + f_windows, + mtr_test_type=typ, + mtr_env=f_windows_env, + mtr_additional_args=util.Property("mtr_additional_args", default=""), + mtr_step_db_pool=mtrDbPool, + mtr_suites=windows_tests[typ].get("suites", ["default"]), + create_scripts=windows_tests[typ].get("create_scripts", False), + ) f_windows.addStep( @@ -300,13 +306,13 @@ f_windows.addStep( ), url=util.Interpolate( f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}' - '/' + "/" "%(prop:tarbuildnum)s" - '/' - 'logs' - '/' - '%(prop:buildername)s' - '/' + "/" + "logs" + "/" + "%(prop:buildername)s" + "/" ), ) ) @@ -331,8 +337,16 @@ f_windows.addStep( ## f_windows_msi f_windows_msi_env = { - "TMP": util.Interpolate("{0}\\%(prop:buildername)s\\build\\tmpdir".format('D:\\DEV\\Buildbot' if os.getenv('ENVIRON') == 'DEV' else 'D:\\Buildbot')), - "TEMP": util.Interpolate("{0}\\%(prop:buildername)s\\build\\tmpdir".format('D:\\DEV\\Buildbot' if os.getenv('ENVIRON') == 'DEV' else 'D:\\Buildbot')), + "TMP": util.Interpolate( + "{0}\\%(prop:buildername)s\\build\\tmpdir".format( + "D:\\DEV\\Buildbot" if os.getenv("ENVIRON") == "DEV" else "D:\\Buildbot" + ) + ), + "TEMP": util.Interpolate( + "{0}\\%(prop:buildername)s\\build\\tmpdir".format( + "D:\\DEV\\Buildbot" if os.getenv("ENVIRON") == "DEV" else "D:\\Buildbot" + ) + ), } f_windows_msi_env.update(MTR_ENV) @@ -457,23 +471,22 @@ f_windows_msi.addStep( ) ) windows_msi_tests = { - "nm": { - "create_scripts": True - }, + "nm": {"create_scripts": True}, "connect": { "suites": ["connect"], }, } for typ in windows_msi_tests: - addWinTests(f_windows_msi, - mtr_test_type=typ, - mtr_env=f_windows_msi_env, - mtr_additional_args=util.Property("mtr_additional_args", default=""), - mtr_step_db_pool=mtrDbPool, - mtr_suites=windows_msi_tests[typ].get('suites',["default"]), - create_scripts=windows_msi_tests[typ].get('create_scripts',False) - ) + addWinTests( + f_windows_msi, + mtr_test_type=typ, + mtr_env=f_windows_msi_env, + mtr_additional_args=util.Property("mtr_additional_args", default=""), + mtr_step_db_pool=mtrDbPool, + mtr_suites=windows_msi_tests[typ].get("suites", ["default"]), + create_scripts=windows_msi_tests[typ].get("create_scripts", False), + ) f_windows_msi.addStep( steps.DirectoryUpload( @@ -488,13 +501,13 @@ f_windows_msi.addStep( ), url=util.Interpolate( f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}' - '/' + "/" "%(prop:tarbuildnum)s" - '/' - 'logs' - '/' - '%(prop:buildername)s' - '/' + "/" + "logs" + "/" + "%(prop:buildername)s" + "/" ), ) ) @@ -528,15 +541,13 @@ f_windows_msi.addStep( mode=0o755, url=util.Interpolate( f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}' - '/' - '%(prop:tarbuildnum)s' - '/' - '%(prop:buildername)s' - '/' - + "/" + "%(prop:tarbuildnum)s" + "/" + "%(prop:buildername)s" + "/" ), - doStepIf=lambda step: savePackageIfBranchMatch(step, - savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES), ) ) f_windows_msi.addStep( @@ -552,14 +563,13 @@ f_windows_msi.addStep( mode=0o755, url=util.Interpolate( f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}' - '/' - '%(prop:tarbuildnum)s' - '/' - '%(prop:buildername)s' - '/' + "/" + "%(prop:tarbuildnum)s" + "/" + "%(prop:buildername)s" + "/" ), - doStepIf=lambda step: savePackageIfBranchMatch(step, - savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES), ) ) f_windows_msi.addStep( @@ -742,7 +752,11 @@ f_dockerlibrary.addStep( f_dockerlibrary.addStep( steps.SetPropertyFromCommand( name="Extract the last tag created", - command=['bash','-c','if [ -f last_tag ]; then cat last_tag; else echo ""; fi'], + command=[ + "bash", + "-c", + 'if [ -f last_tag ]; then cat last_tag; else echo ""; fi', + ], property="lasttag", ) ) @@ -750,14 +764,14 @@ f_dockerlibrary.addStep( steps.SetPropertyFromCommand( name="Determine sha for the last tag", command=[ - "bash", + "bash", "-xc", util.Interpolate( 'curl -s "https://quay.io/api/v1/repository/mariadb-foundation/mariadb-devel/tag/?filter_tag_name=eq:%(prop:lasttag)s&onlyActiveTags=true" | jq -r .tags[0].manifest_digest' ), ], property="lastsha", - doStepIf=lambda step: (str(step.getProperty("lasttag")) != "") + doStepIf=lambda step: (str(step.getProperty("lasttag")) != ""), ) ) f_dockerlibrary.addStep( @@ -768,7 +782,9 @@ f_dockerlibrary.addStep( "-xc", "gh auth login --with-token < ~/gh_auth", ], - doStepIf=lambda step: (str(step.getProperty("lastsha")) != "null" and step.hasProperty("lastsha")), + doStepIf=lambda step: ( + str(step.getProperty("lastsha")) != "null" and step.hasProperty("lastsha") + ), ) ) @@ -1135,8 +1151,8 @@ c["builders"].append( "cxx_compiler": "clang++", "mtr_env": { "WSREP_PROVIDER": "/usr/local/lib/libgalera_smm.so", - } }, + }, nextBuild=nextBuild, factory=f_freebsd, ) diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 2fa011e1..084f8758 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -48,7 +48,7 @@ gs = reporters.GitHubStatusPush( startDescription="Build started.", endDescription="Build done.", verbose=True, - builders=github_status_builders, + builders=GITHUB_STATUS_BUILDERS, ) c["services"].append(gs) c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))] diff --git a/master-web/master.cfg b/master-web/master.cfg index 3e5a691a..c2e6685f 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -125,7 +125,7 @@ import fnmatch def upstream_branch_fn(branch): return ( - branch in branches_main + branch in BRANCHES_MAIN or fnmatch.fnmatch(branch, "bb-*") or fnmatch.fnmatch(branch, "st-*") or fnmatch.fnmatch(branch, "prot-*") diff --git a/master.cfg b/master.cfg index 3c3de16b..4f42972f 100644 --- a/master.cfg +++ b/master.cfg @@ -51,10 +51,12 @@ gs = reporters.GitHubStatusPush( startDescription="Build started.", endDescription="Build done.", verbose=True, - builders=github_status_builders, + builders=GITHUB_STATUS_BUILDERS, ) c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname="/srv/buildbot/master/master-credential-provider")] +c["secretsProviders"] = [ + secrets.SecretInAFile(dirname="/srv/buildbot/master/master-credential-provider") +] ####### PROJECT IDENTITY @@ -155,7 +157,13 @@ for w_name in master_config["workers"]: if image_tag.startswith("ubuntu"): image_tag = image_tag[:-2] + "." + image_tag[-2:] - quay_name = os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + image_tag + quay_name = ( + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + image_tag + ) if builder.startswith("x86"): os_name += "-i386" quay_name += "-386" @@ -235,7 +243,7 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), + doStepIf=(lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)), ) ) f_deb_autobake.addStep( @@ -250,9 +258,11 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=(lambda step: hasInstall(step) - and savePackageIfBranchMatch(step, savedPackageBranches) - and hasFiles(step)), + doStepIf=( + lambda step: hasInstall(step) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + and hasFiles(step) + ), ) ) f_deb_autobake.addStep( @@ -267,9 +277,11 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=(lambda step: hasUpgrade(step) - and savePackageIfBranchMatch(step, savedPackageBranches) - and hasFiles(step)), + doStepIf=( + lambda step: hasUpgrade(step) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + and hasFiles(step) + ), ) ) f_deb_autobake.addStep( @@ -288,9 +300,9 @@ for builder in master_config["builders"]: os_name = "-".join(splits[1:]) mtr_additional_args = None - if "mtr_additional_args" in os_info[os_name]: - if arch in os_info[os_name]["mtr_additional_args"]: - mtr_additional_args = os_info[os_name]["mtr_additional_args"][arch] + if "mtr_additional_args" in OS_INFO[os_name]: + if arch in OS_INFO[os_name]["mtr_additional_args"]: + mtr_additional_args = OS_INFO[os_name]["mtr_additional_args"][arch] if arch == "amd64": arch = "x64" @@ -299,12 +311,12 @@ for builder in master_config["builders"]: if arch == "x86": worker_name = "x64-bbw-docker-" + os_name + "-i386" - build_type = os_info[os_name]["type"] + build_type = OS_INFO[os_name]["type"] # Add builder only if it's not a protected branches one - if builder not in github_status_builders: + if builder not in GITHUB_STATUS_BUILDERS: tags = [os_name] - if arch == "s390x" and builder in builders_galera_mtr: + if arch == "s390x" and builder in BUILDERS_GALERA_MTR: tags += ["experimental"] if "sid" in builder or "stream-9" in builder: tags += ["bleeding-edge"] diff --git a/schedulers_definition.py b/schedulers_definition.py index e58fa9c8..6204121f 100644 --- a/schedulers_definition.py +++ b/schedulers_definition.py @@ -1,15 +1,15 @@ from buildbot.interfaces import IProperties from buildbot.plugins import schedulers, util from constants import ( - builders_autobake, - builders_big, - builders_dockerlibrary, - builders_eco, - builders_install, - builders_upgrade, - builders_wordpress, - github_status_builders, - supportedPlatforms, + BUILDERS_AUTOBAKE, + BUILDERS_BIG, + BUILDERS_DOCKERLIBRARY, + BUILDERS_ECO, + BUILDERS_INSTALL, + BUILDERS_UPGRADE, + BUILDERS_WORDPRESS, + GITHUB_STATUS_BUILDERS, + SUPPORTED_PLATFORMS, ) @@ -18,22 +18,30 @@ ############################ @util.renderer def branchBuilders(props: IProperties) -> list[str]: - master_branch = props.getProperty("master_branch") - builders = supportedPlatforms[master_branch] - return list(filter(lambda x: x not in github_status_builders, builders)) + mBranch = props.getProperty("master_branch") + + builders = list( + filter(lambda x: x not in GITHUB_STATUS_BUILDERS, SUPPORTED_PLATFORMS[mBranch]) + ) + + return builders @util.renderer def protectedBranchBuilders(props: IProperties) -> list[str]: - master_branch = props.getProperty("master_branch") - builders = supportedPlatforms[master_branch] - return list(filter(lambda x: x in builders, github_status_builders)) + mBranch = props.getProperty("master_branch") + + builders = list( + filter(lambda x: x in SUPPORTED_PLATFORMS[mBranch], GITHUB_STATUS_BUILDERS) + ) + + return builders @util.renderer def autobakeBuilders(props: IProperties) -> list[str]: builder_name = props.getProperty("parentbuildername") - for b in builders_autobake: + for b in BUILDERS_AUTOBAKE: if builder_name in b: return [b] return [] @@ -42,7 +50,7 @@ def autobakeBuilders(props: IProperties) -> list[str]: @util.renderer def bigtestBuilders(props: IProperties) -> list[str]: builder_name = props.getProperty("parentbuildername") - for b in builders_big: + for b in BUILDERS_BIG: if builder_name in b: return [b] return [] @@ -51,7 +59,7 @@ def bigtestBuilders(props: IProperties) -> list[str]: @util.renderer def installBuilders(props: IProperties) -> list[str]: builder_name = props.getProperty("parentbuildername") - for b in builders_install: + for b in BUILDERS_INSTALL: if builder_name in b: builders = [b] if "rhel" in builder_name: @@ -65,7 +73,7 @@ def installBuilders(props: IProperties) -> list[str]: def upgradeBuilders(props: IProperties) -> list[str]: builder_name = props.getProperty("parentbuildername") builders = [] - for b in builders_upgrade: + for b in BUILDERS_UPGRADE: if builder_name in b: if "rhel" in builder_name: builders.append(b.replace("rhel", "almalinux")) @@ -78,7 +86,7 @@ def upgradeBuilders(props: IProperties) -> list[str]: def ecoBuilders(props: IProperties) -> list[str]: builder_name = props.getProperty("parentbuildername") builders = [] - for b in builders_eco: + for b in BUILDERS_ECO: if builder_name in b: builders.append(b) return builders @@ -86,12 +94,12 @@ def ecoBuilders(props: IProperties) -> list[str]: @util.renderer def dockerLibraryBuilders(props: IProperties) -> list[str]: - return builders_dockerlibrary[0] + return BUILDERS_DOCKERLIBRARY[0] @util.renderer def wordpressBuilders(props: IProperties) -> list[str]: - return builders_wordpress[0] + return BUILDERS_WORDPRESS[0] SCHEDULERS = [ diff --git a/utils.py b/utils.py index 08ff01ec..21c905d5 100644 --- a/utils.py +++ b/utils.py @@ -17,17 +17,17 @@ from buildbot.process.workerforbuilder import AbstractWorkerForBuilder from buildbot.worker import AbstractWorker from constants import ( + BUILDERS_AUTOBAKE, + BUILDERS_BIG, + BUILDERS_ECO, + BUILDERS_GALERA_MTR, + BUILDERS_INSTALL, + BUILDERS_S3_MTR, + BUILDERS_UPGRADE, DEVELOPMENT_BRANCH, MTR_ENV, - builders_autobake, - builders_big, - builders_eco, - builders_galera_mtr, - builders_install, - builders_s3_mtr, - builders_upgrade, - releaseBranches, - savedPackageBranches, + RELEASE_BRANCHES, + SAVED_PACKAGE_BRANCHES, ) private_config = {"private": {}} @@ -218,7 +218,7 @@ def createDebRepo() -> steps.ShellCommand: ], doStepIf=( lambda step: hasFiles(step) - and savePackageIfBranchMatch(step, savedPackageBranches) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) ), ) @@ -260,7 +260,7 @@ def uploadDebArtifacts() -> steps.ShellCommand: ], doStepIf=( lambda step: hasFiles(step) - and savePackageIfBranchMatch(step, savedPackageBranches) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) ), descriptionDone=util.Interpolate( """ @@ -288,13 +288,11 @@ def build_request_sort_key(request: BuildRequest): # Priority is given to releaseBranches, savePackageBranches # then it's first come, first serve. return ( - not fnmatch_any(branch, releaseBranches), - not fnmatch_any(branch, savedPackageBranches), + not fnmatch_any(branch, RELEASE_BRANCHES), + not fnmatch_any(branch, SAVED_PACKAGE_BRANCHES), request.getSubmitTime(), ) - return sorted(requests, build_request_sort_key)[0] - def canStartBuild( builder: Builder, wfb: AbstractWorkerForBuilder, request: BuildRequest @@ -460,7 +458,7 @@ def hasFiles(step: BuildStep) -> bool: def hasInstall(step: BuildStep) -> bool: builder_name = step.getProperty("buildername") - for b in builders_install: + for b in BUILDERS_INSTALL: if builder_name in b: return True return False @@ -468,7 +466,7 @@ def hasInstall(step: BuildStep) -> bool: def hasUpgrade(step: BuildStep) -> bool: builder_name = step.getProperty("buildername") - for b in builders_upgrade: + for b in BUILDERS_UPGRADE: if builder_name in b: return True return False @@ -476,7 +474,7 @@ def hasUpgrade(step: BuildStep) -> bool: def hasEco(step: BuildStep) -> bool: builder_name = step.getProperty("buildername") - for b in builders_eco: + for b in BUILDERS_ECO: if builder_name in b: return True return False @@ -506,7 +504,7 @@ def hasCompat(step: BuildStep) -> bool: def hasDockerLibrary(step: BuildStep) -> bool: # Can only build with a saved package - if not savePackageIfBranchMatch(step, savedPackageBranches): + if not savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES): return False branch = step.getProperty("master_branch") @@ -550,7 +548,7 @@ def waitIfStaging(step: BuildStep) -> bool: def hasAutobake(step: BuildStep) -> bool: builder_name = step.getProperty("buildername") - for b in builders_autobake: + for b in BUILDERS_AUTOBAKE: if builder_name in b: return True return False @@ -558,7 +556,7 @@ def hasAutobake(step: BuildStep) -> bool: def hasGalera(step: BuildStep) -> bool: builder_name = step.getProperty("buildername") - for b in builders_galera_mtr: + for b in BUILDERS_GALERA_MTR: if builder_name in b: return True return False @@ -566,7 +564,7 @@ def hasGalera(step: BuildStep) -> bool: def hasS3(props): builder_name = props.getProperty("buildername") - for b in builders_s3_mtr: + for b in BUILDERS_S3_MTR: if builder_name == b: return True return False @@ -574,7 +572,7 @@ def hasS3(props): def hasBigtest(step: BuildStep) -> bool: builder_name = step.getProperty("buildername") - for b in builders_big: + for b in BUILDERS_BIG: if builder_name in b: return True return False