From cdd16531513e50b6480aeef80338066f01b93d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 3 Dec 2024 20:07:53 +0200 Subject: [PATCH] Refactor createWorker from utils.py Simplify and ensure it returns a 3-value tuple. --- master-docker-nonstandard-2/master.cfg | 22 +++++------ master-docker-nonstandard/master.cfg | 53 ++++++++++++-------------- master-galera/master.cfg | 13 +++---- master-protected-branches/master.cfg | 49 +++++++++++------------- master.cfg | 40 +++++-------------- utils.py | 37 +++++++++--------- 6 files changed, 89 insertions(+), 125 deletions(-) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index faeb6635..423cc4cc 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -13,6 +13,7 @@ from twisted.internet import defer import docker import os import sys +from collections import defaultdict sys.path.insert(0, "/srv/buildbot/master") sys.setrecursionlimit(10000) @@ -108,7 +109,7 @@ c["schedulers"] = SCHEDULERS # worker name and password must be configured on the worker. c["workers"] = [] -workers = {} +workers = defaultdict(list) def addWorker( @@ -120,7 +121,7 @@ def addWorker( save_packages=False, shm_size="15G", ): - name, instance = createWorker( + base_name, name, instance = createWorker( worker_name_prefix, worker_id, worker_type, @@ -130,21 +131,18 @@ def addWorker( shm_size, ) - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - + workers[base_name].append(name) c["workers"].append(instance) + # Docker workers fqdn = os.environ["BUILDMASTER_WG_IP"] addWorker( "amd-bbw", 1, - "-debian-12-32-bit", + "debian-12-32-bit", os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, @@ -155,7 +153,7 @@ addWorker( addWorker( "amd-bbw", 2, - "-debian-12-32-bit", + "debian-12-32-bit", os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, @@ -166,7 +164,7 @@ addWorker( addWorker( "apexis-bbw", 3, - "-debian-12-32-bit", + "debian-12-32-bit", os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, @@ -177,7 +175,7 @@ addWorker( addWorker( "apexis-bbw", 3, - "-msan-clang-debian-11", + "msan-clang-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan", jobs=20, @@ -187,7 +185,7 @@ addWorker( addWorker( "apexis-bbw", 3, - "-ubuntu-2204-jepsen-mariadb", + "ubuntu-2204-jepsen-mariadb", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04-jepsen-mariadb", jobs=5, diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 5b58edfe..260c6a55 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -13,6 +13,7 @@ from twisted.internet import defer import docker import os import sys +from collections import defaultdict sys.path.insert(0, "/srv/buildbot/master") sys.setrecursionlimit(10000) @@ -108,7 +109,7 @@ c["schedulers"] = SCHEDULERS # worker name and password must be configured on the worker. c["workers"] = [] -workers = {} +workers = defaultdict(list) def addWorker( @@ -120,7 +121,7 @@ def addWorker( save_packages=False, shm_size="15G", ): - name, instance = createWorker( + base_name, name, instance = createWorker( worker_name_prefix, worker_id, worker_type, @@ -130,11 +131,7 @@ def addWorker( shm_size, ) - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - + workers[base_name].append(name) c["workers"].append(instance) @@ -235,7 +232,7 @@ c["workers"].append( addWorker( "hz-bbw", 6, - "-bigtest-ubuntu-2004", + "bigtest-ubuntu-2004", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=20, @@ -247,7 +244,7 @@ for w_name in ["ppc64le-osuosl-bbw"]: addWorker( w_name, 1, - "-ubuntu-2004", + "ubuntu-2004", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, @@ -257,7 +254,7 @@ for w_name in ["ppc64le-osuosl-bbw"]: addWorker( w_name, 1, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=30, @@ -275,7 +272,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( w_name, i, - "-aocc-debian-11", + "aocc-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-aocc", jobs=jobs, @@ -284,7 +281,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( w_name, i, - "-asan-ubuntu-2404", + "asan-ubuntu-2404", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu24.04", jobs=jobs, @@ -293,7 +290,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( w_name, i, - "-icc-ubuntu-2204", + "icc-ubuntu-2204", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04-icc", jobs=jobs, @@ -302,7 +299,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( w_name, i, - "-ubuntu-2004", + "ubuntu-2004", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=jobs, @@ -313,7 +310,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( "amd-bbw", 1, - "-valgrind-fedora-40", + "valgrind-fedora-40", os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, @@ -322,7 +319,7 @@ addWorker( addWorker( "amd-bbw", 2, - "-valgrind-fedora-40", + "valgrind-fedora-40", os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, @@ -331,7 +328,7 @@ addWorker( addWorker( "hz-bbw", 6, - "-valgrind-fedora-40", + "valgrind-fedora-40", os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, @@ -341,7 +338,7 @@ addWorker( addWorker( "hz-bbw", 1, - "-msan-clang-16-debian-11", + "msan-clang-16-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=20, @@ -350,7 +347,7 @@ addWorker( addWorker( "hz-bbw", 4, - "-msan-clang-16-debian-11", + "msan-clang-16-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=20, @@ -359,7 +356,7 @@ addWorker( addWorker( "hz-bbw", 5, - "-msan-clang-16-debian-11", + "msan-clang-16-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=30, @@ -369,7 +366,7 @@ addWorker( addWorker( "hz-bbw", 2, - "-debian-12", + "debian-12", os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=20, @@ -378,7 +375,7 @@ addWorker( addWorker( "hz-bbw", 5, - "-debian-12", + "debian-12", os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=20, @@ -388,7 +385,7 @@ addWorker( addWorker( "aarch64-bbw", 6, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=10, @@ -397,7 +394,7 @@ addWorker( addWorker( "aarch64-bbw", 6, - "-debian-10-bintar", + "debian-10-bintar", os.environ["CONTAINER_REGISTRY_URL"] + "debian10-bintar", jobs=10, @@ -407,7 +404,7 @@ addWorker( addWorker( "hz-bbw", 5, - "-centos-7-bintar", + "centos-7-bintar", os.environ["CONTAINER_REGISTRY_URL"] + "centos7-bintar", jobs=10, @@ -417,7 +414,7 @@ addWorker( addWorker( "s390x-bbw", 1, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, @@ -427,7 +424,7 @@ addWorker( addWorker( "s390x-bbw", 2, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, @@ -437,7 +434,7 @@ addWorker( addWorker( "s390x-bbw", 3, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, diff --git a/master-galera/master.cfg b/master-galera/master.cfg index d1e32fa6..56fd4067 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -12,6 +12,7 @@ import sys import os import docker from datetime import timedelta +from collections import defaultdict sys.setrecursionlimit(10000) @@ -154,7 +155,7 @@ c["workers"].append( # Docker workers GALERA_PACKAGES = os.environ["GALERA_PACKAGES_DIR"] -workers = {} +workers = defaultdict(list) def addWorker( worker_name_prefix, @@ -165,7 +166,7 @@ def addWorker( save_packages=False, shm_size="15G", ): - name, instance = createWorker( + base_name, name, instance = createWorker( worker_name_prefix, worker_id, worker_type, @@ -181,11 +182,7 @@ def addWorker( ], ) - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - + workers[base_name].append(name) c["workers"].append(instance) @@ -236,7 +233,7 @@ for platform in ALL_PLATFORMS: addWorker( w_name, i, - "-" + os_name, + os_name, quay_name, jobs=jobs, save_packages=True, diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 5b5495e9..09180a97 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -1,28 +1,29 @@ # -*- python -*- # ex: set filetype=python: +import os +import sys +from collections import defaultdict +from datetime import timedelta + +import docker from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand +from buildbot.process.properties import Properties, Property +from buildbot.process.remotecommand import RemoteCommand from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver +from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta from twisted.internet import defer -import docker -import os -import sys - sys.setrecursionlimit(10000) sys.path.insert(0, "/srv/buildbot/master") +from common_factories import * from constants import * -from utils import * from locks import * from schedulers_definition import SCHEDULERS -from common_factories import * +from utils import * # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. @@ -144,7 +145,7 @@ c["workers"].append( ) ) -workers = {} +workers = defaultdict(list) def addWorker( @@ -156,7 +157,7 @@ def addWorker( save_packages=False, shm_size="15G", ): - name, instance = createWorker( + base_name, name, worker_instance = createWorker( worker_name_prefix, worker_id, worker_type, @@ -166,12 +167,8 @@ def addWorker( shm_size, ) - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - - c["workers"].append(instance) + workers[base_name].append(name) + c["workers"].append(worker_instance) for w_name in ["hz-bbw"]: @@ -180,7 +177,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-debian-11-debug-ps-embed", + "debian-11-debug-ps-embed", os.environ["CONTAINER_REGISTRY_URL"] + "debian11", jobs=14, save_packages=False, @@ -188,7 +185,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-debian-12", + "debian-12", os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=jobs, save_packages=True, @@ -196,7 +193,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-debian-12-debug-embed", + "debian-12-debug-embed", os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=14, save_packages=False, @@ -204,7 +201,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-fedora-40", + "fedora-40", os.environ["CONTAINER_REGISTRY_URL"] + "fedora40", jobs=jobs, save_packages=True, @@ -212,7 +209,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-last-N-failed", + "last-N-failed", os.environ["CONTAINER_REGISTRY_URL"] + "rhel9", jobs=jobs, save_packages=True, @@ -220,7 +217,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-ubuntu-2004-clang", + "ubuntu-2004-clang", "vladbogo/bb:amd64-ubuntu-2004-clang", jobs=jobs, save_packages=True, @@ -228,7 +225,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=14, save_packages=True, @@ -236,7 +233,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-ubuntu-2204-debug-ps", + "ubuntu-2204-debug-ps", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04", jobs=14, save_packages=False, diff --git a/master.cfg b/master.cfg index 34442f98..49955788 100644 --- a/master.cfg +++ b/master.cfg @@ -2,6 +2,7 @@ # ex: set filetype=python: import os +from collections import defaultdict import yaml @@ -128,35 +129,7 @@ c["workers"] = [] # Docker workers -workers = {} - - -def addWorker( - worker_name_prefix, - worker_id, - worker_type, - dockerfile, - jobs=5, - save_packages=False, - shm_size="15G", -): - name, instance = createWorker( - worker_name_prefix, - worker_id, - worker_type, - dockerfile, - jobs, - save_packages, - shm_size, - ) - - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - - c["workers"].append(instance) - +workers = defaultdict(list) for w_name in master_config["workers"]: jobs = 7 @@ -182,15 +155,20 @@ for w_name in master_config["workers"]: if builder.startswith("x86"): os_name += "-i386" quay_name += "-386" - addWorker( + + base_name, name, worker_instance = createWorker( worker_name, worker_id, - "-" + os_name, + os_name, quay_name, jobs=jobs, save_packages=True, + shm_size="15G", ) + workers[base_name].append(name) + c["workers"].append(worker_instance) + ####### FACTORY CODE f_quick_build = getQuickBuildFactory("nm", mtrDbPool) diff --git a/utils.py b/utils.py index 0a9ff479..d3cd0d08 100644 --- a/utils.py +++ b/utils.py @@ -79,25 +79,22 @@ def createWorker( "/srv/buildbot/packages:/mnt/packages", MASTER_PACKAGES + "/:/packages", ], -) -> Tuple[str, worker.DockerLatentWorker]: +) -> Tuple[str, str, worker.DockerLatentWorker]: worker_name = f"{worker_name_prefix}{worker_id}-docker" - name = f"{worker_name}{worker_type}" - - if worker_name_prefix.startswith("hz"): - b_name = "x64-bbw" - elif worker_name_prefix.startswith("intel"): - b_name = "x64-bbw" - elif worker_name_prefix.startswith("ppc64le"): - b_name = "ppc64le-bbw" - elif worker_name_prefix.startswith("amd"): - b_name = "x64-bbw" - elif worker_name_prefix.startswith("apexis"): - b_name = "x64-bbw" - elif worker_name_prefix.startswith("ns"): - b_name = "x64-bbw" - else: - b_name = worker_name_prefix - base_name = b_name + "-docker" + worker_type + name = f"{worker_name}{worker_type}{worker_name_suffix}" + + # TODO(cvicentiu) Remove this list when refactoring YAML. + b_name = worker_name_prefix + X64_BUILDER_PREFIXES = ["hz", "intel", "amd", "apexis", "ns"] + PPC64LE_BUILDER_PREFIXES = ["ppc64le"] + for x64_prefix in X64_BUILDER_PREFIXES: + if worker_name_prefix.startswith(x64_prefix): + b_name = "x64-bbw" + for ppc_prefix in PPC64LE_BUILDER_PREFIXES: + if worker_name_prefix.startswith(ppc_prefix): + b_name = "ppc64le-bbw" + + base_name = f"{b_name}-docker-{worker_type}" # Set master FQDN - default to wireguard interface fqdn = os.environ["BUILDMASTER_WG_IP"] @@ -113,7 +110,7 @@ def createWorker( need_pull = False worker_instance = worker.DockerLatentWorker( - name + worker_name_suffix, + name, None, docker_host=private_config["private"]["docker_workers"][worker_name], image=image_str, @@ -135,7 +132,7 @@ def createWorker( volumes=volumes, properties={"jobs": jobs, "save_packages": save_packages}, ) - return ((base_name, name + worker_name_suffix), worker_instance) + return (base_name, name, worker_instance) def printEnv() -> steps.ShellCommand: