Skip to content

Commit

Permalink
Refactor createWorker from utils.py
Browse files Browse the repository at this point in the history 
Simplify and ensure it returns a 3-value tuple.
  • Loading branch information
@cvicentiu
cvicentiu committed Jan 12, 2025
1 parent 9d55785 commit cdd1653
Show file tree
Hide file tree
Showing 6 changed files with 89 additions and 125 deletions.
22 changes: 10 additions & 12 deletions master-docker-nonstandard-2/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ from twisted.internet import defer
import docker
import os
import sys
from collections import defaultdict

sys.path.insert(0, "/srv/buildbot/master")
sys.setrecursionlimit(10000)
Expand Down Expand Up @@ -108,7 +109,7 @@ c["schedulers"] = SCHEDULERS
# worker name and password must be configured on the worker.
c["workers"] = []

workers = {}
workers = defaultdict(list)


def addWorker(
Expand All @@ -120,7 +121,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
Expand All @@ -130,21 +131,18 @@ def addWorker(
shm_size,
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)



# Docker workers
fqdn = os.environ["BUILDMASTER_WG_IP"]

addWorker(
"amd-bbw",
1,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
Expand All @@ -155,7 +153,7 @@ addWorker(
addWorker(
"amd-bbw",
2,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
Expand All @@ -166,7 +164,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
Expand All @@ -177,7 +175,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-msan-clang-debian-11",
"msan-clang-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan",
jobs=20,
Expand All @@ -187,7 +185,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-ubuntu-2204-jepsen-mariadb",
"ubuntu-2204-jepsen-mariadb",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu22.04-jepsen-mariadb",
jobs=5,
Expand Down
53 changes: 25 additions & 28 deletions master-docker-nonstandard/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ from twisted.internet import defer
import docker
import os
import sys
from collections import defaultdict

sys.path.insert(0, "/srv/buildbot/master")
sys.setrecursionlimit(10000)
Expand Down Expand Up @@ -108,7 +109,7 @@ c["schedulers"] = SCHEDULERS
# worker name and password must be configured on the worker.
c["workers"] = []

workers = {}
workers = defaultdict(list)


def addWorker(
Expand All @@ -120,7 +121,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
Expand All @@ -130,11 +131,7 @@ def addWorker(
shm_size,
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)


Expand Down Expand Up @@ -235,7 +232,7 @@ c["workers"].append(
addWorker(
"hz-bbw",
6,
"-bigtest-ubuntu-2004",
"bigtest-ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=20,
Expand All @@ -247,7 +244,7 @@ for w_name in ["ppc64le-osuosl-bbw"]:
addWorker(
w_name,
1,
"-ubuntu-2004",
"ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
Expand All @@ -257,7 +254,7 @@ for w_name in ["ppc64le-osuosl-bbw"]:
addWorker(
w_name,
1,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=30,
Expand All @@ -275,7 +272,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-aocc-debian-11",
"aocc-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-aocc",
jobs=jobs,
Expand All @@ -284,7 +281,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-asan-ubuntu-2404",
"asan-ubuntu-2404",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu24.04",
jobs=jobs,
Expand All @@ -293,7 +290,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-icc-ubuntu-2204",
"icc-ubuntu-2204",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu22.04-icc",
jobs=jobs,
Expand All @@ -302,7 +299,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-ubuntu-2004",
"ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=jobs,
Expand All @@ -313,7 +310,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
"amd-bbw",
1,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
Expand All @@ -322,7 +319,7 @@ addWorker(
addWorker(
"amd-bbw",
2,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
Expand All @@ -331,7 +328,7 @@ addWorker(
addWorker(
"hz-bbw",
6,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
Expand All @@ -341,7 +338,7 @@ addWorker(
addWorker(
"hz-bbw",
1,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=20,
Expand All @@ -350,7 +347,7 @@ addWorker(
addWorker(
"hz-bbw",
4,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=20,
Expand All @@ -359,7 +356,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=30,
Expand All @@ -369,7 +366,7 @@ addWorker(
addWorker(
"hz-bbw",
2,
"-debian-12",
"debian-12",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12",
jobs=20,
Expand All @@ -378,7 +375,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-debian-12",
"debian-12",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12",
jobs=20,
Expand All @@ -388,7 +385,7 @@ addWorker(
addWorker(
"aarch64-bbw",
6,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=10,
Expand All @@ -397,7 +394,7 @@ addWorker(
addWorker(
"aarch64-bbw",
6,
"-debian-10-bintar",
"debian-10-bintar",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian10-bintar",
jobs=10,
Expand All @@ -407,7 +404,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-centos-7-bintar",
"centos-7-bintar",
os.environ["CONTAINER_REGISTRY_URL"]
+ "centos7-bintar",
jobs=10,
Expand All @@ -417,7 +414,7 @@ addWorker(
addWorker(
"s390x-bbw",
1,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
Expand All @@ -427,7 +424,7 @@ addWorker(
addWorker(
"s390x-bbw",
2,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
Expand All @@ -437,7 +434,7 @@ addWorker(
addWorker(
"s390x-bbw",
3,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
Expand Down
13 changes: 5 additions & 8 deletions master-galera/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import sys
import os
import docker
from datetime import timedelta
from collections import defaultdict

sys.setrecursionlimit(10000)

Expand Down Expand Up @@ -154,7 +155,7 @@ c["workers"].append(
# Docker workers
GALERA_PACKAGES = os.environ["GALERA_PACKAGES_DIR"]

workers = {}
workers = defaultdict(list)

def addWorker(
worker_name_prefix,
Expand All @@ -165,7 +166,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
Expand All @@ -181,11 +182,7 @@ def addWorker(
],
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)


Expand Down Expand Up @@ -236,7 +233,7 @@ for platform in ALL_PLATFORMS:
addWorker(
w_name,
i,
"-" + os_name,
os_name,
quay_name,
jobs=jobs,
save_packages=True,
Expand Down
Loading

0 comments on commit cdd1653

Please sign in to comment.