Skip to content

Commit

Permalink
Refactor createWorker from utils.py
Browse files Browse the repository at this point in the history 
Simplify and ensure it returns a 3-value tuple.
  • Loading branch information
@cvicentiu
cvicentiu committed Jan 12, 2025
1 parent e645a81 commit 7e4e702
Show file tree
Hide file tree
Showing 6 changed files with 89 additions and 125 deletions.
22 changes: 10 additions & 12 deletions master-docker-nonstandard-2/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ from twisted.internet import defer
import docker
import os
import sys
from collections import defaultdict

sys.path.insert(0, "/srv/buildbot/master")
sys.setrecursionlimit(10000)
Expand Down Expand Up @@ -121,7 +122,7 @@ c["schedulers"] = SCHEDULERS
# worker name and password must be configured on the worker.
c["workers"] = []

workers = {}
workers = defaultdict(list)


def addWorker(
Expand All @@ -133,7 +134,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
Expand All @@ -143,21 +144,18 @@ def addWorker(
shm_size,
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)



# Docker workers
fqdn = os.environ["BUILDMASTER_WG_IP"]

addWorker(
"amd-bbw",
1,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
Expand All @@ -168,7 +166,7 @@ addWorker(
addWorker(
"amd-bbw",
2,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
Expand All @@ -179,7 +177,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-debian-12-32-bit",
"debian-12-32-bit",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12-386",
jobs=10,
Expand All @@ -190,7 +188,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-msan-clang-debian-11",
"msan-clang-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan",
jobs=20,
Expand All @@ -200,7 +198,7 @@ addWorker(
addWorker(
"apexis-bbw",
3,
"-ubuntu-2204-jepsen-mariadb",
"ubuntu-2204-jepsen-mariadb",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu22.04-jepsen-mariadb",
jobs=5,
Expand Down
53 changes: 25 additions & 28 deletions master-docker-nonstandard/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ from twisted.internet import defer
import docker
import os
import sys
from collections import defaultdict

sys.path.insert(0, "/srv/buildbot/master")
sys.setrecursionlimit(10000)
Expand Down Expand Up @@ -121,7 +122,7 @@ c["schedulers"] = SCHEDULERS
# worker name and password must be configured on the worker.
c["workers"] = []

workers = {}
workers = defaultdict(list)


def addWorker(
Expand All @@ -133,7 +134,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
Expand All @@ -143,11 +144,7 @@ def addWorker(
shm_size,
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)


Expand Down Expand Up @@ -248,7 +245,7 @@ c["workers"].append(
addWorker(
"hz-bbw",
6,
"-bigtest-ubuntu-2004",
"bigtest-ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=20,
Expand All @@ -260,7 +257,7 @@ for w_name in ["ppc64le-osuosl-bbw"]:
addWorker(
w_name,
1,
"-ubuntu-2004",
"ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
Expand All @@ -270,7 +267,7 @@ for w_name in ["ppc64le-osuosl-bbw"]:
addWorker(
w_name,
1,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=30,
Expand All @@ -288,7 +285,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-aocc-debian-11",
"aocc-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-aocc",
jobs=jobs,
Expand All @@ -297,7 +294,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-asan-ubuntu-2404",
"asan-ubuntu-2404",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu24.04",
jobs=jobs,
Expand All @@ -306,7 +303,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-icc-ubuntu-2204",
"icc-ubuntu-2204",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu22.04-icc",
jobs=jobs,
Expand All @@ -315,7 +312,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
w_name,
i,
"-ubuntu-2004",
"ubuntu-2004",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=jobs,
Expand All @@ -326,7 +323,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]:
addWorker(
"amd-bbw",
1,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
Expand All @@ -335,7 +332,7 @@ addWorker(
addWorker(
"amd-bbw",
2,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
Expand All @@ -344,7 +341,7 @@ addWorker(
addWorker(
"hz-bbw",
6,
"-valgrind-fedora-40",
"valgrind-fedora-40",
os.environ["CONTAINER_REGISTRY_URL"]
+ "fedora40-valgrind",
jobs=20,
Expand All @@ -354,7 +351,7 @@ addWorker(
addWorker(
"hz-bbw",
1,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=20,
Expand All @@ -363,7 +360,7 @@ addWorker(
addWorker(
"hz-bbw",
4,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=20,
Expand All @@ -372,7 +369,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-msan-clang-16-debian-11",
"msan-clang-16-debian-11",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian11-msan-clang-16",
jobs=30,
Expand All @@ -382,7 +379,7 @@ addWorker(
addWorker(
"hz-bbw",
2,
"-debian-12",
"debian-12",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12",
jobs=20,
Expand All @@ -391,7 +388,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-debian-12",
"debian-12",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian12",
jobs=20,
Expand All @@ -401,7 +398,7 @@ addWorker(
addWorker(
"aarch64-bbw",
6,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=10,
Expand All @@ -410,7 +407,7 @@ addWorker(
addWorker(
"aarch64-bbw",
6,
"-debian-10-bintar",
"debian-10-bintar",
os.environ["CONTAINER_REGISTRY_URL"]
+ "debian10-bintar",
jobs=10,
Expand All @@ -420,7 +417,7 @@ addWorker(
addWorker(
"hz-bbw",
5,
"-centos-7-bintar",
"centos-7-bintar",
os.environ["CONTAINER_REGISTRY_URL"]
+ "centos7-bintar",
jobs=10,
Expand All @@ -430,7 +427,7 @@ addWorker(
addWorker(
"s390x-bbw",
1,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
Expand All @@ -440,7 +437,7 @@ addWorker(
addWorker(
"s390x-bbw",
2,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
Expand All @@ -450,7 +447,7 @@ addWorker(
addWorker(
"s390x-bbw",
3,
"-ubuntu-2004-debug",
"ubuntu-2004-debug",
os.environ["CONTAINER_REGISTRY_URL"]
+ "ubuntu20.04",
jobs=7,
Expand Down
13 changes: 5 additions & 8 deletions master-galera/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import docker
import yaml

from datetime import timedelta
from collections import defaultdict

sys.setrecursionlimit(10000)

Expand Down Expand Up @@ -169,7 +170,7 @@ c["workers"].append(
# Docker workers
GALERA_PACKAGES = os.environ["GALERA_PACKAGES_DIR"]

workers = {}
workers = defaultdict(list)

def addWorker(
worker_name_prefix,
Expand All @@ -180,7 +181,7 @@ def addWorker(
save_packages=False,
shm_size="15G",
):
name, instance = createWorker(
base_name, name, instance = createWorker(
worker_name_prefix,
worker_id,
worker_type,
Expand All @@ -196,11 +197,7 @@ def addWorker(
],
)

if name[0] not in workers:
workers[name[0]] = [name[1]]
else:
workers[name[0]].append(name[1])

workers[base_name].append(name)
c["workers"].append(instance)


Expand Down Expand Up @@ -251,7 +248,7 @@ for platform in ALL_PLATFORMS:
addWorker(
w_name,
i,
"-" + os_name,
os_name,
quay_name,
jobs=jobs,
save_packages=True,
Expand Down
Loading

0 comments on commit 7e4e702

Please sign in to comment.