Skip to content

Commit

Permalink
Constants should use upper case letters
Browse files Browse the repository at this point in the history 
all_platforms -> ALL_PLATFORMS
branches_main -> BRANCHES_MAIN
builders_autobake -> BUILDERS_AUTOBAKE
builders_big -> BUILDERS_BIG
builders_dockerlibrary -> BUILDERS_DOCKERLIBRARY
builders_eco -> BUILDERS_ECO
builders_galera -> BUILDERS_GALERA
builders_install -> BUILDERS_INSTALL
builders_s3 -> BUILDERS_S3
builders_upgrade -> BUILDERS_UPGRADE
builders_wordpress -> BUILDERS_WORDPRESS
github_status_builders -> GITHUB_STATUS_BUILDERS
release_branches -> RELEASE_BRANCHES
savedPackageBranches -> SAVED_PACKAGE_BRANCHES
savedPackageBranches (master-galera) -> SAVED_PACKAGE_BRANCHES_GALERA
supportedPlatforms -> SUPPORTED_PLATFORMS

os_info -> OS_INFO
(yes, it is read from a yaml file, but acts as a constant)
  • Loading branch information
@cvicentiu @RazvanLiviuVarzaru
cvicentiu authored and RazvanLiviuVarzaru committed Jan 8, 2025
1 parent 6fd8905 commit 3d3722a
Show file tree
Hide file tree
Showing 16 changed files with 387 additions and 271 deletions.
19 changes: 11 additions & 8 deletions common_factories.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from buildbot.steps.mtrlogobserver import MTR

# Local
from constants import MTR_ENV, test_type_to_mtr_arg
from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES, test_type_to_mtr_arg
from utils import (
createVar,
dockerfile,
Expand All @@ -34,7 +34,6 @@
mtrEnv,
mtrJobsMultiplier,
printEnv,
savedPackageBranches,
saveLogs,
savePackageIfBranchMatch,
)
Expand Down Expand Up @@ -143,7 +142,9 @@ def addPostTests(factory):
steps.SetPropertyFromCommand(
command="basename mariadb-*-linux-*.tar.gz",
property="mariadb_binary",
doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
doStepIf=(
lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
),
)
)
factory.addStep(
Expand All @@ -159,7 +160,9 @@ def addPostTests(factory):
&& sync /packages/%(prop:tarbuildnum)s
"""
),
doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
doStepIf=(
lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
),
)
)
factory.addStep(
Expand All @@ -176,7 +179,7 @@ def addPostTests(factory):
"parentbuildername": Property("buildername"),
},
doStepIf=(
lambda step: savePackageIfBranchMatch(step, savedPackageBranches)
lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
and hasEco(step)
),
)
Expand Down Expand Up @@ -753,7 +756,7 @@ def getRpmAutobakeFactory(mtrDbPool):
),
doStepIf=(
lambda step: hasFiles(step)
and savePackageIfBranchMatch(step, savedPackageBranches)
and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
),
descriptionDone=util.Interpolate(
"""
Expand Down Expand Up @@ -793,7 +796,7 @@ def getRpmAutobakeFactory(mtrDbPool):
},
doStepIf=(
lambda step: hasInstall(step)
and savePackageIfBranchMatch(step, savedPackageBranches)
and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
and hasFiles(step)
),
)
Expand All @@ -812,7 +815,7 @@ def getRpmAutobakeFactory(mtrDbPool):
},
doStepIf=(
lambda step: hasUpgrade(step)
and savePackageIfBranchMatch(step, savedPackageBranches)
and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
and hasFiles(step)
),
)
Expand Down
100 changes: 50 additions & 50 deletions constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
DEVELOPMENT_BRANCH = "11.3"

# Used to trigger the appropriate main branch
branches_main = [
BRANCHES_MAIN = [
"10.5",
"10.6",
"10.11",
Expand All @@ -22,7 +22,7 @@
]

# Defines what builders report status to GitHub
github_status_builders = [
GITHUB_STATUS_BUILDERS = [
"aarch64-macos-compile-only",
"amd64-debian-12",
"amd64-debian-12-debug-embedded",
Expand All @@ -38,21 +38,21 @@
]

# Special builders triggering
builders_big = ["amd64-ubuntu-2004-bigtest"]
builders_eco = [
BUILDERS_BIG = ["amd64-ubuntu-2004-bigtest"]
BUILDERS_ECO = [
"amd64-debian-10-eco-mysqljs",
"amd64-debian-10-eco-pymysql",
"amd64-ubuntu-2004-eco-php",
]

if os.getenv("ENVIRON") == "DEV":
builders_wordpress = ["amd64-rhel9-wordpress"]
builders_dockerlibrary = ["amd64-rhel9-dockerlibrary"]
BUILDERS_WORDPRESS = ["amd64-rhel9-wordpress"]
BUILDERS_DOCKERLIBRARY = ["amd64-rhel9-dockerlibrary"]
else:
builders_wordpress = ["amd64-rhel8-wordpress"]
builders_dockerlibrary = ["amd64-rhel8-dockerlibrary"]
BUILDERS_WORDPRESS = ["amd64-rhel8-wordpress"]
BUILDERS_DOCKERLIBRARY = ["amd64-rhel8-dockerlibrary"]

builders_galera_mtr = [
BUILDERS_GALERA_MTR = [
"aarch64-debian-12",
"amd64-fedora-39",
"s390x-ubuntu-2004",
Expand All @@ -61,27 +61,27 @@
"ppc64le-ubuntu-2204",
"amd64-freebsd-14",
]
builders_s3_mtr = [
BUILDERS_S3_MTR = [
"aarch64-ubuntu-2004-debug",
"s390x-sles-1506",
]

# Defines branches for which we save packages
savedPackageBranches = branches_main + [
SAVED_PACKAGE_BRANCHES = BRANCHES_MAIN + [
"bb-*-release",
"bb-10.2-compatibility",
"preview-*",
"*pkgtest*",
]

# The trees for which we save binary packages.
releaseBranches = ["bb-*-release", "preview-*"]
RELEASE_BRANCHES = ["bb-*-release", "preview-*"]

# Note:
# Maximum supported branch is the one where the default distro MariaDB package major version <= branch
# For example, if Debian 10 has MariaDB 10.3 by default, we don't support MariaDB 10.2 on it.
supportedPlatforms = {}
supportedPlatforms["10.5"] = [
SUPPORTED_PLATFORMS = {}
SUPPORTED_PLATFORMS["10.5"] = [
"aarch64-centos-stream9",
"aarch64-debian-10-bintar",
"aarch64-debian-11",
Expand Down Expand Up @@ -130,17 +130,17 @@
"x86-debian-12-fulltest",
]

supportedPlatforms["10.6"] = supportedPlatforms["10.5"].copy()
SUPPORTED_PLATFORMS["10.6"] = SUPPORTED_PLATFORMS["10.5"].copy()

# Add only 10.5 supported platforms
supportedPlatforms["10.5"] += [
SUPPORTED_PLATFORMS["10.5"] += [
"amd64-kvm-centos-6-bintar",
"amd64-kvm-ubuntu-1604-bintar",
"x86-kvm-centos-6-bintar",
"x86-kvm-ubuntu-1604-bintar",
]

supportedPlatforms["10.6"] += [
SUPPORTED_PLATFORMS["10.6"] += [
"aarch64-ubuntu-2204",
"amd64-opensuse-1505",
"amd64-sles-1505",
Expand All @@ -151,14 +151,14 @@
"x86-debian-12",
]

supportedPlatforms["10.9"] = supportedPlatforms["10.6"].copy()
SUPPORTED_PLATFORMS["10.9"] = SUPPORTED_PLATFORMS["10.6"].copy()

supportedPlatforms["10.10"] = [
SUPPORTED_PLATFORMS["10.10"] = [
"amd64-debian-11-aocc",
]
supportedPlatforms["10.10"] += supportedPlatforms["10.9"]
SUPPORTED_PLATFORMS["10.10"] += SUPPORTED_PLATFORMS["10.9"]

supportedPlatforms["10.11"] = [
SUPPORTED_PLATFORMS["10.11"] = [
"aarch64-debian-12",
"aarch64-fedora-40",
"aarch64-fedora-41",
Expand All @@ -175,26 +175,26 @@
"s390x-ubuntu-2404",
"ppc64le-debian-12",
]
supportedPlatforms["10.11"] += supportedPlatforms["10.10"]
SUPPORTED_PLATFORMS["10.11"] += SUPPORTED_PLATFORMS["10.10"]

supportedPlatforms["11.0"] = supportedPlatforms["10.11"].copy()
supportedPlatforms["11.1"] = supportedPlatforms["11.0"].copy()
supportedPlatforms["11.2"] = supportedPlatforms["11.1"].copy()
supportedPlatforms["11.3"] = supportedPlatforms["11.2"].copy()
supportedPlatforms["11.4"] = supportedPlatforms["11.3"].copy()
supportedPlatforms["11.4"] += [
SUPPORTED_PLATFORMS["11.0"] = SUPPORTED_PLATFORMS["10.11"].copy()
SUPPORTED_PLATFORMS["11.1"] = SUPPORTED_PLATFORMS["11.0"].copy()
SUPPORTED_PLATFORMS["11.2"] = SUPPORTED_PLATFORMS["11.1"].copy()
SUPPORTED_PLATFORMS["11.3"] = SUPPORTED_PLATFORMS["11.2"].copy()
SUPPORTED_PLATFORMS["11.4"] = SUPPORTED_PLATFORMS["11.3"].copy()
SUPPORTED_PLATFORMS["11.4"] += [
"aarch64-debian-sid",
"aarch64-ubuntu-2410",
"amd64-debian-sid",
"amd64-ubuntu-2410",
"ppc64le-debian-sid",
"x86-debian-sid",
]
supportedPlatforms["11.5"] = supportedPlatforms["11.4"].copy()
supportedPlatforms["11.6"] = supportedPlatforms["11.5"].copy()
supportedPlatforms["11.7"] = supportedPlatforms["11.6"].copy()
supportedPlatforms["11.8"] = supportedPlatforms["11.7"].copy()
supportedPlatforms["main"] = supportedPlatforms["11.8"].copy()
SUPPORTED_PLATFORMS["11.5"] = SUPPORTED_PLATFORMS["11.4"].copy()
SUPPORTED_PLATFORMS["11.6"] = SUPPORTED_PLATFORMS["11.5"].copy()
SUPPORTED_PLATFORMS["11.7"] = SUPPORTED_PLATFORMS["11.6"].copy()
SUPPORTED_PLATFORMS["11.8"] = SUPPORTED_PLATFORMS["11.7"].copy()
SUPPORTED_PLATFORMS["main"] = SUPPORTED_PLATFORMS["11.8"].copy()

# Define environment variables for MTR step
MTR_ENV = {
Expand Down Expand Up @@ -238,30 +238,30 @@
# Edit with care

with open("/srv/buildbot/master/os_info.yaml") as f:
os_info = yaml.safe_load(f)
OS_INFO = yaml.safe_load(f)

# Generate install builders based on the os_info data
builders_install = []
builders_upgrade = []
builders_autobake = []
all_platforms = set()
for os_i in os_info:
for arch in os_info[os_i]["arch"]:
BUILDERS_INSTALL = []
BUILDERS_UPGRADE = []
BUILDERS_AUTOBAKE = []
ALL_PLATFORMS = set()
for os_i in OS_INFO:
for arch in OS_INFO[os_i]["arch"]:
builder_name_autobake = (
arch + "-" + os_i + "-" + os_info[os_i]["type"] + "-autobake"
arch + "-" + os_i + "-" + OS_INFO[os_i]["type"] + "-autobake"
)
if not ("install_only" in os_info[os_i] and os_info[os_i]["install_only"]):
all_platforms.add(arch)
builders_autobake.append(builder_name_autobake)
if not ("install_only" in OS_INFO[os_i] and OS_INFO[os_i]["install_only"]):
ALL_PLATFORMS.add(arch)
BUILDERS_AUTOBAKE.append(builder_name_autobake)
# Currently there are no VMs for x86 and s390x and OpenSUSE and SLES
if arch not in ["s390x", "x86"] and "sles" not in os_i:
builders_install.append(builder_name_autobake + "-install")
builders_upgrade.append(builder_name_autobake + "-minor-upgrade-all")
builders_upgrade.append(
BUILDERS_INSTALL.append(builder_name_autobake + "-install")
BUILDERS_UPGRADE.append(builder_name_autobake + "-minor-upgrade-all")
BUILDERS_UPGRADE.append(
builder_name_autobake + "-minor-upgrade-columnstore"
)
builders_upgrade.append(builder_name_autobake + "-major-upgrade")
BUILDERS_UPGRADE.append(builder_name_autobake + "-major-upgrade")

builders_galera = list(
map(lambda x: "gal-" + "-".join(x.split("-")[:3]), builders_autobake)
BUILDERS_GALERA = list(
map(lambda x: "gal-" + "-".join(x.split("-")[:3]), BUILDERS_AUTOBAKE)
)
8 changes: 4 additions & 4 deletions define_masters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@
exec(open("master-private.cfg").read(), config, {})

with open("os_info.yaml", encoding="utf-8") as file:
os_info = yaml.safe_load(file)
OS_INFO = yaml.safe_load(file)

platforms = {}

for os_name in os_info:
if "install_only" in os_info[os_name] and os_info[os_name]["install_only"]:
for os_name in OS_INFO:
if "install_only" in OS_INFO[os_name] and OS_INFO[os_name]["install_only"]:
continue
for arch in os_info[os_name]["arch"]:
for arch in OS_INFO[os_name]["arch"]:
builder_name = arch + "-" + os_name
if arch not in platforms:
platforms[arch] = []
Expand Down
8 changes: 4 additions & 4 deletions locks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from buildbot.plugins import util

# Local
from constants import builders_install, builders_upgrade, github_status_builders
from constants import BUILDERS_INSTALL, BUILDERS_UPGRADE, GITHUB_STATUS_BUILDERS

LOCKS: dict[str, util.MasterLock] = {}
# worker_locks.yaml currently is in the same folder as locks.py.
Expand All @@ -30,9 +30,9 @@ def getLocks(props):
assert builder_name is not None

if (
builder_name in github_status_builders
or builder_name in builders_install
or builder_name in builders_upgrade
builder_name in GITHUB_STATUS_BUILDERS
or builder_name in BUILDERS_INSTALL
or builder_name in BUILDERS_UPGRADE
):
return []

Expand Down
28 changes: 20 additions & 8 deletions master-bintars/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,17 @@ gs = reporters.GitHubStatusPush(
startDescription="Build started.",
endDescription="Build done.",
verbose=True,
builders=github_status_builders,
builders=GITHUB_STATUS_BUILDERS,
)
c["services"].append(gs)
c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))]
c["secretsProviders"] = [
secrets.SecretInAFile(
dirname=os.getenv(
"MASTER_CREDENTIALS_DIR",
default="/srv/buildbot/master/master-credential-provider",
)
)
]

####### PROJECT IDENTITY

Expand Down Expand Up @@ -124,19 +131,22 @@ default_workers_bintar = """
}
}
"""
workers_bintar = json.loads(os.getenv('MASTER_NONLATENT_BINTARS_WORKERS', default=default_workers_bintar))
workers_bintar = json.loads(
os.getenv("MASTER_NONLATENT_BINTARS_WORKERS", default=default_workers_bintar)
)

c["workers"] = []

for k, v in workers_bintar.items():
c["workers"].append(
mkWorker(
f"{k}",
max_builds=v['max_builds'],
properties={"jobs": v['jobs']},
max_builds=v["max_builds"],
properties={"jobs": v["jobs"]},
)
)


####### FACTORY CODE
def memory_def():
# TODO Dummy implementation, replace with actual logic to define memory
Expand All @@ -161,7 +171,7 @@ def getMTR(**kwargs):
return steps.MTR(
dbpool=mtrDbPool,
warningPattern="MTR's internal check of the test case '.*' failed|Failing test\(s\):.*|\*\*\*Warnings generated in error logs.*|Test warning:.*",
**kwargs
**kwargs,
)


Expand Down Expand Up @@ -265,7 +275,9 @@ def getBintarFactory(
+ " && sync /packages/"
+ "%(prop:tarbuildnum)s"
),
doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
doStepIf=(
lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
),
)
)

Expand Down Expand Up @@ -344,7 +356,7 @@ builder_definitions = {
"centos-6": "centos6",
}

current_port = int(os.getenv('MASTER_NONLATENT_BINTARS_VM_PORT', default=10000))
current_port = int(os.getenv("MASTER_NONLATENT_BINTARS_VM_PORT", default=10000))

for b in builder_definitions:
for arch in ["amd64", "i386"]:
Expand Down
Loading

0 comments on commit 3d3722a

Please sign in to comment.