MDBF-832 & MDBF-828 - Prepare BBM deployment for Production

.github/workflows/bbm_build_container.yml

@@ -81,7 +81,7 @@ jobs:
           for image in master master-web; do
             skopeo copy --all --src-tls-verify=0 \
             docker://localhost:5000/${{ env.REPO }}:${image} \
-            docker://ghcr.io/mariadb/buildbot:${image}
+            docker://ghcr.io/mariadb/buildbot:dev_${image}
           done
       - name: Login to quay.io
         if: ${{ env.DEPLOY_IMAGES == 'true' }}
@@ -99,5 +99,5 @@ jobs:
           for image in master master-web; do
             skopeo copy --all --src-tls-verify=0 \
             docker://localhost:5000/${{ env.REPO }}:${image} \
-            docker://quay.io/mariadb-foundation/${{ env.REPO }}:${image}
+            docker://quay.io/mariadb-foundation/${{ env.REPO }}:dev_${image}
           done

.github/workflows/bbm_deploy.yml

@@ -41,14 +41,23 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - name: Check master.cfg files
-        run: ./validate_master_cfg.sh
+
+      - name: Check master.cfg files for DEV container
+        run: ./validate_master_cfg.sh -e DEV
+
+      # When master container is under development we must ensure
+      # that configuration changes are valid on both buildbot master
+      # versions, PROD/DEV. Let's say one will bring a new feature
+      # to Production independently of a master upgrade.
+      - name: Check master.cfg files for PROD container
+        run: ./validate_master_cfg.sh -e PROD
+
       - name: Check get_ssh_cnx_num.py
         run: |
           cd master-libvirt
           python get_ssh_cnx_num.py
 
-  deploy:
+  development-deploy:
     runs-on: ubuntu-22.04
     needs: check
     if: >
@@ -60,22 +69,47 @@ jobs:
         run: |
           install -m 600 -D /dev/null ~/.ssh/id_ed25519
           install -m 600 -D /dev/null ~/.ssh/known_hosts
-          echo "${{ secrets.SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
-          echo "${{ secrets.SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
+          echo "${{ secrets.BBM_DEV_SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
+          echo "${{ secrets.BBM_DEV_SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
       - name: shutdown stack
         run: |
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "if [[ -f /srv/dev/docker-compose/docker-compose.yaml ]]; then docker-compose -f /srv/dev/docker-compose/docker-compose.yaml down; fi"
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "if [[ -f /srv/dev/docker-compose/docker-compose.yaml ]]; then docker-compose -f /srv/dev/docker-compose/docker-compose.yaml down; fi"
       - name: deploy
         run: |
           # temporary fix of jade templating
           sed -i 's#https://ci.mariadb.org#https://ci.dev.mariadb.org#g' master-web/templates/home.jade
-          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.SERVER_PORT }}" ./ ${{ secrets.USER }}@${{ secrets.SERVER_IP }}:/srv/dev/
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py --env=dev"
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/ && ./define_masters.py"
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose/ && ln -sf .. buildbot"
+          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }}" ./ ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }}:/srv/dev/
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose/ && ./generate-config.py --env=dev"
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/ && ./define_masters.py"
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose/ && ln -sf .. buildbot"
       - name: start stack
         run: |
-          ssh -p ${{ secrets.SERVER_PORT }} ${{ secrets.USER }}@${{ secrets.SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose --env-file .env.dev up -d"
+          ssh -p ${{ secrets.BBM_DEV_SERVER_PORT }} ${{ secrets.BBM_DEV_USER }}@${{ secrets.BBM_DEV_SERVER_IP }} "cd /srv/dev/docker-compose && docker-compose pull && docker-compose --env-file .env.dev up -d"
+      - name: clean
+        run: |
+          rm ~/.ssh/id_ed25519
+
+  production-deploy:
+    runs-on: ubuntu-22.04
+    needs: check
+    if: >
+      github.event_name == 'workflow_dispatch' &&
+      github.ref == 'refs/heads/main' &&
+      github.repository == 'MariaDB/buildbot'
+    steps:
+      - uses: actions/checkout@v4
+      - name: prepare
+        run: |
+          install -m 600 -D /dev/null ~/.ssh/id_ed25519
+          install -m 600 -D /dev/null ~/.ssh/known_hosts
+          echo "${{ secrets.BBM_PROD_SSH_PRIVATE_KEY }}" >~/.ssh/id_ed25519
+          echo "${{ secrets.BBM_PROD_SSH_KNOWN_HOSTS }}" >~/.ssh/known_hosts
+      - name: deploy
+        run: |
+          rsync -a --progress --delete --exclude-from=rsync.exclude -e "ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }}" ./ ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }}:/srv/prod/
+          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/docker-compose/ && ./generate-config.py --env=prod"
+          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/ && ./define_masters.py"
+          ssh -p ${{ secrets.BBM_PROD_SERVER_PORT }} ${{ secrets.BBM_PROD_USER }}@${{ secrets.BBM_PROD_SERVER_IP }} "cd /srv/prod/docker-compose/ && ln -sf .. buildbot"
       - name: clean
         run: |
           rm ~/.ssh/id_ed25519

docker-compose/docker-compose.yaml

@@ -60,7 +60,7 @@ services:
         tag: "bb-nginx"
 
   master-web:
-    image: quay.io/mariadb-foundation/bb-master:master-web
+    image: quay.io/mariadb-foundation/bb-master:dev_master-web
     restart: unless-stopped
     container_name: master-web
     environment:
@@ -96,7 +96,7 @@ services:
         condition: service_started
 
   master-nonlatent:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-nonlatent
     environment:
@@ -136,7 +136,7 @@ services:
         condition: service_started
 
   master-libvirt:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-libvirt
     environment:
@@ -175,7 +175,7 @@ services:
         condition: service_started
 
   autogen_aarch64-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_aarch64-master-0
     environment:
@@ -214,7 +214,7 @@ services:
         condition: service_started
 
   autogen_amd64-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_amd64-master-0
     environment:
@@ -253,7 +253,7 @@ services:
         condition: service_started
 
   autogen_amd64-master-1:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_amd64-master-1
     environment:
@@ -292,7 +292,7 @@ services:
         condition: service_started
 
   autogen_ppc64le-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_ppc64le-master-0
     environment:
@@ -331,7 +331,7 @@ services:
         condition: service_started
 
   autogen_s390x-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_s390x-master-0
     environment:
@@ -370,7 +370,7 @@ services:
         condition: service_started
 
   autogen_x86-master-0:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: autogen_x86-master-0
     environment:
@@ -409,7 +409,7 @@ services:
         condition: service_started
 
   master-docker-nonstandard:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-docker-nonstandard
     environment:
@@ -448,7 +448,7 @@ services:
         condition: service_started
 
   master-galera:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-galera
     environment:
@@ -487,7 +487,7 @@ services:
         condition: service_started
 
   master-protected-branches:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-protected-branches
     environment:
@@ -526,7 +526,7 @@ services:
         condition: service_started
 
   master-docker-nonstandard-2:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-docker-nonstandard-2
     environment:
@@ -565,7 +565,7 @@ services:
         condition: service_started
 
   master-bintars:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:dev_master
     restart: unless-stopped
     container_name: master-bintars
     environment:

docker-compose/generate-config.py

@@ -86,7 +86,7 @@
         tag: "bb-nginx"
 
   master-web:
-    image: quay.io/mariadb-foundation/bb-master:master-web
+    image: quay.io/mariadb-foundation/bb-master:{environment}master-web
     restart: unless-stopped
     container_name: master-web
     hostname: master-web
@@ -105,7 +105,7 @@
 
 DOCKER_COMPOSE_TEMPLATE = """
   {master_name}:
-    image: quay.io/mariadb-foundation/bb-master:master
+    image: quay.io/mariadb-foundation/bb-master:{environment}master
     restart: unless-stopped
     container_name: {master_name}
     hostname: {master_name}
@@ -192,7 +192,9 @@ def main(args):
         )
         file.write(
             start_template.format(
-                port=master_web_port, cr_host_wg_addr=env_vars["CR_HOST_WG_ADDR"]
+                port=master_web_port,
+                cr_host_wg_addr=env_vars["CR_HOST_WG_ADDR"],
+                environment="" if args.env == "prod" else "dev_",
             )
         )
         port = starting_port
@@ -206,6 +208,7 @@ def main(args):
                 port=port,
                 mc_host=mc_host,
                 volumes=generate_volumes(master_volumes[master_name]),
+                environment="" if args.env == "prod" else "dev_",
             )
             port += 1
 

validate_master_cfg.sh

@@ -10,6 +10,43 @@ err() {
   exit 1
 }
 
+usage() {
+  echo "Usage: $0 -e <DEV|PROD>"
+  exit 1
+}
+
+ENVIRONMENT=""
+
+while getopts ":e:" opt; do
+  case ${opt} in
+    e )
+      ENVIRONMENT=$OPTARG
+      ;;
+    \? )
+      usage
+      ;;
+    : )
+      usage
+      ;;
+  esac
+done
+
+if [[ -z "$ENVIRONMENT" ]]; then
+  usage
+fi
+
+case $ENVIRONMENT in
+  DEV)
+    IMAGE="quay.io/mariadb-foundation/bb-master:dev_master"
+    ;;
+  PROD)
+    IMAGE="quay.io/mariadb-foundation/bb-master:master"
+    ;;
+  *)
+    err "Unknown environment: $ENVIRONMENT. Use DEV or PROD."
+    ;;
+esac
+
 mkdir -p master-credential-provider
 [[ -f master-private.cfg ]] ||
   ln -s master-private.cfg-sample master-private.cfg
@@ -33,7 +70,7 @@ python3 define_masters.py
 echo "Checking master.cfg"
 $RUNC run -i -v "$(pwd):/srv/buildbot/master" \
   -w /srv/buildbot/master \
-  quay.io/mariadb-foundation/bb-master:master \
+  $IMAGE \
   buildbot checkconfig master.cfg
 echo -e "done\n"
 # not checking libvirt config file (//TEMP we need to find a solution
@@ -49,7 +86,7 @@ for dir in autogen/* \
   echo "Checking $dir/master.cfg"
   $RUNC run -i -v "$(pwd):/srv/buildbot/master" \
     -w /srv/buildbot/master \
-    quay.io/mariadb-foundation/bb-master:master \
+    $IMAGE \
     bash -c "cd $dir && buildbot checkconfig master.cfg"
   echo -e "done\n"
-done
+done