Re-Implement CI for the repo, plus some housekeeping

This uses deployments to keep runners only using specific credentials for staging and production. Proper handling of staging so people can test deploy shit will come later. This also uses DHCP on wiggle, because we're not using a public IP for it any longer. Rotates the key colmena_ci uses too, plus adds a staging version of the module for uh, staging. Finally, removes frankfurt 2 and 3, plus fixes the pkgs-unstable reference on the arm nodes to use arm.
tgstation-operations · Feb 15, 2025 · 951ad55 · 951ad55
1 parent 3b850bf
commit 951ad55
Show file tree

Hide file tree

Showing 7 changed files with 109 additions and 92 deletions.
diff --git a/.github/workflows/colmena.yml b/.github/workflows/colmena.yml
@@ -8,28 +8,26 @@ on:
 jobs:
   build:
     name: Build
-    runs-on: ubuntu-24.04
     strategy:
       matrix:
+        runner: [ubuntu-24.04]
         node:
-          - tgsatan
-          - blockmoths
-          - vpn
-          - \@relay
-          - \@staging
+          - "tgsatan"
+          - "blockmoths"
+          - "vpn"
+          - "@relay-amd64"
+          - "@staging"
+        include:
+          - node: "@relay-arm"
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
     steps:
       # We use commit hashes for specifying versions here, so a malicious tag can't gain access to our secrets (At least while sha-1 collisions are rare, anyway)
-      - name: Install private ssh key
-        uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # Install our ssh key. TODO: Replace with our own bash script
-        with:
-          key: ${{ secrets.COLMENA_SSH_KEY }}
-          name: id_ed25519
-          known_hosts: ${{ secrets.COLMENA_KNOWN_HOSTS }}
-
       - name: Login to headscale
         uses: tailscale/github-action@8688eb839e58e6b25c1ae96cd99d1c173299b842 # Connect to headscale
+        if: github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main'
         with:
-          authkey: ${{ secrets.TS_AUTHKEY }}
+          authkey: ${{ secrets.TS_BUILD_AUTHKEY }}
           args: --login-server=https://vpn.tgstation13.org
 
       - name: Checkout Repository
@@ -40,14 +38,14 @@ jobs:
         with:
           extra_nix_config: |
             accept-flake-config = true
-            extra-substituters = https://attic.tgstation13.org/tgstation-infrastructure
-            extra-trusted-public-keys = tgstation-infrastructure:tNpjd5GxK1xymRHsJdBLTpeDScA2mVPdKA/eIOLOE0I=
 
       - name: Setup attic Binary Cache
+        if: github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main'
         # Format for pointing to caches is server:cache in these commands
         run: |
           nix profile install nixpkgs#attic-client
           attic login tgstation https://attic.tgstation13.org ${{ secrets.ATTIC_JWT_TOKEN }}
+          attic use tgstation:tgstation-infrastructure
 
       - name: Build closure
         run: nix run github:zhaofengli/colmena -- build --impure -v --eval-node-limit 2 --keep-result --on ${{ matrix.node }}
@@ -59,23 +57,33 @@ jobs:
   deploy:
     name: Deploy
     needs: build
-    runs-on: ubuntu-24.04
     if: ${{ github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main' }}
+    environment: ${{ matrix.environment }}
     strategy:
       matrix:
+        runner: [ubuntu-24.04]
         node:
-          - tgsatan
-          - blockmoths
-          - vpn
-          - \@relay
-          - \@staging
+          - "tgsatan"
+          - "blockmoths"
+          - "vpn"
+          - "@relay-amd64"
+        environment:
+          - production
+        include:
+          - runner: ubuntu-24.04-arm
+            node: "@relay-arm"
+            environment: production
+          - runner: ubuntu-24.04
+            node: "@staging"
+            environment: staging
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Install private ssh key
         uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # Install our ssh key. TODO: Replace with our own bash script
         with:
           key: ${{ secrets.COLMENA_SSH_KEY }}
           name: id_ed25519
-          known_hosts: ${{ secrets.COLMENA_KNOWN_HOSTS }}
+          known_hosts: ${{ vars.COLMENA_KNOWN_HOSTS }}
 
       - name: Login to headscale
         uses: tailscale/github-action@8688eb839e58e6b25c1ae96cd99d1c173299b842 # Connect to headscale

diff --git a/flake.nix b/flake.nix
@@ -121,7 +121,7 @@
         targetHost = "chicago.tg.lan";
         targetUser = "deploy";
         tags = [
-          "relay"
+          "relay-amd64"
         ];
       };
       imports =
@@ -138,7 +138,7 @@
         targetHost = "atlanta.tg.lan";
         targetUser = "deploy";
         tags = [
-          "relay"
+          "relay-amd64"
         ];
       };
       imports =
@@ -150,40 +150,6 @@
         ];
     };
 
-    frankfurt2 = {
-      deployment = {
-        targetHost = "frankfurt2.tg.lan";
-        targetUser = "deploy";
-        tags = [
-          "relay"
-        ];
-      };
-      imports =
-        flakeModules
-        ++ [
-          (import ./modules/base.nix)
-          (import ./users)
-          (import ./nixos_systems/relay-node/eu/frankfurt2.nix)
-        ];
-    };
-
-    frankfurt3 = {
-      deployment = {
-        targetHost = "frankfurt3.tg.lan";
-        targetUser = "deploy";
-        tags = [
-          "relay"
-        ];
-      };
-      imports =
-        flakeModules
-        ++ [
-          (import ./modules/base.nix)
-          (import ./users)
-          (import ./nixos_systems/relay-node/eu/frankfurt3.nix)
-        ];
-    };
-
     blockmoths = {
       deployment = {
         targetHost = "blockmoths.tg.lan";
@@ -235,7 +201,7 @@
         targetHost = "lime.tg.lan";
         targetUser = "deploy";
         tags = [
-          "relay"
+          "relay-amd64"
         ];
       };
       imports =
@@ -268,7 +234,7 @@
         targetHost = "dachshund.tg.lan";
         targetUser = "deploy";
         tags = [
-          "relay"
+          "relay-arm"
         ];
       };
       nixpkgs.system = "aarch64-linux";
@@ -285,7 +251,7 @@
         targetHost = "knipp.tg.lan";
         targetUser = "deploy";
         tags = [
-          "relay"
+          "relay-arm"
         ];
       };
       nixpkgs.system = "aarch64-linux";
@@ -305,8 +271,6 @@
         vpn
         chicago
         atlanta
-        frankfurt2
-        frankfurt3
         blockmoths
         wiggle
         warsaw
@@ -342,19 +306,19 @@
           };
           bratwurst = {
             pkgs-unstable = import nixpkgs-unstable {
-              system = "x86_64-linux";
+              system = "aarch64-linux";
               config.allowUnfree = true;
             };
           };
           dachshund = {
             pkgs-unstable = import nixpkgs-unstable {
-              system = "x86_64-linux";
+              system = "aarch64-linux";
               config.allowUnfree = true;
             };
           };
           knipp = {
             pkgs-unstable = import nixpkgs-unstable {
-              system = "x86_64-linux";
+              system = "aarch64-linux";
               config.allowUnfree = true;
             };
           };
@@ -442,28 +406,6 @@
           };
         };
       };
-      frankfurt2 = nixpkgs.lib.nixosSystem {
-        system = "x86_64-linux";
-        modules = flakeModules ++ frankfurt2.imports;
-        specialArgs = {
-          inherit self inputs nixpkgs fenix;
-          pkgs-unstable = import nixpkgs-unstable {
-            system = "x86_64-linux";
-            config.allowUnfree = true;
-          };
-        };
-      };
-      frankfurt3 = nixpkgs.lib.nixosSystem {
-        system = "x86_64-linux";
-        modules = flakeModules ++ frankfurt3.imports;
-        specialArgs = {
-          inherit self inputs nixpkgs fenix;
-          pkgs-unstable = import nixpkgs-unstable {
-            system = "x86_64-linux";
-            config.allowUnfree = true;
-          };
-        };
-      };
       warsaw = nixpkgs.lib.nixosSystem {
         system = "x86_64-linux";
         modules = flakeModules ++ warsaw.imports;

diff --git a/modules/colmena_ci.nix b/modules/colmena_ci.nix
@@ -17,7 +17,7 @@ in {
     openssh.authorizedKeys.keys =
       deployUsers
       ++ [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINerE77pg5ziJ2adbSZ7ftCa3kX49C1C2FSJd6h6XVP+ deploy@tgstation-nix"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPQx1+Obgbo+YUubcQNFr4ry5Iob3U0fW3myAcG4PS79 deploy@tgstation-infra"
       ];
   };
 

diff --git a/modules/colmena_ci_staging.nix b/modules/colmena_ci_staging.nix
@@ -0,0 +1,31 @@
+{
+  pkgs,
+  config,
+  ...
+}: let
+  deployUsers = import ./ssh_keys.nix;
+in {
+  # Configuration required to use github actions to deploy to nodes
+  users.users.deploy = {
+    isNormalUser = true;
+
+    extraGroups = [
+      "wheel" # Needed for nixos-rebuild. Originally the idea was to just limit it to a group and setup sudo to allow nixos-rebuild as that user, but that would result in them being able to modify system.activationScripts regardless and run scripts as root, so it's not very useful
+    ];
+
+    group = "deploy";
+    openssh.authorizedKeys.keys =
+      deployUsers
+      ++ [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL9Pizmnyye3CmgonAAzdVBeiVUyXs+pk8LFPAaUFnVi deploy@tgstation-infra-staging"
+      ];
+  };
+
+  users.groups.deploy = {};
+  nix.settings = {
+    # Allow our user to use binary caches during builds explicitly
+    trusted-users = [
+      "deploy"
+    ];
+  };
+}
diff --git a/modules/headscale/headscale-acl.json b/modules/headscale/headscale-acl.json
@@ -48,6 +48,12 @@
     "tag:ci": [
       "group:op"
     ],
+    "tag:ci-staging": [
+      "group:op"
+    ],
+    "tag:ci-build": [
+      "group:op"
+    ],
     "tag:dbcluster": [
       "group:op"
     ],
@@ -73,6 +79,27 @@
         "tag:backend:443"
       ]
     },
+    {
+      "action": "accept",
+      "src": [
+        "tag:ci-build"
+      ],
+      "dst": [
+        "tag:backend:80",
+        "tag:backend:443"
+      ]
+    },
+    {
+      "action": "accept",
+      "src": [
+        "tag:ci-staging"
+      ],
+      "dst": [
+        "tag:staging:22",
+        "tag:backend:80",
+        "tag:backend:443"
+      ]
+    },
     {
       "action": "accept",
       "src": [

diff --git a/nixos_systems/relay-node/staging/base.nix b/nixos_systems/relay-node/staging/base.nix
@@ -8,7 +8,7 @@
     ../../../modules/fail2ban.nix
     ../../../modules/openssh.nix
     ../../../modules/tailscale.nix
-    ../../../modules/colmena_ci.nix
+    ../../../modules/colmena_ci_staging.nix
     ./haproxy
     ./tailscale
     ./caddy

diff --git a/nixos_systems/staging/wiggle/default.nix b/nixos_systems/staging/wiggle/default.nix
@@ -13,6 +13,7 @@
   ];
   localModules = [
     (modulesPath + "/profiles/qemu-guest.nix")
+    ../../../modules/colmena_ci_staging.nix
     ../../../modules/muffin-button.nix
     ../../../modules/tgs
     ../../../modules/fail2ban.nix
@@ -40,7 +41,15 @@ in {
 
   programs.nix-ld.enable = true;
 
-  # FIXME: Add networking info here
+  systemd.network = {
+    enable = true;
+    networks = {
+      "10-en" = {
+        matchConfig.name = "en*";
+        networkConfig.DHCP = "yes";
+      };
+    };
+  };
 
   networking.nameservers = [
     "9.9.9.9"