From 73051aa42e42fc12d3d9d6c63a8716cfce2b6c01 Mon Sep 17 00:00:00 2001 From: Lorwp Date: Sat, 15 Feb 2025 13:34:23 +1100 Subject: [PATCH 1/7] Re-Implement CI for the repo, plus some housekeeping This uses deployments to keep runners only using specific credentials for staging and production. Proper handling of staging so people can test deploy shit will come later. This also uses DHCP on wiggle, because we're not using a public IP for it any longer. Rotates the key colmena_ci uses too, plus adds a staging version of the module for uh, staging. Finally, removes frankfurt 2 and 3, plus fixes the pkgs-unstable reference on the arm nodes to use arm. Co-authored-by: alexkar598 <25136265+alexkar598@users.noreply.github.com> --- .github/workflows/colmena.yml | 54 ++++++++++------- flake.nix | 74 +++-------------------- modules/colmena_ci.nix | 2 +- modules/colmena_ci_staging.nix | 31 ++++++++++ modules/headscale/headscale-acl.json | 27 +++++++++ nixos_systems/relay-node/staging/base.nix | 2 +- nixos_systems/staging/wiggle/default.nix | 11 +++- 7 files changed, 109 insertions(+), 92 deletions(-) create mode 100644 modules/colmena_ci_staging.nix diff --git a/.github/workflows/colmena.yml b/.github/workflows/colmena.yml index 75d7e97..374ba0c 100644 --- a/.github/workflows/colmena.yml +++ b/.github/workflows/colmena.yml @@ -8,28 +8,26 @@ on: jobs: build: name: Build - runs-on: ubuntu-24.04 strategy: matrix: + runner: [ubuntu-24.04] node: - - tgsatan - - blockmoths - - vpn - - \@relay - - \@staging + - "tgsatan" + - "blockmoths" + - "vpn" + - "@relay-amd64" + - "@staging" + include: + - node: "@relay-arm" + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} steps: # We use commit hashes for specifying versions here, so a malicious tag can't gain access to our secrets (At least while sha-1 collisions are rare, anyway) - - name: Install private ssh key - uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # Install our ssh key. TODO: Replace with our own bash script - with: - key: ${{ secrets.COLMENA_SSH_KEY }} - name: id_ed25519 - known_hosts: ${{ secrets.COLMENA_KNOWN_HOSTS }} - - name: Login to headscale uses: tailscale/github-action@8688eb839e58e6b25c1ae96cd99d1c173299b842 # Connect to headscale + if: github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main' with: - authkey: ${{ secrets.TS_AUTHKEY }} + authkey: ${{ secrets.TS_BUILD_AUTHKEY }} args: --login-server=https://vpn.tgstation13.org - name: Checkout Repository @@ -40,14 +38,14 @@ jobs: with: extra_nix_config: | accept-flake-config = true - extra-substituters = https://attic.tgstation13.org/tgstation-infrastructure - extra-trusted-public-keys = tgstation-infrastructure:tNpjd5GxK1xymRHsJdBLTpeDScA2mVPdKA/eIOLOE0I= - name: Setup attic Binary Cache + if: github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main' # Format for pointing to caches is server:cache in these commands run: | nix profile install nixpkgs#attic-client attic login tgstation https://attic.tgstation13.org ${{ secrets.ATTIC_JWT_TOKEN }} + attic use tgstation:tgstation-infrastructure - name: Build closure run: nix run github:zhaofengli/colmena -- build --impure -v --eval-node-limit 2 --keep-result --on ${{ matrix.node }} @@ -59,23 +57,33 @@ jobs: deploy: name: Deploy needs: build - runs-on: ubuntu-24.04 if: ${{ github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main' }} + environment: ${{ matrix.environment }} strategy: matrix: + runner: [ubuntu-24.04] node: - - tgsatan - - blockmoths - - vpn - - \@relay - - \@staging + - "tgsatan" + - "blockmoths" + - "vpn" + - "@relay-amd64" + environment: + - production + include: + - runner: ubuntu-24.04-arm + node: "@relay-arm" + environment: production + - runner: ubuntu-24.04 + node: "@staging" + environment: staging + runs-on: ${{ matrix.runner }} steps: - name: Install private ssh key uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # Install our ssh key. TODO: Replace with our own bash script with: key: ${{ secrets.COLMENA_SSH_KEY }} name: id_ed25519 - known_hosts: ${{ secrets.COLMENA_KNOWN_HOSTS }} + known_hosts: ${{ vars.COLMENA_KNOWN_HOSTS }} - name: Login to headscale uses: tailscale/github-action@8688eb839e58e6b25c1ae96cd99d1c173299b842 # Connect to headscale diff --git a/flake.nix b/flake.nix index 1d1ffab..7553b01 100755 --- a/flake.nix +++ b/flake.nix @@ -121,7 +121,7 @@ targetHost = "chicago.tg.lan"; targetUser = "deploy"; tags = [ - "relay" + "relay-amd64" ]; }; imports = @@ -138,7 +138,7 @@ targetHost = "atlanta.tg.lan"; targetUser = "deploy"; tags = [ - "relay" + "relay-amd64" ]; }; imports = @@ -150,40 +150,6 @@ ]; }; - frankfurt2 = { - deployment = { - targetHost = "frankfurt2.tg.lan"; - targetUser = "deploy"; - tags = [ - "relay" - ]; - }; - imports = - flakeModules - ++ [ - (import ./modules/base.nix) - (import ./users) - (import ./nixos_systems/relay-node/eu/frankfurt2.nix) - ]; - }; - - frankfurt3 = { - deployment = { - targetHost = "frankfurt3.tg.lan"; - targetUser = "deploy"; - tags = [ - "relay" - ]; - }; - imports = - flakeModules - ++ [ - (import ./modules/base.nix) - (import ./users) - (import ./nixos_systems/relay-node/eu/frankfurt3.nix) - ]; - }; - blockmoths = { deployment = { targetHost = "blockmoths.tg.lan"; @@ -235,7 +201,7 @@ targetHost = "lime.tg.lan"; targetUser = "deploy"; tags = [ - "relay" + "relay-amd64" ]; }; imports = @@ -268,7 +234,7 @@ targetHost = "dachshund.tg.lan"; targetUser = "deploy"; tags = [ - "relay" + "relay-arm" ]; }; nixpkgs.system = "aarch64-linux"; @@ -285,7 +251,7 @@ targetHost = "knipp.tg.lan"; targetUser = "deploy"; tags = [ - "relay" + "relay-arm" ]; }; nixpkgs.system = "aarch64-linux"; @@ -305,8 +271,6 @@ vpn chicago atlanta - frankfurt2 - frankfurt3 blockmoths wiggle warsaw @@ -342,19 +306,19 @@ }; bratwurst = { pkgs-unstable = import nixpkgs-unstable { - system = "x86_64-linux"; + system = "aarch64-linux"; config.allowUnfree = true; }; }; dachshund = { pkgs-unstable = import nixpkgs-unstable { - system = "x86_64-linux"; + system = "aarch64-linux"; config.allowUnfree = true; }; }; knipp = { pkgs-unstable = import nixpkgs-unstable { - system = "x86_64-linux"; + system = "aarch64-linux"; config.allowUnfree = true; }; }; @@ -442,28 +406,6 @@ }; }; }; - frankfurt2 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = flakeModules ++ frankfurt2.imports; - specialArgs = { - inherit self inputs nixpkgs fenix; - pkgs-unstable = import nixpkgs-unstable { - system = "x86_64-linux"; - config.allowUnfree = true; - }; - }; - }; - frankfurt3 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = flakeModules ++ frankfurt3.imports; - specialArgs = { - inherit self inputs nixpkgs fenix; - pkgs-unstable = import nixpkgs-unstable { - system = "x86_64-linux"; - config.allowUnfree = true; - }; - }; - }; warsaw = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = flakeModules ++ warsaw.imports; diff --git a/modules/colmena_ci.nix b/modules/colmena_ci.nix index 9026a9b..d1ed9da 100644 --- a/modules/colmena_ci.nix +++ b/modules/colmena_ci.nix @@ -17,7 +17,7 @@ in { openssh.authorizedKeys.keys = deployUsers ++ [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINerE77pg5ziJ2adbSZ7ftCa3kX49C1C2FSJd6h6XVP+ deploy@tgstation-nix" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPQx1+Obgbo+YUubcQNFr4ry5Iob3U0fW3myAcG4PS79 deploy@tgstation-infra" ]; }; diff --git a/modules/colmena_ci_staging.nix b/modules/colmena_ci_staging.nix new file mode 100644 index 0000000..fd80270 --- /dev/null +++ b/modules/colmena_ci_staging.nix @@ -0,0 +1,31 @@ +{ + pkgs, + config, + ... +}: let + deployUsers = import ./ssh_keys.nix; +in { + # Configuration required to use github actions to deploy to nodes + users.users.deploy = { + isNormalUser = true; + + extraGroups = [ + "wheel" # Needed for nixos-rebuild. Originally the idea was to just limit it to a group and setup sudo to allow nixos-rebuild as that user, but that would result in them being able to modify system.activationScripts regardless and run scripts as root, so it's not very useful + ]; + + group = "deploy"; + openssh.authorizedKeys.keys = + deployUsers + ++ [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL9Pizmnyye3CmgonAAzdVBeiVUyXs+pk8LFPAaUFnVi deploy@tgstation-infra-staging" + ]; + }; + + users.groups.deploy = {}; + nix.settings = { + # Allow our user to use binary caches during builds explicitly + trusted-users = [ + "deploy" + ]; + }; +} diff --git a/modules/headscale/headscale-acl.json b/modules/headscale/headscale-acl.json index b756419..8b737cb 100755 --- a/modules/headscale/headscale-acl.json +++ b/modules/headscale/headscale-acl.json @@ -48,6 +48,12 @@ "tag:ci": [ "group:op" ], + "tag:ci-staging": [ + "group:op" + ], + "tag:ci-build": [ + "group:op" + ], "tag:dbcluster": [ "group:op" ], @@ -73,6 +79,27 @@ "tag:backend:443" ] }, + { + "action": "accept", + "src": [ + "tag:ci-build" + ], + "dst": [ + "tag:backend:80", + "tag:backend:443" + ] + }, + { + "action": "accept", + "src": [ + "tag:ci-staging" + ], + "dst": [ + "tag:staging:22", + "tag:backend:80", + "tag:backend:443" + ] + }, { "action": "accept", "src": [ diff --git a/nixos_systems/relay-node/staging/base.nix b/nixos_systems/relay-node/staging/base.nix index 65ec996..47f4490 100644 --- a/nixos_systems/relay-node/staging/base.nix +++ b/nixos_systems/relay-node/staging/base.nix @@ -8,7 +8,7 @@ ../../../modules/fail2ban.nix ../../../modules/openssh.nix ../../../modules/tailscale.nix - ../../../modules/colmena_ci.nix + ../../../modules/colmena_ci_staging.nix ./haproxy ./tailscale ./caddy diff --git a/nixos_systems/staging/wiggle/default.nix b/nixos_systems/staging/wiggle/default.nix index 170ea5e..10ef5d9 100644 --- a/nixos_systems/staging/wiggle/default.nix +++ b/nixos_systems/staging/wiggle/default.nix @@ -13,6 +13,7 @@ ]; localModules = [ (modulesPath + "/profiles/qemu-guest.nix") + ../../../modules/colmena_ci_staging.nix ../../../modules/muffin-button.nix ../../../modules/tgs ../../../modules/fail2ban.nix @@ -40,7 +41,15 @@ in { programs.nix-ld.enable = true; - # FIXME: Add networking info here + systemd.network = { + enable = true; + networks = { + "10-en" = { + matchConfig.name = "en*"; + networkConfig.DHCP = "yes"; + }; + }; + }; networking.nameservers = [ "9.9.9.9" From e1c3a74fec6f3d67af5fb9dfb0a01a5a3f95b64b Mon Sep 17 00:00:00 2001 From: Lorwp Date: Sun, 16 Feb 2025 11:13:03 +1100 Subject: [PATCH 2/7] Apply suggestions from code review Co-authored-by: Riggle <27156122+RigglePrime@users.noreply.github.com> --- .github/workflows/colmena.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/colmena.yml b/.github/workflows/colmena.yml index 374ba0c..ce80a80 100644 --- a/.github/workflows/colmena.yml +++ b/.github/workflows/colmena.yml @@ -25,7 +25,7 @@ jobs: # We use commit hashes for specifying versions here, so a malicious tag can't gain access to our secrets (At least while sha-1 collisions are rare, anyway) - name: Login to headscale uses: tailscale/github-action@8688eb839e58e6b25c1ae96cd99d1c173299b842 # Connect to headscale - if: github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main' + if: github.repository == 'tgstation-operations/infrastructure' && github.ref == 'refs/heads/main' with: authkey: ${{ secrets.TS_BUILD_AUTHKEY }} args: --login-server=https://vpn.tgstation13.org @@ -40,7 +40,7 @@ jobs: accept-flake-config = true - name: Setup attic Binary Cache - if: github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main' + if: github.repository == 'tgstation-operations/infrastructure' && github.ref == 'refs/heads/main' # Format for pointing to caches is server:cache in these commands run: | nix profile install nixpkgs#attic-client @@ -57,7 +57,7 @@ jobs: deploy: name: Deploy needs: build - if: ${{ github.repository == 'tgstation-operations/tgstation-nix' && github.ref == 'refs/heads/main' }} + if: ${{ github.repository == 'tgstation-operations/infrastructure' && github.ref == 'refs/heads/main' }} environment: ${{ matrix.environment }} strategy: matrix: From a72c7806598521d87a7346b3bae4db09a1c38057 Mon Sep 17 00:00:00 2001 From: Lorwp Date: Tue, 18 Feb 2025 11:55:43 +1100 Subject: [PATCH 3/7] rm networkd conf Co-authored-by: Riggle <27156122+RigglePrime@users.noreply.github.com> --- nixos_systems/staging/wiggle/default.nix | 9 --------- 1 file changed, 9 deletions(-) diff --git a/nixos_systems/staging/wiggle/default.nix b/nixos_systems/staging/wiggle/default.nix index 8d70d23..2fc68e7 100644 --- a/nixos_systems/staging/wiggle/default.nix +++ b/nixos_systems/staging/wiggle/default.nix @@ -42,15 +42,6 @@ in { programs.nix-ld.enable = true; - systemd.network = { - enable = true; - networks = { - "10-en" = { - matchConfig.name = "en*"; - networkConfig.DHCP = "yes"; - }; - }; - }; networking.nameservers = [ "9.9.9.9" From e8677b2e3c3a755b83f044acc23c36a784391fd1 Mon Sep 17 00:00:00 2001 From: Lorwp Date: Tue, 18 Feb 2025 12:22:21 +1100 Subject: [PATCH 4/7] Update flake.lock (excl TGS) Since node insists on building, so it's probably no longer in the cache upstream for the specific version we're pinned to --- flake.lock | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/flake.lock b/flake.lock index 5ac0422..9abee38 100644 --- a/flake.lock +++ b/flake.lock @@ -99,11 +99,11 @@ ] }, "locked": { - "lastModified": 1739634831, - "narHash": "sha256-xFnU+uUl48Icas2wPQ+ZzlL2O3n8f6J2LrzNK9f2nng=", + "lastModified": 1739791827, + "narHash": "sha256-l6ooDEtfzet9qRQxlb5A+H6eY7VPpdiGMwqX0nqD1xM=", "owner": "nix-community", "repo": "disko", - "rev": "fa5746ecea1772cf59b3f34c5816ab3531478142", + "rev": "af4a580628e98302bb922c01e1169ce08d7bee57", "type": "github" }, "original": { @@ -157,11 +157,11 @@ "rust-analyzer-src": "rust-analyzer-src_2" }, "locked": { - "lastModified": 1739687593, - "narHash": "sha256-K7+n5+W2OrqEjeVb4422YxwNw1m4lCfnd+QWCnm+Dgs=", + "lastModified": 1739774189, + "narHash": "sha256-ZupCPljRian/MbkNdwsfMOePRwd6OATKxp/45DE3EL0=", "owner": "nix-community", "repo": "fenix", - "rev": "a712b739a49e10fe73de366a42a43b2714e41bfc", + "rev": "6b5b5f59f75aa4743fe4c150acf7cb8f8cabf787", "type": "github" }, "original": { @@ -224,11 +224,11 @@ ] }, "locked": { - "lastModified": 1739570999, - "narHash": "sha256-eCc0/Q4bPpe4/AS+uzIrHLJcR6BxPQ69q2kD0/Qe6rU=", + "lastModified": 1739757849, + "narHash": "sha256-Gs076ot1YuAAsYVcyidLKUMIc4ooOaRGO0PqTY7sBzA=", "owner": "nix-community", "repo": "home-manager", - "rev": "254d47082e23dbf72fdeca1da6fe1da420f478d8", + "rev": "9d3d080aec2a35e05a15cedd281c2384767c2cfe", "type": "github" }, "original": { @@ -276,11 +276,11 @@ }, "nixos-hardware": { "locked": { - "lastModified": 1738816619, - "narHash": "sha256-5yRlg48XmpcX5b5HesdGMOte+YuCy9rzQkJz+imcu6I=", + "lastModified": 1739798439, + "narHash": "sha256-GyipmjbbQEaosel/+wq1xihCKbv0/e1LU00x/8b/fP4=", "owner": "NixOS", "repo": "nixos-hardware", - "rev": "2eccff41bab80839b1d25b303b53d339fbb07087", + "rev": "3e2ea8a49d4d76276b0f4e2041df8ca5c0771371", "type": "github" }, "original": { @@ -308,11 +308,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1739611738, - "narHash": "sha256-3bnOIZz8KXtzcaXGuH9Eriv0HiQyr1EIfcye+VHLQZE=", + "lastModified": 1739822428, + "narHash": "sha256-eVH9Ggf0eojNMoRkksP7SfOMpI8ITLNfmoZrKyfQ8hU=", "owner": "nixos", "repo": "nixpkgs", - "rev": "31ff66eb77d02e9ac34b7256a02edb1c43fb9998", + "rev": "a3a07ac733f5aa4a1b1800d4a4042b65c6a9865f", "type": "github" }, "original": { @@ -324,11 +324,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1739484910, - "narHash": "sha256-wjWLzdM7PIq4ZAe7k3vyjtgVJn6b0UeodtRFlM/6W5U=", + "lastModified": 1739758141, + "narHash": "sha256-uq6A2L7o1/tR6VfmYhZWoVAwb3gTy7j4Jx30MIrH0rE=", "owner": "nixos", "repo": "nixpkgs", - "rev": "0b73e36b1962620a8ac551a37229dd8662dac5c8", + "rev": "c618e28f70257593de75a7044438efc1c1fc0791", "type": "github" }, "original": { @@ -391,11 +391,11 @@ "rust-analyzer-src_2": { "flake": false, "locked": { - "lastModified": 1739512757, - "narHash": "sha256-QfmtsyySvQSEKLuB850AmyqpNQRP+T57vuZnGIpmGD4=", + "lastModified": 1739771915, + "narHash": "sha256-i76MMFSkCr4kDwurK8CACwZq7qEgVEgIzkOr2kiuAKk=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "40e4f9130f4e44f20961a7cf4ade46325126698b", + "rev": "84b6936e0856d0cac8d616c5ba3306155d8b3b1d", "type": "github" }, "original": { From 130cd1788da8774ea2e27786b41789b42aceeb61 Mon Sep 17 00:00:00 2001 From: Lorwp Date: Tue, 18 Feb 2025 13:01:06 +1100 Subject: [PATCH 5/7] Add step to show the local nix config before building --- .github/workflows/colmena.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/colmena.yml b/.github/workflows/colmena.yml index ce80a80..4767e8c 100644 --- a/.github/workflows/colmena.yml +++ b/.github/workflows/colmena.yml @@ -47,6 +47,9 @@ jobs: attic login tgstation https://attic.tgstation13.org ${{ secrets.ATTIC_JWT_TOKEN }} attic use tgstation:tgstation-infrastructure + - name: Print nix config before Build + run: nix config show + - name: Build closure run: nix run github:zhaofengli/colmena -- build --impure -v --eval-node-limit 2 --keep-result --on ${{ matrix.node }} @@ -106,6 +109,9 @@ jobs: run: | nix profile install nixpkgs#attic-client attic login tgstation https://attic.tgstation13.org ${{ secrets.ATTIC_JWT_TOKEN }} + + - name: Print nix config before Build + run: nix config show - name: Deploy closure to Nodes run: nix run github:zhaofengli/colmena -- apply --impure -v --on ${{ matrix.node }} From 47f920513ef07a69620018c0c181c87cc9f0fd02 Mon Sep 17 00:00:00 2001 From: Lorwp Date: Tue, 18 Feb 2025 13:05:35 +1100 Subject: [PATCH 6/7] Ensure we're using the nix-community cache Apparently colmena is too good to use the flake config. Annoying --- .github/workflows/colmena.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/colmena.yml b/.github/workflows/colmena.yml index 4767e8c..2b07b55 100644 --- a/.github/workflows/colmena.yml +++ b/.github/workflows/colmena.yml @@ -38,6 +38,9 @@ jobs: with: extra_nix_config: | accept-flake-config = true + extra-substituters = https://nix-community.cachix.org + extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= + - name: Setup attic Binary Cache if: github.repository == 'tgstation-operations/infrastructure' && github.ref == 'refs/heads/main' From 43b9be70d3241747a70eedfdf52fa32b17226344 Mon Sep 17 00:00:00 2001 From: Lorwp Date: Tue, 18 Feb 2025 13:13:40 +1100 Subject: [PATCH 7/7] Make sure we're only running one job at a time --- .github/workflows/colmena.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/colmena.yml b/.github/workflows/colmena.yml index 2b07b55..5a3834d 100644 --- a/.github/workflows/colmena.yml +++ b/.github/workflows/colmena.yml @@ -5,6 +5,10 @@ on: branches: [main] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: name: Build