feat(kubernetes): deploy brand new ms-01 cluster

buroa · Nov 4, 2024 · 17fa0bc · 17fa0bc
1 parent d6bb880
commit 17fa0bc
Show file tree

Hide file tree

Showing 154 changed files with 400 additions and 644 deletions.
diff --git a/.github/renovate/groups.json5 b/.github/renovate/groups.json5
@@ -65,7 +65,7 @@
     {
       description: ["Talos Group"],
       groupName: "Talos",
-      matchPackagePatterns: ["buroa/installer", "siderolabs/talosctl"],
+      matchPackagePatterns: ["siderolabs/installer", "siderolabs/talosctl"],
       matchDatasources: ["docker"],
       group: {
         commitMessageTopic: "{{{groupName}}} group",

diff --git a/.taskfiles/bootstrap/Taskfile.yaml b/.taskfiles/bootstrap/Taskfile.yaml
@@ -13,7 +13,6 @@ tasks:
       - task: etcd
       - task: kubeconfig
       - task: apps
-      - task: labels
       - task: rook
       - task: flux
     preconditions:
@@ -39,41 +38,33 @@ tasks:
     preconditions:
       - test -f {{.TALOS_DIR}}/apps/helmfile.yaml
 
-  labels:
-    internal: true
-    cmds:
-      - for: { var: nodes }
-        cmd: kubectl label node {{.ITEM}} node-role.kubernetes.io/worker=
-    vars:
-      nodes: w0 w1 w2
-
   rook:
     internal: true
     cmds:
       - for: { var: nodes }
         task: rook-data
         vars:
           node: '{{.ITEM}}'
-      - for: { var: w0 }
+      - for: { var: m0 }
         task: rook-disk
         vars:
-          node: w0
+          node: m0
           serial: '{{.ITEM}}'
-      - for: { var: w1 }
+      - for: { var: m1 }
         task: rook-disk
         vars:
-          node: w1
+          node: m1
           serial: '{{.ITEM}}'
-      - for: { var: w2 }
+      - for: { var: m2 }
         task: rook-disk
         vars:
-          node: w2
+          node: m2
           serial: '{{.ITEM}}'
     vars:
-      nodes: m0 m1 m2 w0 w1 w2
-      w0: S6S2NS0TC14873N S6S2NS0TC14865L
-      w1: S6S2NS0TC14871H S6S2NS0W122087H
-      w2: S6S2NS0W120772T S6S2NS0TC14864B
+      nodes: m0 m1 m2
+      m0: S6S2NS0TC14865L # TODO: UPDATE ME
+      m1: S6S2NS0W122087H # TODO: UPDATE ME
+      m2: S6S2NS0TC14864B # TODO: UPDATE ME
 
   rook-data:
     internal: true
@@ -101,7 +92,7 @@ tasks:
       - kubectl --namespace default logs job/{{.job}}
       - kubectl --namespace default delete job {{.job}}
     env:
-      disk: /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_2TB_{{.serial}}
+      disk: /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_2TB_{{.serial}} # TODO: UPDATE ME
       job: '{{.job}}'
       node: '{{.node}}'
     vars:

diff --git a/.taskfiles/talos/Taskfile.yaml b/.taskfiles/talos/Taskfile.yaml
@@ -4,6 +4,8 @@ version: "3"
 vars:
   TALOS_CONTROLLER:
     sh: talosctl config info --output json | jq --raw-output '.endpoints[]' | shuf -n 1
+  TALOS_SCHEMATIC_ID:
+    sh: yq 'select(document_index == 1).spec.postBuild.substitute.TALOS_SCHEMATIC_ID' {{.KUBERNETES_DIR}}/apps/system-upgrade/system-upgrade-controller/ks.yaml
   TALOS_VERSION:
     sh: yq 'select(document_index == 1).spec.postBuild.substitute.TALOS_VERSION' {{.KUBERNETES_DIR}}/apps/system-upgrade/system-upgrade-controller/ks.yaml
   KUBERNETES_VERSION:
@@ -38,7 +40,7 @@ tasks:
   upgrade:
     desc: Upgrade Talos on a node
     prompt: Upgrade Talos node {{.node}} ... continue?
-    cmd: talosctl --nodes {{.node}} upgrade --image=ghcr.io/buroa/installer:{{.TALOS_VERSION}}
+    cmd: talosctl --nodes {{.node}} upgrade --image=factory.talos.dev/installer/{{.TALOS_SCHEMATIC_ID}}:{{.TALOS_VERSION}}
     requires:
       vars: ["node"]
     preconditions:

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -31,12 +31,12 @@
     "actions-runner-system": "github",
     "cert-manager": "guard",
     "default": "home",
-    "democratic-csi": "dump",
     "flux-system": "pipe",
     "kube-system": "kubernetes",
     "media": "video",
     "monitoring": "event",
     "networking": "connection",
+    "openebs-system": "dump",
     "rook-ceph": "dump",
     "security": "guard",
     "system-upgrade": "update",

diff --git a/README.md b/README.md
@@ -47,7 +47,7 @@ This is a repository for my home infrastructure and Kubernetes cluster. I try to
 
 ## ⛵ Kubernetes
 
-This semi hyper-converged cluster operates on [Talos Linux](https://github.com/siderolabs/talos), an immutable and ephemeral Linux distribution tailored for [Kubernetes](https://github.com/kubernetes/kubernetes), and is deployed on bare-metal [Apple Mac Minis](https://www.apple.com/mac-mini). [Rook](https://github.com/rook/rook) supplies my workloads with persistent block, object, and file storage, while a separate server handles media file storage. The cluster is designed to enable a full teardown without any data loss.
+This semi hyper-converged cluster operates on [Talos Linux](https://github.com/siderolabs/talos), an immutable and ephemeral Linux distribution tailored for [Kubernetes](https://github.com/kubernetes/kubernetes), and is deployed on bare-metal [MS-01](https://store.minisforum.com/products/minisforum-ms-01) workstations. [Rook](https://github.com/rook/rook) supplies my workloads with persistent block, object, and file storage, while a separate server handles media file storage. The cluster is designed to enable a full teardown without any data loss.
 
 There is a template at [onedr0p/cluster-template](https://github.com/onedr0p/cluster-template) if you want to follow along with some of the practices I use here.
 
@@ -127,20 +127,17 @@ I have two instances of `external-dns` running in my cluster. The private DNS in
   <img src="https://github.com/user-attachments/assets/e983d6c8-0899-4046-8325-b865cacb0ff9" align="center" alt="rack"/>
 </details>
 
-| Device                                          | Count | OS Disk Size | Data Disk Size | Ram  | Operating System | Purpose            |
-|-------------------------------------------------|-------|--------------|----------------|------|------------------|--------------------|
-| [Apple Mac Mini](## "Intel i7 3.2GHz w/ 10GbE") | 3     | 1TB NVMe     | -              | 64GB | Talos            | Kubernetes Workers |
-| [Apple Mac Mini](## "Intel i7 3.2GHz w/ 1GbE")  | 3     | 512GB NVMe   | -              | 32GB | Talos            | Kubernetes Masters |
-| APC SMT15000RM2UNC                              | 1     | -            | -              | -    | -                | UPS                |
-| Sabrent NVMe M.2 Thunderbolt 3 Enclosure        | 6     | -            | 2TB NVMe       | -    | -                | Rook Ceph          |
-| Sonnet 10GbE Thunderbolt 3 Adapter              | 3     | -            | -              | -    | -                | 10GbE              |
-| Synology NAS RS1221+                            | 1     | -            | 8x22TB HDD     | 32GB | -                | NFS                |
-| UDM Pro Max                                     | 1     | -            | 2x16TB HDD     | -    | UniFi OS         | Router & NVR       |
-| USP PDU Pro                                     | 1     | -            | -              | -    | UniFi OS         | PDU                |
-| USW Aggregation                                 | 1     | -            | -              | -    | UniFi OS         | Core Switch        |
-| USW Enterprise XG 24                            | 1     | -            | -              | -    | UniFi OS         | 10GbE Switch       |
-| USW Pro Max 24 PoE                              | 1     | -            | -              | -    | UniFi OS         | 2.5GbE PoE Switch  |
-
+| Device                    | Count | OS Disk Size    | Data Disk Size              | Ram  | Operating System | Purpose         |
+|---------------------------|-------|-----------------|-----------------------------|------|------------------|-----------------|
+| MS-01 (i9-13900H)         | 3     | 1.92TB M.2 NVMe | 3.84TB U.2 NVMe (rook-ceph) | 96GB | Talos            | Kubernetes      |
+| USW Pro Max 24 PoE        | 1     | -               | -                           | -    | UniFi OS         | 2.5G PoE Switch |
+| USW Pro Aggregation       | 1     | -               | -                           | -    | UniFi OS         | 10G/25G Switch  |
+| USP PDU Pro               | 1     | -               | -                           | -    | UniFi OS         | PDU             |
+| UDM Pro Max               | 1     | -               | 2x16TB HDD                  | -    | UniFi OS         | Router & NVR    |
+| Synology NAS RS1221+      | 1     | -               | 8x22TB HDD                  | 32GB | -                | NFS             |
+| APC SMT15000RM2UNC        | 1     | -               | -                           | -    | -                | UPS             |
+| TESmart 8 Port KVM Switch | 1     | -               | -                           | -    | -                | KVM             |
+| PiKVM (RasPi 4)           | 1     | 64GB (SD)       | -                           | 4GB  | PiKVM (Arch)     | KVM             |
 ---
 
 ## ⭐ Stargazers

diff --git a/kubernetes/apps/actions-runner-system/actions-runner-controller/app/helmrelease.yaml b/kubernetes/apps/actions-runner-system/actions-runner-controller/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 0.9.3
       sourceRef:
         kind: HelmRepository
-        name: actions-runner-controller-charts
+        name: actions-runner-controller
         namespace: flux-system
   install:
     crds: CreateReplace

diff --git a/kubernetes/apps/actions-runner-system/actions-runner-controller/runner/helmrelease.yaml b/kubernetes/apps/actions-runner-system/actions-runner-controller/runner/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 0.9.3
       sourceRef:
         kind: HelmRepository
-        name: actions-runner-controller-charts
+        name: actions-runner-controller
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml b/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: v1.16.1
       sourceRef:
         kind: HelmRepository
-        name: jetstack-charts
+        name: jetstack
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/databases/cloudnative-pg/app/helmrelease.yaml b/kubernetes/apps/databases/cloudnative-pg/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 0.22.1
       sourceRef:
         kind: HelmRepository
-        name: cloudnative-pg-charts
+        name: cloudnative-pg
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml b/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml
@@ -9,7 +9,7 @@ spec:
   primaryUpdateStrategy: unsupervised
   storage:
     size: 20Gi
-    storageClass: democratic-csi-local-hostpath
+    storageClass: openebs-hostpath
   superuserSecret:
     name: cloudnative-pg-secret
   enableSuperuserAccess: true

diff --git a/kubernetes/apps/databases/cloudnative-pg/ks.yaml b/kubernetes/apps/databases/cloudnative-pg/ks.yaml
@@ -32,8 +32,8 @@ spec:
     labels:
       app.kubernetes.io/name: *app
   dependsOn:
+    - name: openebs
     - name: cloudnative-pg
-    - name: democratic-csi-local-hostpath
   path: ./kubernetes/apps/databases/cloudnative-pg/cluster
   prune: true
   sourceRef:

diff --git a/kubernetes/apps/databases/dragonfly/app/helmrelease.yaml b/kubernetes/apps/databases/dragonfly/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 3.5.1
       sourceRef:
         kind: HelmRepository
-        name: bjw-s-charts
+        name: bjw-s
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/databases/emqx/app/helmrelease.yaml b/kubernetes/apps/databases/emqx/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 2.2.25
       sourceRef:
         kind: HelmRepository
-        name: emqx-charts
+        name: emqx
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/democratic-csi/democratic-csi/ks.yaml b/kubernetes/apps/democratic-csi/democratic-csi/ks.yaml
diff --git a/kubernetes/apps/democratic-csi/democratic-csi/local-hostpath/helmrelease.yaml b/kubernetes/apps/democratic-csi/democratic-csi/local-hostpath/helmrelease.yaml
diff --git a/kubernetes/apps/home/atuin/app/helmrelease.yaml b/kubernetes/apps/home/atuin/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 3.5.1
       sourceRef:
         kind: HelmRepository
-        name: bjw-s-charts
+        name: bjw-s
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/home/go2rtc/app/helmrelease.yaml b/kubernetes/apps/home/go2rtc/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 3.5.1
       sourceRef:
         kind: HelmRepository
-        name: bjw-s-charts
+        name: bjw-s
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/home/hajimari/app/helmrelease.yaml b/kubernetes/apps/home/hajimari/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 2.0.2
       sourceRef:
         kind: HelmRepository
-        name: hajimari-charts
+        name: hajimari
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/home/home-assistant/app/helmrelease.yaml b/kubernetes/apps/home/home-assistant/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 3.5.1
       sourceRef:
         kind: HelmRepository
-        name: bjw-s-charts
+        name: bjw-s
         namespace: flux-system
   install:
     remediation:

diff --git a/kubernetes/apps/home/home-assistant/app/volsync.yaml b/kubernetes/apps/home/home-assistant/app/volsync.yaml
@@ -51,7 +51,7 @@ spec:
     volumeSnapshotClassName: csi-ceph-block
     cacheAccessModes: ["ReadWriteOnce"]
     cacheCapacity: 8Gi
-    cacheStorageClassName: democratic-csi-local-hostpath
+    cacheStorageClassName: openebs-hostpath
     moverSecurityContext:
       runAsUser: 568
       runAsGroup: 568
@@ -75,7 +75,7 @@ spec:
     volumeSnapshotClassName: csi-ceph-block
     cacheAccessModes: ["ReadWriteOnce"]
     cacheCapacity: 8Gi
-    cacheStorageClassName: democratic-csi-local-hostpath
+    cacheStorageClassName: openebs-hostpath
     moverSecurityContext:
       runAsUser: 568
       runAsGroup: 568

diff --git a/kubernetes/apps/home/miniflux/app/helmrelease.yaml b/kubernetes/apps/home/miniflux/app/helmrelease.yaml
@@ -11,7 +11,7 @@ spec:
       version: 3.5.1
       sourceRef:
         kind: HelmRepository
-        name: bjw-s-charts
+        name: bjw-s
         namespace: flux-system
   install:
     remediation: