chore: Merge stage to testnet (#111)

* chore: Correct path for kubeconfig (#67)

Correct name for kubeconfig

* chore: fixed ccp-worker version (#68)

fixed ccp-worker version

* chore: Add release-please for terraform modules (#73)

* lightmare chart -> 0.1.1 (#75)

* chore: Separate module releases (#77)

Separate releases

* chore: Bump module version in examples (#81)

Bump version

* chore: Fix path in extra-files version bump (#82)

F

* chore: Correct version bump in examples (#86)

Correct version bump in examples

* chore(deps): update hashicorp/vault-action action to v3.1.0 (#66)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

* chore(main): release terraform-module-spectrum 0.1.0 (#89)

* chore(main): release terraform-module-talos 0.1.0 (#88)

Co-authored-by: Anatolios Laskaris <[email protected]>

* chore(main): release terraform-module-k3s 0.1.0 (#87)

Co-authored-by: Anatolios Laskaris <[email protected]>

* chore(deps): update helm release kubernetes-dashboard to v7.10.1 (#70)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

* feat: Update talos to version 1.9.1 and add selinux workaround (#90)

Update talos to version 1.9.1 and add selinux workaround

* feat!: talos multinode initial support (#72)

* Update

* Fix instance type

* Fix name

* F

* chore: Actually login to docker registry (#92)

Actually login to docker registry

* chore: Fix kubeconfig missing (#93)

* chore: Fix kubeconfig

* fix: kubeconfig creation waits for bootstrap (#94)

Kubeconfig depends on bootstrap

* chore(main): release terraform-module-talos 0.2.0 (#91)

* chore: Bump kubevirt cdi (#96)

Bump kubevirt cdi

* feat: Setup system metrics collection (host, k8s, cilium, kubevirt) (#95)

* fix: Bump talos version to 1.9.2 (#98)

* chore(main): release terraform-module-spectrum 0.1.1 (#99)

* chore(main): release terraform-module-talos 0.2.1 (#100)

Co-authored-by: Anatolios Laskaris <[email protected]>

* fix: Disable creation of service monitor for now (#102)

* Disable cilium metrics for now

* Disable kubevirt metrics for now

* chore(main): release terraform-module-spectrum 0.1.1 (#101)

* chore: Fix dependency on service-monitor and collect metrics from kube-scheduler and kube-controller (#103)

* chore: Disable eip on ephemeral instance (#108)

Disable eip on ephemeral instance

* chore: Test metrics push (#107)

* feat: update ccp-cu-worker to 0.16.1 (#109)

update ccp-cu-worker to 0.16.1

* chore(deps): update dependency rancher/local-path-provisioner to v0.0.31 (#105)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

---------

Co-authored-by: Enje Shakirova <[email protected]>
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: fluencebot <[email protected]>
Co-authored-by: Mike Voronov <[email protected]>

.github/release-please/config.json

@@ -3,15 +3,22 @@
   "bump-minor-pre-major": true,
   "bump-patch-for-minor-pre-major": true,
   "initial-version": "0.1.0",
+  "separate-pull-requests": true,
   "packages": {
     "terraform-modules/spectrum": {
-      "component": "terraform-module-spectrum"
+      "component": "terraform-module-spectrum",
+      "extra-files": [
+        "/examples/k3s/spectrum.tf",
+        "/examples/talos/spectrum.tf"
+      ]
     },
     "terraform-modules/k3s": {
-      "component": "terraform-module-k3s"
+      "component": "terraform-module-k3s",
+      "extra-files": ["/examples/k3s/main.tf"]
     },
     "terraform-modules/talos": {
-      "component": "terraform-module-talos"
+      "component": "terraform-module-talos",
+      "extra-files": ["/examples/talos/main.tf"]
     }
   }
 }

.github/release-please/manifest.json

@@ -1 +1,5 @@
-{}
+{
+  "terraform-modules/talos": "0.2.1",
+  "terraform-modules/spectrum": "0.1.1",
+  "terraform-modules/k3s": "0.1.0"
+}

.github/workflows/terraform.yml

@@ -45,7 +45,7 @@ jobs:
           ref: ${{ inputs.command == 'destroy' && 'main' || inputs.branch }}
 
       - name: Get Vault token
-        uses: hashicorp/vault-action@v3.0.0
+        uses: hashicorp/vault-action@v3.1.0
         with:
           url: "https://vault.fluence.dev"
           path: jwt/github
@@ -108,7 +108,7 @@ jobs:
         run: |
           case ${{ inputs.command }} in
           create)
-            terraform apply -target module.talos.kubeconfig.path -auto-approve
+            terraform apply -target module.talos.local_sensitive_file.kubeconfig -auto-approve
             terraform apply -auto-approve
           ;;
           destroy)

ephemeral/instance.tf

@@ -12,11 +12,12 @@ resource "digitalocean_ssh_key" "spectrum" {
 }
 
 data "digitalocean_image" "talos" {
-  name = "talos-v1.8.4"
+  name = "talos-v1.9.2"
 }
 
-resource "digitalocean_droplet" "talos" {
-  name     = "rnd-${local.prefix}-spectrum-cp"
+resource "digitalocean_droplet" "cp" {
+  count    = 1
+  name     = "rnd-${local.prefix}-spectrum-cp-${count.index}"
   size     = "s-8vcpu-16gb"
   image    = data.digitalocean_image.talos.id
   region   = "fra1"
@@ -30,8 +31,3 @@ resource "digitalocean_droplet" "talos" {
     local.prefix,
   ]
 }
-
-resource "digitalocean_reserved_ip" "l2" {
-  droplet_id = digitalocean_droplet.talos.id
-  region     = digitalocean_droplet.talos.region
-}

ephemeral/main.tf

@@ -13,10 +13,18 @@ data "vault_generic_secret" "docker" {
 module "talos" {
   source       = "../terraform-modules/talos"
   cluster_name = local.prefix
-  server_ip    = digitalocean_droplet.talos.ipv4_address
 
-  config_patches = [
-    file("${path.root}/config_patch.yml"),
+  control_planes = [
+    {
+      name      = "cp-0"
+      server_ip = digitalocean_droplet.cp[0].ipv4_address
+      config_patches = [
+        templatefile("${path.root}/patches/registry.yml", {
+          docker_username = data.vault_generic_secret.docker.data.username,
+          docker_password = data.vault_generic_secret.docker.data.password
+        })
+      ]
+    },
   ]
 }
 
@@ -36,7 +44,6 @@ module "spectrum" {
     DOTOKEN         = base64encode(data.vault_generic_secret.spectrum.data.token)
     DOMAIN          = "${local.prefix}.fluence.dev"
     PREFIX          = local.prefix
-    LOADBALANCER_IP = digitalocean_droplet.talos.ipv4_address
-    L2_IP           = digitalocean_reserved_ip.l2.ip_address
+    LOADBALANCER_IP = digitalocean_droplet.cp[0].ipv4_address
   }
 }

examples/k3s/main.tf

@@ -1,21 +1,9 @@
 module "k3s" {
-  source               = "github.com/fluencelabs/spectrum//terraform-modules/k3s"
+  source               = "git::https://github.com/fluencelabs/spectrum.git//terraform-modules/k3s?ref=terraform-module-k3s-v0.1.0" # x-release-please-version
   kubeconfigs_location = "${path.root}/secrets"
   server_name          = local.server_name
   server_ip_address    = local.server_ip_address
   ssh_key              = local.ssh_key
   ssh_user             = local.ssh_user
   ssh_port             = local.ssh_port
 }
-
-provider "helm" {
-  kubernetes {
-    config_path = module.k3s.kubeconfig_file
-  }
-}
-
-module "spectrum" {
-  depends_on      = [module.k3s]
-  source          = "github.com/fluencelabs/spectrum//terraform-modules/spectrum"
-  cluster_flavour = "k3s"
-}

examples/k3s/spectrum.tf

@@ -0,0 +1,11 @@
+provider "helm" {
+  kubernetes {
+    config_path = module.k3s.kubeconfig_file
+  }
+}
+
+module "spectrum" {
+  depends_on      = [module.k3s]
+  source          = "git::https://github.com/fluencelabs/spectrum.git//terraform-modules/spectrum?ref=terraform-module-spectrum-v0.1.1" # x-release-please-version
+  cluster_flavour = "k3s"
+}

examples/talos/README.md

@@ -2,9 +2,9 @@
 
 ## Server customization
 
-In talos machine is configured from a single configuration file in yaml format. Talos terraform module allows to specify overlays of the main configuration file maintained by cloudless labs [here](https://github.com/fluencelabs/spectrum/blob/main/terraform-modules/talos/templates/controlplane_patch.yml) with `config_pathes` option.
+In talos machine is configured from a single configuration file in yaml format. Talos terraform module allows to specify overlays of the main configuration file maintained by cloudless labs [here](https://github.com/fluencelabs/spectrum/blob/main/terraform-modules/talos/base_config.yml) with `config_pathes` option.
 
-You can configure server specific things like layout of disks or network configuration. Checkout [talos documentation](https://www.talos.dev/v1.9/reference/configuration/v1alpha1/config/) and see `config_patch.yml` for an example of a `bond` interface configuration.
+You can configure server specific things like layout of disks or network configuration. Checkout [talos documentation](https://www.talos.dev/v1.9/reference/configuration/v1alpha1/config/) and see [`cp-0.yml`](https://github.com/fluencelabs/blob/main/examples/talos/patches/cp-0.yml) for an example of a `bond` interface configuration.
 
 ## Terraform state
 

examples/talos/main.tf

@@ -21,27 +21,17 @@ provider "helm" {
 }
 
 module "talos" {
-  source       = "git::https://github.com/fluencelabs/spectrum.git//terraform-modules/talos?ref=main"
-  cluster_name = var.cluster_name
-  server_ip    = var.server_ip
-  # config_patches = [
-  #   file("${path.root}/config_patch.yml"),
-  # ]
-}
-
-module "spectrum" {
-  depends_on = [module.talos]
-  source     = "git::https://github.com/fluencelabs/spectrum.git//terraform-modules/spectrum?ref=main"
-  network    = "main"
-  cluster    = "default"
-}
-
-variable "server_ip" {
-  type        = string
-  description = "IP at which server is accessible"
-}
+  source       = "git::https://github.com/fluencelabs/spectrum.git//terraform-modules/talos?ref=terraform-module-talos-v0.2.1" # x-release-please-version
+  cluster_name = "my-cluster"
 
-variable "cluster_name" {
-  type        = string
-  description = "Name used in k8s and talos to distinguish between clusters"
+  control_planes = [
+    {
+      name      = "cp-0"
+      server_ip = "1.2.3.4"
+      config_patches = [
+        file("${path.root}/patches/base.yml"),
+        file("${path.root}/patches/cp-0.yml"),
+      ]
+    },
+  ]
 }

examples/talos/patches/base.yml

@@ -0,0 +1,4 @@
+machine:
+  time:
+    servers:
+      - time.cloudflare.com

examples/talos/config_patch.yml → examples/talos/patches/cp-0.yml

@@ -30,7 +30,6 @@ machine:
       serial: Y0L0A031T5N8
 
   network:
-    hostname: foobar
     interfaces:
       - interface: bond0
         dhcp: false

examples/talos/spectrum.tf

@@ -0,0 +1,6 @@
+module "spectrum" {
+  depends_on = [module.talos]
+  source     = "git::https://github.com/fluencelabs/spectrum.git//terraform-modules/spectrum?ref=terraform-module-spectrum-v0.1.1" # x-release-please-version
+  network    = "main"
+  cluster    = "default"
+}

flux/apps/kubernetes-dashboard/helm-release.yml

@@ -9,7 +9,7 @@ spec:
   chart:
     spec:
       chart: kubernetes-dashboard
-      version: 7.10.0
+      version: 7.10.1
       sourceRef:
         kind: HelmRepository
         name: kubernetes-dashboard

flux/clusters/ephemeral/grafana-ingress.yml

@@ -4,7 +4,7 @@ kind: Namespace
 metadata:
   name: monitoring
   labels:
-    app.kubernetes.io/component: monitoring
+    app.kubernetes.io/component: spectrum-monitoring
     pod-security.kubernetes.io/enforce: privileged
 ---
 apiVersion: networking.k8s.io/v1

flux/clusters/ephemeral/kustomization.yml

@@ -13,5 +13,5 @@ resources:
   - ../../components/monitoring
   - hubble-ingress.yml
   - grafana-ingress.yml
-  - cilium-l2.yml
+  #  - cilium-l2.yml
   - lightmare.yml

flux/components/kubevirt/app/kustomization.yml

@@ -3,8 +3,8 @@ kind: Kustomization
 resources:
   - https://github.com/kubevirt/kubevirt/releases/download/v1.4.0/kubevirt-operator.yaml
   - https://github.com/kubevirt/kubevirt/releases/download/v1.4.0/kubevirt-cr.yaml
-  - https://github.com/kubevirt/containerized-data-importer/releases/download/v1.60.3/cdi-operator.yaml
-  - https://github.com/kubevirt/containerized-data-importer/releases/download/v1.60.3/cdi-cr.yaml
+  - https://github.com/kubevirt/containerized-data-importer/releases/download/v1.61.0/cdi-operator.yaml
+  - https://github.com/kubevirt/containerized-data-importer/releases/download/v1.61.0/cdi-cr.yaml
 patches:
   - target:
       kind: Deployment

flux/monitoring/configs/kustomization.yaml → flux/components/monitoring/configs/flux/kustomization.yaml

@@ -10,8 +10,10 @@ configMapGenerator:
       - dashboards/cluster.json
       - dashboards/logs.json
     options:
+      annotations:
+        grafana_folder: "Flux"
       labels:
         grafana_dashboard: "1"
-        app.kubernetes.io/part-of: flux
-        app.kubernetes.io/component: monitoring
+        app.kubernetes.io/part-of: spectrum-monitoring
+        app.kubernetes.io/component: flux
         kustomize.toolkit.fluxcd.io/substitute: disabled

flux/monitoring/configs/podmonitor.yaml → flux/components/monitoring/configs/flux/podmonitor.yaml

@@ -3,8 +3,8 @@ kind: PodMonitor
 metadata:
   name: flux-system
   labels:
-    app.kubernetes.io/part-of: flux
-    app.kubernetes.io/component: monitoring
+    app.kubernetes.io/part-of: spectrum-monitoring
+    app.kubernetes.io/component: flux
 spec:
   namespaceSelector:
     matchNames:

flux/components/monitoring/configs/kubernetes/kustomization.yml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: monitoring
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    grafana_dashboard: "1"
+    app.kubernetes.io/component: kubernetes
+    app.kubernetes.io/part-of: spectrum-monitoring
+commonAnnotations:
+  grafana_folder: "Kubernetes"
+
+configMapGenerator:
+  - name: dashboards-k8s-views-global
+    files:
+      - https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/refs/tags/v2.5.3/dashboards/k8s-views-global.json
+
+  - name: dashboards-k8s-views-namespaces
+    files:
+      - https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/refs/tags/v2.5.3/dashboards/k8s-views-namespaces.json
+
+  - name: dashboards-k8s-views-nodes
+    files:
+      - https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/refs/tags/v2.5.3/dashboards/k8s-views-nodes.json
+
+  - name: dashboards-k8s-views-pods
+    files:
+      - https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/refs/tags/v2.5.3/dashboards/k8s-views-pods.json

flux/components/monitoring/configs/kubevirt.yml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kubevirt-monitoring
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  path: ./flux/components/monitoring/configs/kubevirt
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: spectrum
+    namespace: flux-system
+  dependsOn:
+    - name: kubevirt
+      namespace: flux-system

flux/components/monitoring/configs/kubevirt/kustomization.yml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: monitoring
+resources:
+  - service-monitor.yml
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    grafana_dashboard: "1"
+    app.kubernetes.io/part-of: spectrum-monitoring
+    app.kubernetes.io/component: kubevirt
+commonAnnotations:
+  grafana_folder: "Kubevirt"
+
+configMapGenerator:
+  - name: dashboards-k8s-kubevirt
+    files:
+      - https://raw.githubusercontent.com/aenix-io/cozystack/refs/tags/v0.23.1/dashboards/kubevirt/kubevirt-control-plane.json

flux/components/monitoring/configs/kubevirt/service-monitor.yml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kubevirt
+  labels:
+    app.kubernetes.io/part-of: spectrum-monitoring
+    app.kubernetes.io/component: kubevirt
+spec:
+  selector:
+    matchLabels:
+      prometheus.kubevirt.io: "true"
+  namespaceSelector:
+    matchNames:
+    - kubevirt
+  endpoints:
+  - port: metrics
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true

flux/components/monitoring/configs/kustomization.yml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - flux
+  - system
+  - kubernetes
+  - kubevirt.yml