diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index ffbb26b3..58c0fb9c 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -64,7 +64,7 @@ jobs: method: jwt jwtGithubAudience: "https://github.com/fluencelabs" jwtTtl: 300 - exportToken: false + exportToken: true secrets: | kv/digitalocean/gitops token | DIGITALOCEAN_TOKEN ; kv/cloudflare/gitops token | CLOUDFLARE_API_TOKEN ; diff --git a/Justfile b/Justfile index 15aedf05..e8803232 100644 --- a/Justfile +++ b/Justfile @@ -1,4 +1,3 @@ download: rm -f kubeconfig talosconfig gh run download -n configs - nix-direnv-reload diff --git a/README.md b/README.md index c5aa446f..4798198b 100644 --- a/README.md +++ b/README.md @@ -39,3 +39,10 @@ export KUBECONFIG=./kubeconfig #### Start using the cluster https://kubernetes.io/docs/reference/kubectl/quick-reference/ + + +## Misc +### Accessing kubernetes dashboard +``` +kubectl -n kubernetes-dashboard create token kubernetes-dashboard-admin +``` diff --git a/flake.nix b/flake.nix index 5ec6de31..2ddc50bf 100644 --- a/flake.nix +++ b/flake.nix @@ -32,7 +32,7 @@ pkgs.kubernetes-helm pkgs.kubevirt pkgs.cilium-cli - pkgs.flux + pkgs.fluxcd pkgs.terraform ]; diff --git a/flux/dev/cert-issuer/app/cluster-issuer.yml b/flux/dev/cert-issuer/app/cluster-issuer.yml new file mode 100644 index 00000000..5b2c8c98 --- /dev/null +++ b/flux/dev/cert-issuer/app/cluster-issuer.yml @@ -0,0 +1,18 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt + namespace: kube-system +spec: + acme: + email: devops@fluence.one + # server: https://acme-v02.api.letsencrypt.org/directory + server: https://acme-staging-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: letsencrypt + solvers: + - dns01: + digitalocean: + tokenSecretRef: + name: digitalocean-token + key: token diff --git a/flux/dev/cert-issuer/app/kustomization.yml b/flux/dev/cert-issuer/app/kustomization.yml new file mode 100644 index 00000000..86fec029 --- /dev/null +++ b/flux/dev/cert-issuer/app/kustomization.yml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cluster-issuer.yml diff --git a/flux/dev/cert-issuer/ks.yml b/flux/dev/cert-issuer/ks.yml new file mode 100644 index 00000000..bcbef0be --- /dev/null +++ b/flux/dev/cert-issuer/ks.yml @@ -0,0 +1,15 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: cluster-issuer + namespace: flux-system +spec: + interval: 2m0s + path: ./flux/dev/cert-issuer/app + prune: true + sourceRef: + kind: GitRepository + name: spectrum + namespace: flux-system + dependsOn: + - name: cert-manager diff --git a/flux/dev/cert-issuer/kustomization.yml b/flux/dev/cert-issuer/kustomization.yml new file mode 100644 index 00000000..1f8ae66b --- /dev/null +++ b/flux/dev/cert-issuer/kustomization.yml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ks.yml diff --git a/flux/dev/cert-manager/app/helm-release.yml b/flux/dev/cert-manager/app/helm-release.yml new file mode 100644 index 00000000..8bb6edad --- /dev/null +++ b/flux/dev/cert-manager/app/helm-release.yml @@ -0,0 +1,18 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: cert-manager + namespace: kube-system +spec: + interval: 30m + chart: + spec: + chart: cert-manager + version: "1.x" + sourceRef: + kind: HelmRepository + name: cert-manager + namespace: flux-system + interval: 12h + values: + installCRDs: true diff --git a/flux/dev/cert-manager/app/helm-repository.yml b/flux/dev/cert-manager/app/helm-repository.yml new file mode 100644 index 00000000..2629f361 --- /dev/null +++ b/flux/dev/cert-manager/app/helm-repository.yml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: cert-manager + namespace: flux-system +spec: + interval: 24h + url: https://charts.jetstack.io diff --git a/flux/dev/cert-manager/app/kustomization.yml b/flux/dev/cert-manager/app/kustomization.yml new file mode 100644 index 00000000..c4907f6a --- /dev/null +++ b/flux/dev/cert-manager/app/kustomization.yml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - helm-repository.yml + - helm-release.yml diff --git a/flux/dev/cert-manager/ks.yml b/flux/dev/cert-manager/ks.yml new file mode 100644 index 00000000..e6e8ae07 --- /dev/null +++ b/flux/dev/cert-manager/ks.yml @@ -0,0 +1,13 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: cert-manager + namespace: flux-system +spec: + interval: 1m0s + path: ./flux/dev/cert-manager/app + prune: true + sourceRef: + kind: GitRepository + name: spectrum + namespace: flux-system diff --git a/flux/dev/cert-manager/kustomization.yml b/flux/dev/cert-manager/kustomization.yml new file mode 100644 index 00000000..1f8ae66b --- /dev/null +++ b/flux/dev/cert-manager/kustomization.yml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ks.yml diff --git a/flux/dev/external-dns/helm-release.yml b/flux/dev/external-dns/helm-release.yml new file mode 100644 index 00000000..d075a753 --- /dev/null +++ b/flux/dev/external-dns/helm-release.yml @@ -0,0 +1,28 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: external-dns + namespace: kube-system +spec: + interval: 5m + chart: + spec: + chart: external-dns + version: 1.15.0 + sourceRef: + kind: HelmRepository + name: external-dns-charts + namespace: flux-system + interval: 5m + values: + env: + - name: DO_TOKEN + valueFrom: + secretKeyRef: + name: digitalocean-token + key: token + txtOwnerId: "${PREFIX}" + sources: + - ingress + policy: sync + provider: digitalocean diff --git a/flux/dev/external-dns/helm-repository.yml b/flux/dev/external-dns/helm-repository.yml new file mode 100644 index 00000000..0bebbe3b --- /dev/null +++ b/flux/dev/external-dns/helm-repository.yml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: external-dns-charts + namespace: flux-system +spec: + interval: 1h + url: https://kubernetes-sigs.github.io/external-dns diff --git a/flux/dev/external-dns/kustomization.yml b/flux/dev/external-dns/kustomization.yml new file mode 100644 index 00000000..c1f399d2 --- /dev/null +++ b/flux/dev/external-dns/kustomization.yml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - helm-release.yml + - helm-repository.yml diff --git a/flux/dev/ingress-nginx/helm-release.yml b/flux/dev/ingress-nginx/helm-release.yml index 41ceb502..c7872899 100644 --- a/flux/dev/ingress-nginx/helm-release.yml +++ b/flux/dev/ingress-nginx/helm-release.yml @@ -1,14 +1,8 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx ---- apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: name: ingress-nginx - namespace: ingress-nginx + namespace: kube-system spec: interval: 30m chart: @@ -18,14 +12,31 @@ spec: sourceRef: kind: HelmRepository name: ingress-nginx - namespace: ingress-nginx + namespace: flux-system interval: 12h values: controller: kind: DaemonSet + publishService: + enabled: true service: type: LoadBalancer + externalIPs: + - ${LOADBALANCER_IP} nodePorts: http: 30100 https: 30101 externalTrafficPolicy: Local + ingress: + enabled: true + hosts: + - kube.${DOMAIN} + ingressClassName: nginx + useDefaultAnnotations: true + pathType: ImplementationSpecific + path: / + issuer: + name: letsencrypt + scope: cluster + tls: + enabled: true diff --git a/flux/dev/ingress-nginx/helm-repository.yml b/flux/dev/ingress-nginx/helm-repository.yml index 88131c78..eb02c143 100644 --- a/flux/dev/ingress-nginx/helm-repository.yml +++ b/flux/dev/ingress-nginx/helm-repository.yml @@ -2,7 +2,7 @@ apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: ingress-nginx - namespace: ingress-nginx + namespace: flux-system spec: interval: 24h url: https://kubernetes.github.io/ingress-nginx diff --git a/flux/dev/kubernetes-dashboard/helm-release.yml b/flux/dev/kubernetes-dashboard/helm-release.yml index 32dc9bb7..1c82cfb3 100644 --- a/flux/dev/kubernetes-dashboard/helm-release.yml +++ b/flux/dev/kubernetes-dashboard/helm-release.yml @@ -1,35 +1,11 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: kubernetes-dashboard ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubernetes-dashboard-admin - namespace: kubernetes-dashboard ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubernetes-dashboard-admin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: kubernetes-dashboard-admin - namespace: kubernetes-dashboard ---- apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: name: kubernetes-dashboard - namespace: kubernetes-dashboard + namespace: kube-system spec: releaseName: kubernetes-dashboard + interval: 5m chart: spec: chart: kubernetes-dashboard @@ -38,10 +14,20 @@ spec: kind: HelmRepository name: kubernetes-dashboard namespace: flux-system - interval: 5m values: serviceAccount: create: true rbac: create: true replicas: 1 + app: + ingress: + enabled: true + hosts: + - dashboard.${DOMAIN} + ingressClassName: nginx + issuer: + name: letsencrypt + scope: cluster + tls: + enabled: true diff --git a/flux/dev/kubernetes-dashboard/kustomization.yml b/flux/dev/kubernetes-dashboard/kustomization.yml index c4907f6a..bb4d2b56 100644 --- a/flux/dev/kubernetes-dashboard/kustomization.yml +++ b/flux/dev/kubernetes-dashboard/kustomization.yml @@ -3,3 +3,4 @@ kind: Kustomization resources: - helm-repository.yml - helm-release.yml + - serviceaccount.yml diff --git a/flux/dev/kubernetes-dashboard/serviceaccount.yml b/flux/dev/kubernetes-dashboard/serviceaccount.yml new file mode 100644 index 00000000..f619b747 --- /dev/null +++ b/flux/dev/kubernetes-dashboard/serviceaccount.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubernetes-dashboard-admin + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard-admin + namespace: kube-system diff --git a/flux/dev/kubevirt-manager/kustomization.yml b/flux/dev/kubevirt-manager/kustomization.yml deleted file mode 100644 index cf5583e5..00000000 --- a/flux/dev/kubevirt-manager/kustomization.yml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -namespace: kubevirt-manager -resources: - - https://raw.githubusercontent.com/kubevirt-manager/kubevirt-manager/main/kubernetes/bundled.yaml diff --git a/flux/dev/kustomization.yml b/flux/dev/kustomization.yml index b9c5b84e..e1c8c7b1 100644 --- a/flux/dev/kustomization.yml +++ b/flux/dev/kustomization.yml @@ -1,7 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - external-dns - ingress-nginx + - cert-manager + - cert-issuer - kubernetes-dashboard - kubevirt - - kubevirt-manager diff --git a/terraform/backend.tf b/terraform/backend.tf index 21fbfea0..74e5dd31 100644 --- a/terraform/backend.tf +++ b/terraform/backend.tf @@ -24,6 +24,6 @@ data "cloudflare_zone" "fluence_dev" { name = "fluence.dev" } -data "cloudflare_accounts" "fluence" { - name = "fluence" +provider "vault" { + address = "https://vault.fluence.dev" } diff --git a/terraform/controlplane.tf b/terraform/controlplane.tf index f75e9f39..a26f3067 100644 --- a/terraform/controlplane.tf +++ b/terraform/controlplane.tf @@ -1,6 +1,6 @@ locals { cp = [ - for i in range(1) : format("%s-%d", "cp", i) + for i in range(3) : format("%s-%d", "cp", i) ] } @@ -46,11 +46,12 @@ resource "digitalocean_loadbalancer" "cp" { droplet_tag = "${local.prefix}-controlplane" } -resource "cloudflare_record" "endpoint" { - zone_id = data.cloudflare_zone.fluence_dev.zone_id - name = "kube.${local.prefix}.fluence.dev" - content = digitalocean_loadbalancer.cp.ip - type = "A" +resource "digitalocean_record" "endpoint" { + domain = digitalocean_domain.spectrum.id + type = "A" + name = "kube" + value = digitalocean_loadbalancer.cp.ip + ttl = 30 } resource "talos_machine_secrets" "this" { @@ -67,10 +68,14 @@ data "talos_machine_configuration" "cp" { talos_version = "v1.8" config_patches = [ templatefile("${path.module}/templates/controlplane_patch.yml", { - loadbalancerdns = "${local.loadbalancer_dns}", + loadbalancerdns = "kube.${local.prefix}.fluence.dev" + loadbalancerip = digitalocean_loadbalancer.cp.ip hostdns = "${each.key}.${local.prefix}.fluence.dev", subnet = data.digitalocean_vpc.spectrum.ip_range, branch = var.github_branch + dotoken = base64encode(data.vault_generic_secret.spectrum.data.token) + domain = "${local.prefix}.fluence.dev" + prefix = local.prefix }) ] } @@ -79,7 +84,7 @@ data "talos_client_configuration" "this" { cluster_name = terraform.workspace client_configuration = talos_machine_secrets.this.client_configuration endpoints = [ - local.loadbalancer_dns, + "kube.${local.prefix}.fluence.dev" ] } @@ -112,7 +117,7 @@ resource "cloudflare_record" "cp" { type = "A" } -resource "talos_machine_bootstrap" "bootstrap" { +resource "talos_machine_bootstrap" "this" { client_configuration = talos_machine_secrets.this.client_configuration endpoint = digitalocean_droplet.cp["cp-0"].ipv4_address node = digitalocean_droplet.cp["cp-0"].ipv4_address @@ -134,3 +139,19 @@ resource "talos_cluster_kubeconfig" "this" { # control_plane_nodes = [for droplet in digitalocean_droplet.cp : droplet.ipv4_address] # endpoints = data.talos_client_configuration.this.endpoints #} + +#data "http" "talos_health" { +# for_each = toset([ +# "cp-0.${local.prefix}.fluence.dev", +# "cp-1.${local.prefix}.fluence.dev", +# "cp-2.${local.prefix}.fluence.dev", +# ]) +# url = "https://${each.key}:6443/version" +# insecure = true +# retry { +# attempts = 60 +# min_delay_ms = 5000 +# max_delay_ms = 5000 +# } +# depends_on = [talos_machine_bootstrap.this] +#} diff --git a/terraform/dns.tf b/terraform/dns.tf new file mode 100644 index 00000000..452bc38f --- /dev/null +++ b/terraform/dns.tf @@ -0,0 +1,16 @@ +resource "digitalocean_domain" "spectrum" { + name = "${local.prefix}.fluence.dev" +} + +resource "cloudflare_record" "ns" { + for_each = toset([ + "ns1.digitalocean.com", + "ns2.digitalocean.com", + "ns3.digitalocean.com", + ]) + + zone_id = data.cloudflare_zone.fluence_dev.zone_id + name = "${local.prefix}.fluence.dev" + content = each.key + type = "NS" +} diff --git a/terraform/main.tf b/terraform/main.tf index cf870ed7..a68a7133 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -15,3 +15,7 @@ resource "digitalocean_ssh_key" "spectrum" { data "digitalocean_image" "talos" { name = "talos-v1.8.2" } + +data "vault_generic_secret" "spectrum" { + path = "kv/digitalocean/spectrum" +} diff --git a/terraform/templates/controlplane_patch.yml b/terraform/templates/controlplane_patch.yml index 13f65e89..aa094b39 100644 --- a/terraform/templates/controlplane_patch.yml +++ b/terraform/templates/controlplane_patch.yml @@ -128,7 +128,29 @@ cluster: kind: GitRepository metadata: name: spectrum - namespace: default + namespace: flux-system + spec: + interval: 1m0s + url: https://github.com/fluencelabs/spectrum.git + ref: + branch: ${branch} + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: terraform-config + namespace: flux-system + data: + DOMAIN: ${domain} + LOADBALANCER_IP: ${loadbalancerip} + BRANCH: ${branch} + PREFIX: ${prefix} + --- + apiVersion: source.toolkit.fluxcd.io/v1 + kind: GitRepository + metadata: + name: spectrum + namespace: flux-system spec: interval: 1m0s url: https://github.com/fluencelabs/spectrum.git @@ -139,7 +161,7 @@ cluster: kind: Kustomization metadata: name: deploy-spectrum - namespace: default + namespace: flux-system spec: interval: 1m0s path: "./flux/dev" @@ -147,6 +169,19 @@ cluster: sourceRef: kind: GitRepository name: spectrum - namespace: default + namespace: flux-system validation: client timeout: 2m + postBuild: + substituteFrom: + - kind: ConfigMap + name: terraform-config + --- + apiVersion: v1 + kind: Secret + metadata: + name: digitalocean-token + namespace: kube-system + type: Opaque + data: + token: ${dotoken} diff --git a/terraform/test.yml b/terraform/test.yml deleted file mode 100644 index 38078df6..00000000 --- a/terraform/test.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kuard -spec: - selector: - matchLabels: - app: kuard - replicas: 1 - template: - metadata: - labels: - app: kuard - spec: - containers: - - image: gcr.io/kuar-demo/kuard-amd64:1 - imagePullPolicy: Always - name: kuard - ports: - - containerPort: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: kuard -spec: - ports: - - port: 80 - targetPort: 8080 - protocol: TCP - selector: - app: kuard ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: kuard - annotations: - cert-manager.io/cluster-issuer: "letsencrypt-staging" -spec: - ingressClassName: nginx - tls: - - hosts: - - test.rnd-setup-ingress.fluence.dev - secretName: quickstart-example-tls - rules: - - host: test.rnd-setup-ingress.fluence.dev - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: kuard - port: - number: 80