ci: build and deploy from runner (TT-1536) (#24)

* Update action versions, get secrets from vault

* Add k8s deployment file and pipeline job

* Change sed separator

* Revert "Change sed separator"

This reverts commit 75accac.

* Add deploy-to-prod job

* Remove 'v' from image version

* Use same k8s file template for stage and prod

* Deploy to stage from main only

.github/workflows/pipeline.yml

@@ -2,16 +2,19 @@ name: CI/CD pipeline
 
 on: [push]
 
+env:
+  APP_VERSION: ${{ github.ref_name }}
+
 jobs:
 
   lint-and-test:
     name: Run linting, typecheck, and tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
@@ -32,59 +35,27 @@ jobs:
       - name: Running tests
         run: python -m pytest --cov=metadata_extract
 
-  build:
-    name: Create Docker image
-    needs: lint-and-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: nationallibraryofnorway/meteor
-          tags: |
-            type=semver,pattern={{version}}
-            type=ref,event=branch
-            type=ref,event=pr
-
-      - name: Build image
-        uses: docker/build-push-action@v4
-        with:
-          push: false
-          context: .
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          file: Dockerfile
-          build-args: |
-            USE_GIELLA=true
-
-  publish:
+  build-and-publish:
     name: Create and push Docker image
     needs: lint-and-test
     if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: nationallibraryofnorway/meteor
           tags: |
@@ -93,7 +64,7 @@ jobs:
             type=ref,event=pr
 
       - name: Build image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           context: .
@@ -104,10 +75,101 @@ jobs:
             USE_GIELLA=true
 
       - name: Docker Hub Description
-        uses: peter-evans/dockerhub-description@v3
+        uses: peter-evans/dockerhub-description@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
           repository: nationallibraryofnorway/meteor
           short-description: ${{ github.event.repository.description }}
           readme-filepath: ./README.md
+
+  deploy-to-stage:
+    name: Deploy to kubernetes stage environment
+    needs: build-and-publish
+    runs-on: [self-hosted, Linux]
+    environment: stage
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Import secrets
+        id: import-secrets
+        uses: hashicorp/vault-action@v3
+        with:
+          url: ${{ secrets.VAULT_URL }}
+          method: approle
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          secrets: |
+            kv/team/text/data/harvestk8s-text-stage *
+
+      - name: Setup Kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: 'v1.26.5'
+
+      - name: Deploy to k8s
+        run: |
+          echo "Deploy version ${{ env.APP_VERSION }}" to stage
+          kubectl config set-cluster k8s --server="${{ steps.import-secrets.outputs.K8S_STAGE_SERVER }}"
+          kubectl config set clusters.k8s.certificate-authority-data ${{ steps.import-secrets.outputs.K8S_STAGE_NB_NO_CA }}
+          kubectl config set-credentials ${{ steps.import-secrets.outputs.K8S_STAGE_USER }} --token=${{ steps.import-secrets.outputs.K8S_STAGE_NB_NO_TOKEN }}
+          kubectl config set-context meteor --cluster=k8s --user=${{ steps.import-secrets.outputs.K8S_STAGE_USER }} --namespace=tekst-stage
+          kubectl config use-context meteor
+          kubectl config view
+          kubectl version
+          sed -i "s/<pvc_claim_name>/dimo-fileserver-pvc/g" k8s/meteor.yml
+          sed -i "s/<version>/${{ env.APP_VERSION }}/g" k8s/meteor.yml
+          sed -i "s/<host_url>/${{ steps.import-secrets.outputs.K8S_HOST_URL }}/g" k8s/meteor.yml
+          kubectl apply -f k8s/meteor.yml
+          kubectl rollout restart deploy/meteor
+
+  deploy-to-prod:
+    name: Deploy to kubernetes prod environment
+    needs: build-and-publish
+    runs-on: [self-hosted, Linux]
+    if: startsWith(github.event.ref, 'refs/tags/v')
+    environment: prod
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Import secrets
+        id: import-secrets
+        uses: hashicorp/vault-action@v3
+        with:
+          url: ${{ secrets.VAULT_URL }}
+          method: approle
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          secrets: |
+            kv/team/text/data/harvestk8s-text-prod *
+
+      - name: Setup Kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: 'v1.26.5'
+
+      - name: Set Version
+        uses: actions/github-script@v7
+        id: set_version
+        with:
+          script: |
+            const no_v = context.ref.replace('refs/tags/v', '')
+            core.setOutput('img_version', no_v)
+
+      - name: Deploy to k8s
+        run: |
+          echo "Deploy version ${{ env.APP_VERSION }}" to prod
+          kubectl config set-cluster k8s --server="${{ steps.import-secrets.outputs.K8S_SERVER }}"
+          kubectl config set clusters.k8s.certificate-authority-data ${{ steps.import-secrets.outputs.K8S_NB_NO_CA }}
+          kubectl config set-credentials ${{ steps.import-secrets.outputs.K8S_USER }} --token=${{ steps.import-secrets.outputs.K8S_NB_NO_TOKEN }}
+          kubectl config set-context meteor --cluster=k8s --user=${{ steps.import-secrets.outputs.K8S_USER }} --namespace=tekst-prod
+          kubectl config use-context meteor
+          kubectl config view
+          kubectl version
+          sed -i "s/<pvc_claim_name>/dimo-fileserver-expandable-pvc/g" k8s/meteor.yml
+          sed -i "s/<version>/${{ steps.set_version.outputs.img_version }}/g" k8s/meteor.yml
+          sed -i "s/<host_url>/${{ steps.import-secrets.outputs.K8S_HOST_URL }}/g" k8s/meteor.yml
+          kubectl apply -f k8s/meteor.yml
+          kubectl rollout restart deploy/meteor

k8s/meteor.yml

@@ -0,0 +1,94 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: meteor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: meteor
+  template:
+    metadata:
+      labels:
+        app: meteor
+    spec:
+      volumes:
+        - name: dimo-file-server-volume
+          persistentVolumeClaim:
+            claimName: <pvc_claim_name>
+      containers:
+        - name: app
+          image: nationallibraryofnorway/meteor:<version>
+          ports:
+            - containerPort: 8000
+          env:
+            - name: REGISTRY_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: meteor-registry-secret
+                  key: host
+            - name: REGISTRY_DATABASE
+              valueFrom:
+                secretKeyRef:
+                  name: meteor-registry-secret
+                  key: database
+            - name: REGISTRY_USER
+              valueFrom:
+                secretKeyRef:
+                  name: meteor-registry-secret
+                  key: username
+            - name: REGISTRY_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: meteor-registry-secret
+                  key: password
+            - name: MOUNT_FOLDER
+              value: "/dimo-file-server"
+            - name: MAX_FILE_SIZE_MB
+              value: "10000"
+            - name: ENVIRONMENT
+              value: "stage"
+            - name: LANGUAGES
+              value: "mul,eng,nob,nno"
+            - name: USE_GIELLADETECT
+              value: "True"
+            - name: GIELLADETECT_LANGS
+              value: "nno,nob,eng,sme,sma,smj"
+          volumeMounts:
+            - name: dimo-file-server-volume
+              mountPath: /dimo-file-server
+          imagePullPolicy: Always
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: meteor-service
+spec:
+  ports:
+    - port: 8000
+      name: http
+      targetPort: 8000
+  selector:
+    app: meteor
+  type: ClusterIP
+
+---
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: meteor-ingress
+spec:
+  rules:
+    - host: <host_url>
+      http:
+        paths:
+          - path: /meteor
+            pathType: Prefix
+            backend:
+              service:
+                name: meteor-service
+                port:
+                  number: 8000