Merge pull request #814 from makerdao/docker

Use Docker and deploy to Staging K8s

.github/workflows/angular-dev.yml

@@ -1,52 +1,86 @@
-# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
-
-name: Build DEV Angular
+name: Build DEV Angular - Frontend
 
 on: 
   push:
-    branches: [ develop ]
+    branches: [ docker ]
     paths:
       - "frontend/**"
-  # pull_request:   
-  #   branches: [ develop ]    
-  #   paths:
-  #     - "frontend/**"
-  workflow_dispatch:
+      - ".github/workflows/angular-dev.yml"
 
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [12.x]
+    env:
+      REGION: us-east-2
+      CLUSTER_NAME: maker-staging
+      SERVICE_NAME: mips-frontend
+      AWS_ECR_NAME: mips-frontend-staging
+      ENVIRONMENT_TAG: staging
+      HELM_FILE: helm/staging/frontend.yaml
 
     steps:
-      - uses: actions/checkout@v1
-
-      - name: Cache node modules
-        uses: actions/cache@v3
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
-      - name: Node ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: npm install and npm run build
-        run: |
-          npm i
-          npm run build --prod
-        working-directory: frontend
-      - name: Copy file via scp
-        uses: appleboy/[email protected]
-        env:
-          HOST: ${{ secrets.SSH_HOST }}
-          USERNAME: ${{ secrets.SSH_USER }}
-          PORT: ${{ secrets.SSH_PORT }}
-          PASSWORD: ${{ secrets.SSH_PASS }}
-        with:
-          source: "./frontend/dist"
-          target: "/var/warehouse/makerdao-mips-develop"
+    - uses: actions/checkout@v3
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ env.REGION }}
+
+    - name: Login to AWS ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+
+    - name: Extract commit hash
+      id: vars
+      if: ${{ !contains(github.event.head_commit.message , '[skip build]') }}
+      shell: bash
+      run: |
+        echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+    - name: Build, tag, and push image to ECR
+      id: build-image
+      if: ${{ !contains(github.event.head_commit.message , '[skip build]') }}
+      working-directory: frontend
+      env:
+        SHA_TAG: ${{ steps.vars.outputs.sha_short }}
+        LATEST_TAG: latest
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+      run: |
+        # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }}
+        docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true
+        docker build -t $AWS_ECR_NAME \
+                     -t $ECR_REGISTRY/$AWS_ECR_NAME:$SHA_TAG \
+                     -t $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG \
+                     -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \
+                     -f Dockerfile \
+                     .
+
+        docker push $ECR_REGISTRY/$AWS_ECR_NAME --all-tags
+
+    - name: Replace variables in the Helm values file
+      id: replace-vars
+      if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }}
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        # USERS_LIST: ${{ secrets.STAGING_USERS_LIST }}
+      run: |
+        sed -i 's/${ECR_REGISTRY}/'$ECR_REGISTRY/ $HELM_FILE
+        sed -i 's/${USERS_LIST}/'$USERS_LIST/ $HELM_FILE
+
+    - name: Deploying Service to Kubernetes with Helm
+      id: deploy
+      if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }}
+      uses: bitovi/[email protected]
+      with:
+        values: image.repository=${{ steps.login-ecr.outputs.registry }}/${{ env.AWS_ECR_NAME }},image.tag=${{ steps.vars.outputs.sha_short }}
+        cluster-name: ${{ env.CLUSTER_NAME }}
+        config-files: ${{ env.HELM_FILE }}
+        chart-path: techops-services/common
+        namespace: mips
+        timeout: 5m0s
+        name: ${{ env.SERVICE_NAME }}
+        chart-repository: https://techops-services.github.io/helm-charts
+        version: 0.0.31
+        atomic: true

.github/workflows/node.js-dev.yml

@@ -1,76 +1,104 @@
-# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
-
-name: Node.js develop CI
+name: Node.js develop CI - Backend
 
 on:
   push:
-    branches: [ develop ] 
+    branches: [ docker ] 
     paths:
         - "backend/**"
         - ".github/workflows/node.js-dev.yml"
 
 jobs:
-  build:
+  build-deploy:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [14.x]
-        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+    env:
+      REGION: us-east-2
+      CLUSTER_NAME: maker-staging
+      SERVICE_NAME: mips-backend
+      AWS_ECR_NAME: mips-backend-staging
+      ENVIRONMENT_TAG: staging
+      HELM_FILE: helm/staging/backend.yaml
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
-      with: 
-        node-version: ${{ matrix.node-version }}
+    - name: Checkout
+      uses: actions/checkout@v3
+
     - name: Running the Test Suit
       env:
         REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }}
+        GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }}
         MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }}
         MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }}
-        GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }}
       run: |
         cp '.env example' .env
         npm install
         npm run pre-start-develop
         npm test
-      working-directory: backend  
+      working-directory: backend
 
-  deploy:
-    needs: [build]
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Copy file via scp
-      uses: appleboy/scp-action@master
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
       with:
-        host: ${{ secrets.SSH_HOST }}
-        username: ${{ secrets.SSH_USER }}
-        port: ${{ secrets.SSH_PORT }}
-        password: ${{ secrets.SSH_PASS }}
-        source: "./backend"
-        target: "/var/warehouse/makerdao-mips-develop"
+        aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ env.REGION }}
+
+    - name: Login to AWS ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+
+    - name: Extract commit hash
+      id: vars
+      if: ${{ !contains(github.event.head_commit.message , '[skip build]') }}
+      shell: bash
+      run: |
+        echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
 
-    - name: Executing remote command
-      uses: appleboy/ssh-action@master
+    - name: Build, tag, and push image to ECR
+      id: build-image
+      if: ${{ !contains(github.event.head_commit.message , '[skip build]') }}
+      working-directory: backend
       env:
+        SHA_TAG: ${{ steps.vars.outputs.sha_short }}
+        LATEST_TAG: latest
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
         REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }}
+        GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }}
         MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }}
         MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }}
-        GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }}
+      run: |
+        # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }}
+        docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true
+        docker build -t $AWS_ECR_NAME \
+                     -t $ECR_REGISTRY/$AWS_ECR_NAME:$SHA_TAG \
+                     -t $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG \
+                     -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \
+                     --build-arg REQUEST_GITHUB_URL_API_ENDPOINT=$REQUEST_GITHUB_URL_API_ENDPOINT \
+                     --build-arg GIT_ACCESS_API_TOKEN=$GIT_ACCESS_API_TOKEN \
+                     -f Dockerfile \
+                     .
+
+        docker push $ECR_REGISTRY/$AWS_ECR_NAME --all-tags
+
+    - name: Replace variables in the Helm values file
+      id: replace-vars
+      if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }}
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+      run: |
+        sed -i 's/${ECR_REGISTRY}/'$ECR_REGISTRY/ $HELM_FILE
+
+    - name: Deploying Service to Kubernetes with Helm
+      id: deploy
+      if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }}
+      uses: bitovi/[email protected]
       with:
-        host: ${{ secrets.SSH_HOST }}
-        username: ${{ secrets.SSH_USER }}
-        port: ${{ secrets.SSH_PORT }}
-        password: ${{ secrets.SSH_PASS }}
-        script: | 
-          cd /var/warehouse/makerdao-mips-develop/backend
-          pm2 stop dist/main.js
-          npm install
-          npm run build
-          npm run pre-start-develop
-          npx nestjs-command drop:db
-          npx nestjs-command parse:mips
-          pm2 start dist/main.js --name "api-dev"
+        values: image.repository=${{ steps.login-ecr.outputs.registry }}/${{ env.AWS_ECR_NAME }},image.tag=${{ steps.vars.outputs.sha_short }}
+        cluster-name: ${{ env.CLUSTER_NAME }}
+        config-files: ${{ env.HELM_FILE }}
+        chart-path: techops-services/common
+        namespace: mips
+        timeout: 5m0s
+        name: ${{ env.SERVICE_NAME }}
+        chart-repository: https://techops-services.github.io/helm-charts
+        version: 0.0.31
+        atomic: true

backend/Dockerfile

@@ -0,0 +1,39 @@
+FROM node:18.8.0-alpine3.16 as appbuild
+WORKDIR /usr/src/backend
+COPY ./package*.json ./
+RUN npm install
+COPY . .
+RUN npm run build
+
+FROM mongo:4.4.8-focal
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG REQUEST_GITHUB_URL_API_ENDPOINT
+ARG GIT_ACCESS_API_TOKEN
+ENV MONGODB_URI=mongodb://127.0.0.1:27017/
+
+# Install dependencies
+RUN apt-get update && apt-get install -y \
+    curl gnupg software-properties-common git
+
+# Install Node.js
+RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
+    apt-get install -y nodejs
+
+WORKDIR /usr/src/backend
+COPY . .
+COPY --from=appbuild /usr/src/backend/dist ./dist
+COPY --from=appbuild /usr/src/backend/package*.json ./
+RUN npm install
+
+EXPOSE 3000
+
+# parse the mips entries
+RUN mongod --fork --logpath=/dev/null && \
+    npm run pre-start && \
+    npx nestjs-command drop:db && \
+    npx nestjs-command parse:mips
+
+# start mongo and the app
+CMD mongod --fork --logpath=/dev/null && \
+    npm run start:prod

docker-compose.yml

@@ -0,0 +1,35 @@
+version: "3.3"
+
+networks:
+  default:
+
+services:
+  backend:
+    build: 
+      context: backend
+      dockerfile: Dockerfile
+    platform: linux/amd64
+    ports:
+      - 3000:3000
+    environment:
+      - MONGODB_URI=${MONGODB_URI}
+      - FOLDER_REPOSITORY_NAME=${FOLDER_REPOSITORY_NAME}
+      - FOLDER_PATTERN=${FOLDER_PATTERN}
+      - WEBHOOKS_SECRET_TOKEN=${WEBHOOKS_SECRET_TOKEN}
+      - GIT_ACCESS_API_TOKEN=${GIT_ACCESS_API_TOKEN}
+      - GITHUB_REPOSITORY_ID=${GITHUB_REPOSITORY_ID}
+      - MIP_GITHUB_REPOSITORY=${MIP_GITHUB_REPOSITORY}
+      - MIP_GITHUB_REPOSITORY_OWNER=${MIP_GITHUB_REPOSITORY_OWNER}
+      - REPO_BRANCH=${REPO_BRANCH}
+      - REPO_PATH=${REPO_PATH}
+      - GITHUB_LINKS=${GITHUB_LINKS}
+    restart: no
+
+  frontend:
+    depends_on:
+      - backend
+    build:
+      context: frontend
+      dockerfile: Dockerfile
+    ports:
+      - 8080:80

frontend/Dockerfile

@@ -0,0 +1,19 @@
+FROM node:14.16.1-alpine3.13 as builder
+
+WORKDIR /usr/src/app
+
+COPY . .
+
+RUN npm install
+
+# prod
+RUN npm run build-prod
+# dev
+# RUN npm run build
+
+FROM nginx:alpine
+
+COPY --from=builder /usr/src/app/dist/frontend /usr/share/nginx/html
+COPY --from=builder /usr/src/app/nginx.conf  /etc/nginx/conf.d/default.conf
+
+EXPOSE 80