From f1613b7ccda2e9e59d36bf620c6ef4ab9a8a0958 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Tue, 16 Jan 2024 17:49:28 +1100 Subject: [PATCH 01/32] localhost --- backend/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 0410881c..bda9bb37 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -9,7 +9,7 @@ services: volumes: - mongo:/var/lib/mongo ports: - - 27018:27017 + - localhost:27018:27017 environment: MONGO_INITDB_ROOT_USERNAME: admin MONGO_INITDB_ROOT_PASSWORD: password From 5643fae958e8507c8a1a82d5ad8e5e7b3fa276a8 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 12:27:40 +1100 Subject: [PATCH 02/32] Add docker --- backend/Dockerfile | 36 +++++++++++++++++ backend/docker-compose.yml | 16 -------- docker-compose.yml | 39 +++++++++++++++++++ frontend/Dockerfile | 14 +++++++ frontend/nginx.conf | 20 ++++++++++ frontend/src/environments/environment.prod.ts | 3 +- 6 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 backend/Dockerfile delete mode 100644 backend/docker-compose.yml create mode 100644 docker-compose.yml create mode 100644 frontend/Dockerfile create mode 100644 frontend/nginx.conf diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..37f20133 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,36 @@ +FROM node:18.8.0-alpine3.16 as appbuild +WORKDIR /usr/src/backend +COPY ./package*.json ./ +RUN npm install +COPY . . +RUN npm run build + +FROM mongo:7.0 + +ARG DEBIAN_FRONTEND=noninteractive + +# Install dependencies +RUN apt-get update && apt-get install -y \ + curl gnupg software-properties-common git + +# Install Node.js +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ + apt-get install -y nodejs + +WORKDIR /usr/src/backend +COPY . . +COPY --from=appbuild /usr/src/backend/dist ./dist +COPY --from=appbuild /usr/src/backend/package*.json ./ +RUN npm install + +EXPOSE 3000 + +# parse the mips entries +RUN mongod --fork --logpath=/dev/null && \ + npm run pre-start && \ + npx nestjs-command drop:db && \ + npx nestjs-command parse:mips + +# start mongo and the app +CMD mongod --fork --logpath=/dev/null && \ + npm run start:prod diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml deleted file mode 100644 index bda9bb37..00000000 --- a/backend/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.3" - -volumes: - mongo: - -services: - mongodb: - image: mongo:4.4.4-bionic - volumes: - - mongo:/var/lib/mongo - ports: - - localhost:27018:27017 - environment: - MONGO_INITDB_ROOT_USERNAME: admin - MONGO_INITDB_ROOT_PASSWORD: password - restart: always diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..77a8334d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +version: "3.3" + +volumes: + mongo: + +networks: + default: + +services: + backend: + build: + context: backend + dockerfile: Dockerfile + ports: + - 3000:3000 + environment: + - MONGODB_URI=${MONGODB_URI} + - FOLDER_REPOSITORY_NAME=${FOLDER_REPOSITORY_NAME} + - FOLDER_PATTERN=${FOLDER_PATTERN} + - WEBHOOKS_SECRET_TOKEN=${WEBHOOKS_SECRET_TOKEN} + - GIT_ACCESS_API_TOKEN=${GIT_ACCESS_API_TOKEN} + - GITHUB_REPOSITORY_ID=${GITHUB_REPOSITORY_ID} + - MIP_GITHUB_REPOSITORY=${MIP_GITHUB_REPOSITORY} + - MIP_GITHUB_REPOSITORY_OWNER=${MIP_GITHUB_REPOSITORY_OWNER} + - REPO_BRANCH=${REPO_BRANCH} + - REPO_PATH=${REPO_PATH} + - GITHUB_LINKS=${GITHUB_LINKS} + restart: no + volumes: + - mongo:/data/db + + frontend: + depends_on: + - backend + build: + context: frontend + dockerfile: Dockerfile + ports: + - 8080:80 diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..8051460e --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,14 @@ +FROM node:14.16.1-alpine3.13 as builder + +WORKDIR /usr/src/app + +COPY . . +RUN npm install +RUN npm run build-prod + +FROM nginx:alpine + +COPY --from=builder /usr/src/app/dist/frontend /usr/share/nginx/html +COPY --from=builder /usr/src/app/nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 00000000..b3be85a2 --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,20 @@ +server { + listen 80; + sendfile on; + default_type application/octet-stream; + + gzip on; + gzip_http_version 1.1; + gzip_disable "MSIE [1-6]\."; + gzip_min_length 256; + gzip_vary on; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_comp_level 9; + + root /usr/share/nginx/html; + + location / { + try_files $uri $uri/ /index.html =404; + } +} diff --git a/frontend/src/environments/environment.prod.ts b/frontend/src/environments/environment.prod.ts index 08e54bb4..10f06025 100644 --- a/frontend/src/environments/environment.prod.ts +++ b/frontend/src/environments/environment.prod.ts @@ -1,6 +1,7 @@ export const environment = { production: true, - apiUrl: 'https://mips-api.makerdao.com', + // apiUrl: 'https://mips-api.makerdao.com', + apiUrl: 'http://backend:3000', repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xvodvpze', githubURL: 'https://github.com/', From 1bdad38edaf6a3ae3b061f3f2dd9a3bc13333fa9 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 12:47:41 +1100 Subject: [PATCH 03/32] dev --- frontend/Dockerfile | 7 ++++++- frontend/src/environments/environment.fullstack.ts | 2 +- frontend/src/environments/environment.prod.ts | 3 +-- frontend/src/environments/environment.ts | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 8051460e..5f064111 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -4,7 +4,12 @@ WORKDIR /usr/src/app COPY . . RUN npm install -RUN npm run build-prod +# prod +# RUN npm run build-prod + +# dev +RUN npm run build + FROM nginx:alpine diff --git a/frontend/src/environments/environment.fullstack.ts b/frontend/src/environments/environment.fullstack.ts index e7046e86..f8feee24 100644 --- a/frontend/src/environments/environment.fullstack.ts +++ b/frontend/src/environments/environment.fullstack.ts @@ -4,7 +4,7 @@ export const environment = { production: false, - apiUrl: 'http://localhost:3001', + apiUrl: 'http://backend:3000', //apiUrl: 'http://159.203.86.45:3001', // apiUrl: "https://mips-api.makerdao.com", repoUrl: 'https://github.com/makerdao/mips/blob/master', diff --git a/frontend/src/environments/environment.prod.ts b/frontend/src/environments/environment.prod.ts index 10f06025..08e54bb4 100644 --- a/frontend/src/environments/environment.prod.ts +++ b/frontend/src/environments/environment.prod.ts @@ -1,7 +1,6 @@ export const environment = { production: true, - // apiUrl: 'https://mips-api.makerdao.com', - apiUrl: 'http://backend:3000', + apiUrl: 'https://mips-api.makerdao.com', repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xvodvpze', githubURL: 'https://github.com/', diff --git a/frontend/src/environments/environment.ts b/frontend/src/environments/environment.ts index 563f3108..fe0fc0be 100644 --- a/frontend/src/environments/environment.ts +++ b/frontend/src/environments/environment.ts @@ -4,8 +4,8 @@ export const environment = { production: false, - // apiUrl: 'http://localhost:3001', - apiUrl: 'http://159.203.86.45:3001', + apiUrl: 'http://backend:3000', + // apiUrl: 'http://159.203.86.45:3001', // apiUrl: "https://mips-api.makerdao.com", repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xzbyjjnb', From 63761c9947d165131b495b165a82bcc7334fcde5 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 12:48:00 +1100 Subject: [PATCH 04/32] prod --- frontend/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 5f064111..2de5c2d7 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -5,10 +5,10 @@ WORKDIR /usr/src/app COPY . . RUN npm install # prod -# RUN npm run build-prod +RUN npm run build-prod # dev -RUN npm run build +# RUN npm run build FROM nginx:alpine From 8763d9d69ab867f119f0b655143fb0377ab79b20 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 15:32:30 +1100 Subject: [PATCH 05/32] add backend deploy --- .github/workflows/angular-dev.yml | 2 +- .github/workflows/node.js-dev.yml | 116 ++++++++++++++++++------------ backend/helm/staging/values.yaml | 104 +++++++++++++++++++++++++++ frontend/Dockerfile | 4 +- frontend/helm/staging/values.yaml | 0 5 files changed, 178 insertions(+), 48 deletions(-) create mode 100644 backend/helm/staging/values.yaml create mode 100644 frontend/helm/staging/values.yaml diff --git a/.github/workflows/angular-dev.yml b/.github/workflows/angular-dev.yml index 6d562350..2b17de45 100644 --- a/.github/workflows/angular-dev.yml +++ b/.github/workflows/angular-dev.yml @@ -1,7 +1,7 @@ # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions -name: Build DEV Angular +name: Build DEV Angular - Frontend on: push: diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index 2c120de5..c84521b9 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -1,7 +1,7 @@ # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions -name: Node.js develop CI +name: Node.js develop CI - Backend on: push: @@ -11,19 +11,20 @@ on: - ".github/workflows/node.js-dev.yml" jobs: - build: + build-deploy: runs-on: ubuntu-latest - strategy: - matrix: - node-version: [14.x] - # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ + env: + REGION: us-east-2 + CLUSTER_NAME: maker-staging + SERVICE_NAME: mips-backend + AWS_ECR_NAME: mips-backend-staging + ENVIRONMENT_TAG: staging + HELM_FILE: backend/helm/staging/values.yaml steps: - - uses: actions/checkout@v3 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} + - name: Checkout + uses: actions/checkout@v4 + - name: Running the Test Suit env: REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }} @@ -35,42 +36,67 @@ jobs: npm install npm run pre-start-develop npm test - working-directory: backend - - deploy: - needs: [build] - runs-on: ubuntu-latest + working-directory: backend - steps: - - uses: actions/checkout@v3 - - name: Copy file via scp - uses: appleboy/scp-action@master + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - port: ${{ secrets.SSH_PORT }} - password: ${{ secrets.SSH_PASS }} - source: "./backend" - target: "/var/warehouse/makerdao-mips-develop" + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.REGION }} + + - name: Login to AWS ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 - - name: Executing remote command - uses: appleboy/ssh-action@master + - name: Extract commit hash + id: vars + if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} + shell: bash + run: | + echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + + - name: Build, tag, and push image to ECR + id: build-image + if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} env: - REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }} - MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }} - MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }} - GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} + SHA_TAG: ${{ steps.vars.outputs.sha_short }} + LATEST_TAG: latest + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }} + docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true + docker build -t $AWS_ECR_NAME \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$SHA_TAG \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \ + -f backend/Dockerfile \ + . + + docker push $ECR_REGISTRY/$AWS_ECR_NAME --all-tags + + - name: Replace variables in the Helm values file + id: replace-vars + if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # USERS_LIST: ${{ secrets.STAGING_USERS_LIST }} + run: | + sed -i 's/${ECR_REGISTRY}/'$ECR_REGISTRY/ $HELM_FILE + sed -i 's/${USERS_LIST}/'$USERS_LIST/ $HELM_FILE + + - name: Deploying Service to Kubernetes with Helm + id: deploy + if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} + uses: bitovi/github-actions-deploy-eks-helm@v1.2.8 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - port: ${{ secrets.SSH_PORT }} - password: ${{ secrets.SSH_PASS }} - script: | - cd /var/warehouse/makerdao-mips-develop/backend - pm2 stop dist/main.js - npm install - npm run build - npm run pre-start-develop - npx nestjs-command drop:db - npx nestjs-command parse:mips - pm2 start dist/main.js --name "api-dev" + values: image.repository=${{ steps.login-ecr.outputs.registry }}/${{ env.AWS_ECR_NAME }},image.tag=${{ steps.vars.outputs.sha_short }} + cluster-name: ${{ env.CLUSTER_NAME }} + config-files: ${{ env.HELM_FILE }} + chart-path: techops-services/common + namespace: ${{ env.SERVICE_NAME }} + timeout: 5m0s + name: ${{ env.SERVICE_NAME }} + chart-repository: https://techops-services.github.io/helm-charts + version: 0.0.29 + atomic: true diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml new file mode 100644 index 00000000..f3ef3207 --- /dev/null +++ b/backend/helm/staging/values.yaml @@ -0,0 +1,104 @@ +replicaCount: 1 +service: + enabled: true + name: mips-backend + port: 3000 + type: ClusterIP + containerPort: 3000 + tls: + enabled: true + issuerName: letsencrypt + +image: + repository: ${ECR_REGISTRY}/mips-backend-staging + pullPolicy: Always + tag: latest + +serviceAccount: + create: false + +ingress: + enabled: true + hosts: + - mips-api-staging.makerdao.com + annotations: + external-dns.alpha.kubernetes.io/cloudflare-proxied: "false" + +httpBasicAuth: + enabled: false + # usersList: "$${USERS_LIST}" + +# If enabled will create Traefik Middleware and apply to Ingress +# to redirect http to https and www to non-www +httpWwwRedirect: + enabled: false + +podAnnotations: + reloader.stakater.com/auto: "true" + +resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 250m + memory: 256Mi + +env: + NODE_ENV: + type: kv + value: development + FOLDER_REPOSITORY_NAME: + type: kv + value: mips_repository + REPO_PATH: + type: kv + value: https://github.com/makerdao/mips.git + REPO_BRANCH: + type: kv + value: master + FOLDER_PATTERN: + type: kv + value: 'MIP*' + REQUEST_GITHUB_URL_API_ENDPOINT: + type: kv + value: https://api.github.com/graphql + MIP_GITHUB_REPOSITORY: + type: kv + value: mips + MIP_GITHUB_REPOSITORY_OWNER: + type: kv + value: makerdao + GITHUB_LINKS: + type: kv + value: https://github.com/makerdao/mips/blob/master + MONGODB_URI: + type: parameterStore + name: mongodb-uri + parameter_name: /eks/maker-staging/mips-backend/mongodb-uri + WEBHOOKS_SECRET_TOKEN: + type: parameterStore + name: webhooks-secret-token + parameter_name: /eks/maker-staging/mips-backend/webhooks-secret-token + GIT_ACCESS_API_TOKEN: + type: parameterStore + name: git-access-api-token + parameter_name: /eks/maker-staging/mips-backend/git-access-api-token + GITHUB_REPOSITORY_ID: + type: parameterStore + name: github-repository-id + parameter_name: /eks/maker-staging/mips-backend/github-repository-id + +externalSecrets: + clusterSecretStoreName: maker-staging + +livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + tcpSocket: + port: 3000 +readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + tcpSocket: + port: 3000 diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 2de5c2d7..5f064111 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -5,10 +5,10 @@ WORKDIR /usr/src/app COPY . . RUN npm install # prod -RUN npm run build-prod +# RUN npm run build-prod # dev -# RUN npm run build +RUN npm run build FROM nginx:alpine diff --git a/frontend/helm/staging/values.yaml b/frontend/helm/staging/values.yaml new file mode 100644 index 00000000..e69de29b From c84b4d4ab441659f78ebbc983421ce277c619fce Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 15:57:16 +1100 Subject: [PATCH 06/32] secret --- .github/workflows/node.js-dev.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index c84521b9..a483614b 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -5,7 +5,7 @@ name: Node.js develop CI - Backend on: push: - branches: [ develop ] + branches: [ docker ] paths: - "backend/**" - ".github/workflows/node.js-dev.yml" @@ -41,8 +41,8 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.REGION }} - name: Login to AWS ECR From 1b403b853441b8e7511694d8a96c879b1a657e53 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 16:18:26 +1100 Subject: [PATCH 07/32] working --- .github/workflows/node.js-dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index a483614b..3be4a71c 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -59,6 +59,7 @@ jobs: - name: Build, tag, and push image to ECR id: build-image if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} + working-directory: backend env: SHA_TAG: ${{ steps.vars.outputs.sha_short }} LATEST_TAG: latest @@ -70,7 +71,7 @@ jobs: -t $ECR_REGISTRY/$AWS_ECR_NAME:$SHA_TAG \ -t $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG \ -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \ - -f backend/Dockerfile \ + -f Dockerfile \ . docker push $ECR_REGISTRY/$AWS_ECR_NAME --all-tags From de4d278bb62291a4cd8b1f1e5a6458ce6ac01763 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 16:26:53 +1100 Subject: [PATCH 08/32] env --- .github/workflows/node.js-dev.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index 3be4a71c..5a4e0564 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -64,6 +64,10 @@ jobs: SHA_TAG: ${{ steps.vars.outputs.sha_short }} LATEST_TAG: latest ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }} + MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }} + MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }} + GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} run: | # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }} docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true From 26f2dbf8686cc5aa7ed6059e9004f404e61045b2 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 16:35:44 +1100 Subject: [PATCH 09/32] secret --- .github/workflows/node.js-dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index 5a4e0564..3f006b5d 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -41,8 +41,8 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.REGION }} - name: Login to AWS ECR From d9ff3d8fa2003dda2be884c2ee352353ee342522 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 17:35:39 +1100 Subject: [PATCH 10/32] add args --- .github/workflows/node.js-dev.yml | 4 +++- backend/Dockerfile | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index 3f006b5d..da1bb09b 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -65,9 +65,9 @@ jobs: LATEST_TAG: latest ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }} + GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }} MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }} - GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} run: | # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }} docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true @@ -75,6 +75,8 @@ jobs: -t $ECR_REGISTRY/$AWS_ECR_NAME:$SHA_TAG \ -t $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG \ -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \ + --build-arg REQUEST_GITHUB_URL_API_ENDPOINT=$REQUEST_GITHUB_URL_API_ENDPOINT \ + --build-arg GIT_ACCESS_API_TOKEN=$GIT_ACCESS_API_TOKEN \ -f Dockerfile \ . diff --git a/backend/Dockerfile b/backend/Dockerfile index 37f20133..0a313b68 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -8,6 +8,8 @@ RUN npm run build FROM mongo:7.0 ARG DEBIAN_FRONTEND=noninteractive +ARG REQUEST_GITHUB_URL_API_ENDPOINT +ARG GIT_ACCESS_API_TOKEN # Install dependencies RUN apt-get update && apt-get install -y \ From 3f795dda402627398a267cd573fb31fa4a045c91 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 17:49:54 +1100 Subject: [PATCH 11/32] mongo --- .github/workflows/node.js-dev.yml | 4 +++- backend/Dockerfile | 1 + docker-compose.yml | 5 ----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index da1bb09b..3697b697 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -28,9 +28,9 @@ jobs: - name: Running the Test Suit env: REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }} + GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }} MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }} - GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} run: | cp '.env example' .env npm install @@ -68,6 +68,7 @@ jobs: GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }} MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }} + MONGODB_URI: ${{ secrets.MONGODB_URI }} run: | # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }} docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true @@ -77,6 +78,7 @@ jobs: -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \ --build-arg REQUEST_GITHUB_URL_API_ENDPOINT=$REQUEST_GITHUB_URL_API_ENDPOINT \ --build-arg GIT_ACCESS_API_TOKEN=$GIT_ACCESS_API_TOKEN \ + --build-arg MONGODB_URI=$MONGODB_URI \ -f Dockerfile \ . diff --git a/backend/Dockerfile b/backend/Dockerfile index 0a313b68..ffcd6eba 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -10,6 +10,7 @@ FROM mongo:7.0 ARG DEBIAN_FRONTEND=noninteractive ARG REQUEST_GITHUB_URL_API_ENDPOINT ARG GIT_ACCESS_API_TOKEN +ARG MONGODB_URI # Install dependencies RUN apt-get update && apt-get install -y \ diff --git a/docker-compose.yml b/docker-compose.yml index 77a8334d..cd86fa45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,5 @@ version: "3.3" -volumes: - mongo: - networks: default: @@ -26,8 +23,6 @@ services: - REPO_PATH=${REPO_PATH} - GITHUB_LINKS=${GITHUB_LINKS} restart: no - volumes: - - mongo:/data/db frontend: depends_on: From c33dd92a113f052159748f56dd7346fd31cc357d Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 18:01:36 +1100 Subject: [PATCH 12/32] remove some vars --- backend/helm/staging/values.yaml | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml index f3ef3207..791c1481 100644 --- a/backend/helm/staging/values.yaml +++ b/backend/helm/staging/values.yaml @@ -48,30 +48,6 @@ env: NODE_ENV: type: kv value: development - FOLDER_REPOSITORY_NAME: - type: kv - value: mips_repository - REPO_PATH: - type: kv - value: https://github.com/makerdao/mips.git - REPO_BRANCH: - type: kv - value: master - FOLDER_PATTERN: - type: kv - value: 'MIP*' - REQUEST_GITHUB_URL_API_ENDPOINT: - type: kv - value: https://api.github.com/graphql - MIP_GITHUB_REPOSITORY: - type: kv - value: mips - MIP_GITHUB_REPOSITORY_OWNER: - type: kv - value: makerdao - GITHUB_LINKS: - type: kv - value: https://github.com/makerdao/mips/blob/master MONGODB_URI: type: parameterStore name: mongodb-uri @@ -80,14 +56,6 @@ env: type: parameterStore name: webhooks-secret-token parameter_name: /eks/maker-staging/mips-backend/webhooks-secret-token - GIT_ACCESS_API_TOKEN: - type: parameterStore - name: git-access-api-token - parameter_name: /eks/maker-staging/mips-backend/git-access-api-token - GITHUB_REPOSITORY_ID: - type: parameterStore - name: github-repository-id - parameter_name: /eks/maker-staging/mips-backend/github-repository-id externalSecrets: clusterSecretStoreName: maker-staging From bf7c46a462f3df801c29e2ea1c0eb35e513bb743 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 19:28:07 +1100 Subject: [PATCH 13/32] mongo --- backend/Dockerfile | 2 +- docker-compose.yml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index ffcd6eba..9ade089f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -5,7 +5,7 @@ RUN npm install COPY . . RUN npm run build -FROM mongo:7.0 +FROM mongo:4.4.8-focal ARG DEBIAN_FRONTEND=noninteractive ARG REQUEST_GITHUB_URL_API_ENDPOINT diff --git a/docker-compose.yml b/docker-compose.yml index cd86fa45..88e861a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,9 +5,11 @@ networks: services: backend: - build: - context: backend - dockerfile: Dockerfile + # build: + # context: backend + # dockerfile: Dockerfile + image: 068992353948.dkr.ecr.us-east-2.amazonaws.com/mips-backend-staging + platform: linux/amd64 ports: - 3000:3000 environment: From dff87ed9953d64895655a08cc7d26dfcb0773fb4 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 20:18:46 +1100 Subject: [PATCH 14/32] default --- backend/Dockerfile | 3 ++- backend/helm/staging/values.yaml | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 9ade089f..f81c08d8 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -10,7 +10,8 @@ FROM mongo:4.4.8-focal ARG DEBIAN_FRONTEND=noninteractive ARG REQUEST_GITHUB_URL_API_ENDPOINT ARG GIT_ACCESS_API_TOKEN -ARG MONGODB_URI +ARG MONGODB_URI=mongodb://localhost:27017/dao +ENV MONGODB_URI=${MONGODB_URI} # Install dependencies RUN apt-get update && apt-get install -y \ diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml index 791c1481..a5af24b3 100644 --- a/backend/helm/staging/values.yaml +++ b/backend/helm/staging/values.yaml @@ -48,10 +48,6 @@ env: NODE_ENV: type: kv value: development - MONGODB_URI: - type: parameterStore - name: mongodb-uri - parameter_name: /eks/maker-staging/mips-backend/mongodb-uri WEBHOOKS_SECRET_TOKEN: type: parameterStore name: webhooks-secret-token From 1ab7ba103a12f7ad2379fca23f466da21374c8f3 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 20:19:13 +1100 Subject: [PATCH 15/32] no set --- .github/workflows/node.js-dev.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index 3697b697..f038650c 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -78,7 +78,6 @@ jobs: -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \ --build-arg REQUEST_GITHUB_URL_API_ENDPOINT=$REQUEST_GITHUB_URL_API_ENDPOINT \ --build-arg GIT_ACCESS_API_TOKEN=$GIT_ACCESS_API_TOKEN \ - --build-arg MONGODB_URI=$MONGODB_URI \ -f Dockerfile \ . From 9d42365d849789f7df55b4195eefbaf565817388 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 15 Mar 2024 21:03:04 +1100 Subject: [PATCH 16/32] uri --- backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index f81c08d8..295350c6 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -10,7 +10,7 @@ FROM mongo:4.4.8-focal ARG DEBIAN_FRONTEND=noninteractive ARG REQUEST_GITHUB_URL_API_ENDPOINT ARG GIT_ACCESS_API_TOKEN -ARG MONGODB_URI=mongodb://localhost:27017/dao +ARG MONGODB_URI=mongodb://localhost:27017/ ENV MONGODB_URI=${MONGODB_URI} # Install dependencies From e29e7314bb714cf3a06316fc4d7b1e5eceb072ed Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Mon, 15 Apr 2024 16:59:57 +1000 Subject: [PATCH 17/32] update actions --- .github/workflows/angular-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/angular-dev.yml b/.github/workflows/angular-dev.yml index 2b17de45..bd475980 100644 --- a/.github/workflows/angular-dev.yml +++ b/.github/workflows/angular-dev.yml @@ -22,7 +22,7 @@ jobs: node-version: [12.x] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Cache node modules uses: actions/cache@v3 From 39103860b5e94c66a444eb0b3a26fca0e3c2f8f4 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Mon, 15 Apr 2024 17:06:44 +1000 Subject: [PATCH 18/32] version --- .github/workflows/angular-dev.yml | 2 +- .github/workflows/node.js-dev.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/angular-dev.yml b/.github/workflows/angular-dev.yml index bd475980..11b369b6 100644 --- a/.github/workflows/angular-dev.yml +++ b/.github/workflows/angular-dev.yml @@ -22,7 +22,7 @@ jobs: node-version: [12.x] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - name: Cache node modules uses: actions/cache@v3 diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index f038650c..a919bc47 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Running the Test Suit env: From 28b8d1f2639271938b25266dae91181608eb54e2 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Mon, 15 Apr 2024 17:26:18 +1000 Subject: [PATCH 19/32] creds --- backend/helm/staging/values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml index a5af24b3..cdd33028 100644 --- a/backend/helm/staging/values.yaml +++ b/backend/helm/staging/values.yaml @@ -48,6 +48,12 @@ env: NODE_ENV: type: kv value: development + MONGO_INITDB_ROOT_USERNAME: + type: kv + value: admin + MONGO_INITDB_ROOT_PASSWORD: + type: kv + value: password WEBHOOKS_SECRET_TOKEN: type: parameterStore name: webhooks-secret-token From 29d65e76b2e4c0337088cdb7330ce411359aa295 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Tue, 16 Apr 2024 17:13:40 +1000 Subject: [PATCH 20/32] run --- backend/Dockerfile | 3 +-- backend/helm/staging/values.yaml | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 295350c6..c0faaaaa 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -10,8 +10,7 @@ FROM mongo:4.4.8-focal ARG DEBIAN_FRONTEND=noninteractive ARG REQUEST_GITHUB_URL_API_ENDPOINT ARG GIT_ACCESS_API_TOKEN -ARG MONGODB_URI=mongodb://localhost:27017/ -ENV MONGODB_URI=${MONGODB_URI} +ENV MONGODB_URI=mongodb://localhost:27017/ # Install dependencies RUN apt-get update && apt-get install -y \ diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml index cdd33028..5a5c85f6 100644 --- a/backend/helm/staging/values.yaml +++ b/backend/helm/staging/values.yaml @@ -14,6 +14,12 @@ image: pullPolicy: Always tag: latest +command: + - tail +args: + - -f + - /dev/null + serviceAccount: create: false @@ -48,12 +54,6 @@ env: NODE_ENV: type: kv value: development - MONGO_INITDB_ROOT_USERNAME: - type: kv - value: admin - MONGO_INITDB_ROOT_PASSWORD: - type: kv - value: password WEBHOOKS_SECRET_TOKEN: type: parameterStore name: webhooks-secret-token @@ -62,13 +62,13 @@ env: externalSecrets: clusterSecretStoreName: maker-staging -livenessProbe: - initialDelaySeconds: 5 - periodSeconds: 30 - tcpSocket: - port: 3000 -readinessProbe: - initialDelaySeconds: 5 - periodSeconds: 30 - tcpSocket: - port: 3000 +# livenessProbe: +# initialDelaySeconds: 5 +# periodSeconds: 30 +# tcpSocket: +# port: 3000 +# readinessProbe: +# initialDelaySeconds: 5 +# periodSeconds: 30 +# tcpSocket: +# port: 3000 From 8064b070b7f3032d2eeddcadb1b1beb8e16256c0 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Tue, 16 Apr 2024 17:19:47 +1000 Subject: [PATCH 21/32] probes --- backend/helm/staging/values.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml index 5a5c85f6..6640fb03 100644 --- a/backend/helm/staging/values.yaml +++ b/backend/helm/staging/values.yaml @@ -62,12 +62,12 @@ env: externalSecrets: clusterSecretStoreName: maker-staging -# livenessProbe: -# initialDelaySeconds: 5 -# periodSeconds: 30 -# tcpSocket: -# port: 3000 -# readinessProbe: +livenessProbe: {} + # initialDelaySeconds: 5 + # periodSeconds: 30 + # tcpSocket: + # port: 3000 +readinessProbe: {} # initialDelaySeconds: 5 # periodSeconds: 30 # tcpSocket: From 03f5dd1809bc0c0332958a723b2f37f1372a1298 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Tue, 16 Apr 2024 17:34:08 +1000 Subject: [PATCH 22/32] command --- .github/workflows/node.js-dev.yml | 1 - backend/helm/staging/values.yaml | 24 ++++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index a919bc47..a4a71c16 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -68,7 +68,6 @@ jobs: GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }} MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }} - MONGODB_URI: ${{ secrets.MONGODB_URI }} run: | # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }} docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml index 6640fb03..2b67b209 100644 --- a/backend/helm/staging/values.yaml +++ b/backend/helm/staging/values.yaml @@ -62,13 +62,21 @@ env: externalSecrets: clusterSecretStoreName: maker-staging -livenessProbe: {} - # initialDelaySeconds: 5 - # periodSeconds: 30 +livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + # tcpSocket: + # port: 3000 + command: + - sh + - -c + - ps aux +readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + command: + - sh + - -c + - ps aux # tcpSocket: # port: 3000 -readinessProbe: {} -# initialDelaySeconds: 5 -# periodSeconds: 30 -# tcpSocket: -# port: 3000 From 6bd8f9a61e357daf0295c6bf0e959bcc8d41cdb5 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Tue, 16 Apr 2024 17:46:43 +1000 Subject: [PATCH 23/32] exec --- backend/helm/staging/values.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml index 2b67b209..4fcf8b61 100644 --- a/backend/helm/staging/values.yaml +++ b/backend/helm/staging/values.yaml @@ -67,16 +67,18 @@ livenessProbe: periodSeconds: 30 # tcpSocket: # port: 3000 - command: - - sh - - -c - - ps aux + exec: + command: + - sh + - -c + - ps aux readinessProbe: initialDelaySeconds: 5 periodSeconds: 30 - command: - - sh - - -c - - ps aux + exec: + command: + - sh + - -c + - ps aux # tcpSocket: # port: 3000 From 0fe061766042a5713f1739238d603de8dd6568a6 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Wed, 17 Apr 2024 13:58:44 +1000 Subject: [PATCH 24/32] back --- backend/Dockerfile | 2 +- backend/helm/staging/values.yaml | 24 ++++-------------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index c0faaaaa..6c4400ca 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -10,7 +10,7 @@ FROM mongo:4.4.8-focal ARG DEBIAN_FRONTEND=noninteractive ARG REQUEST_GITHUB_URL_API_ENDPOINT ARG GIT_ACCESS_API_TOKEN -ENV MONGODB_URI=mongodb://localhost:27017/ +ENV MONGODB_URI=mongodb://127.0.0.1:27017/ # Install dependencies RUN apt-get update && apt-get install -y \ diff --git a/backend/helm/staging/values.yaml b/backend/helm/staging/values.yaml index 4fcf8b61..a5af24b3 100644 --- a/backend/helm/staging/values.yaml +++ b/backend/helm/staging/values.yaml @@ -14,12 +14,6 @@ image: pullPolicy: Always tag: latest -command: - - tail -args: - - -f - - /dev/null - serviceAccount: create: false @@ -65,20 +59,10 @@ externalSecrets: livenessProbe: initialDelaySeconds: 5 periodSeconds: 30 - # tcpSocket: - # port: 3000 - exec: - command: - - sh - - -c - - ps aux + tcpSocket: + port: 3000 readinessProbe: initialDelaySeconds: 5 periodSeconds: 30 - exec: - command: - - sh - - -c - - ps aux - # tcpSocket: - # port: 3000 + tcpSocket: + port: 3000 From 9438ddccb07ebaba28c9597d3488c6f4f650a2e0 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Wed, 17 Apr 2024 15:52:43 +1000 Subject: [PATCH 25/32] helm --- .github/workflows/node.js-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index a4a71c16..ee4c24a9 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -105,5 +105,5 @@ jobs: timeout: 5m0s name: ${{ env.SERVICE_NAME }} chart-repository: https://techops-services.github.io/helm-charts - version: 0.0.29 + version: 0.0.31 atomic: true From 13d45177baa98a72c1e55932bb6cdaf04fa35e10 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Thu, 18 Apr 2024 13:15:29 +1000 Subject: [PATCH 26/32] frontend --- .github/workflows/angular-dev.yml | 114 ++++++++++++------ .github/workflows/node.js-dev.yml | 5 +- frontend/Dockerfile | 4 +- frontend/src/environments/environment.ts | 5 +- .../values.yaml => helm/staging/backend.yaml | 0 helm/staging/frontend.yaml | 64 ++++++++++ 6 files changed, 143 insertions(+), 49 deletions(-) rename backend/helm/staging/values.yaml => helm/staging/backend.yaml (100%) create mode 100644 helm/staging/frontend.yaml diff --git a/.github/workflows/angular-dev.yml b/.github/workflows/angular-dev.yml index 11b369b6..fad2fcfc 100644 --- a/.github/workflows/angular-dev.yml +++ b/.github/workflows/angular-dev.yml @@ -1,52 +1,86 @@ -# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions - name: Build DEV Angular - Frontend on: push: - branches: [ develop ] + branches: [ docker ] paths: - "frontend/**" - # pull_request: - # branches: [ develop ] - # paths: - # - "frontend/**" - workflow_dispatch: + - ".github/workflows/angular-dev.yml" jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - node-version: [12.x] + env: + REGION: us-east-2 + CLUSTER_NAME: maker-staging + SERVICE_NAME: mips-frontend + AWS_ECR_NAME: mips-frontend-staging + ENVIRONMENT_TAG: staging + HELM_FILE: helm/staging/frontend.yaml steps: - - uses: actions/checkout@v3 - - - name: Cache node modules - uses: actions/cache@v3 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - name: Node ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - name: npm install and npm run build - run: | - npm i - npm run build --prod - working-directory: frontend - - name: Copy file via scp - uses: appleboy/scp-action@v0.1.3 - env: - HOST: ${{ secrets.SSH_HOST }} - USERNAME: ${{ secrets.SSH_USER }} - PORT: ${{ secrets.SSH_PORT }} - PASSWORD: ${{ secrets.SSH_PASS }} - with: - source: "./frontend/dist" - target: "/var/warehouse/makerdao-mips-develop" + - uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.REGION }} + + - name: Login to AWS ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Extract commit hash + id: vars + if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} + shell: bash + run: | + echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + + - name: Build, tag, and push image to ECR + id: build-image + if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} + working-directory: frontend + env: + SHA_TAG: ${{ steps.vars.outputs.sha_short }} + LATEST_TAG: latest + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }} + docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true + docker build -t $AWS_ECR_NAME \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$SHA_TAG \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \ + -f Dockerfile \ + . + + docker push $ECR_REGISTRY/$AWS_ECR_NAME --all-tags + + - name: Replace variables in the Helm values file + id: replace-vars + if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # USERS_LIST: ${{ secrets.STAGING_USERS_LIST }} + run: | + sed -i 's/${ECR_REGISTRY}/'$ECR_REGISTRY/ $HELM_FILE + sed -i 's/${USERS_LIST}/'$USERS_LIST/ $HELM_FILE + + - name: Deploying Service to Kubernetes with Helm + id: deploy + if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} + uses: bitovi/github-actions-deploy-eks-helm@v1.2.8 + with: + values: image.repository=${{ steps.login-ecr.outputs.registry }}/${{ env.AWS_ECR_NAME }},image.tag=${{ steps.vars.outputs.sha_short }} + cluster-name: ${{ env.CLUSTER_NAME }} + config-files: ${{ env.HELM_FILE }} + chart-path: techops-services/common + namespace: ${{ env.SERVICE_NAME }} + timeout: 5m0s + name: ${{ env.SERVICE_NAME }} + chart-repository: https://techops-services.github.io/helm-charts + version: 0.0.31 + atomic: true diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index ee4c24a9..91a31219 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -1,6 +1,3 @@ -# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions - name: Node.js develop CI - Backend on: @@ -19,7 +16,7 @@ jobs: SERVICE_NAME: mips-backend AWS_ECR_NAME: mips-backend-staging ENVIRONMENT_TAG: staging - HELM_FILE: backend/helm/staging/values.yaml + HELM_FILE: helm/staging/backend.yaml steps: - name: Checkout diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 5f064111..ef568080 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -3,14 +3,14 @@ FROM node:14.16.1-alpine3.13 as builder WORKDIR /usr/src/app COPY . . + RUN npm install + # prod # RUN npm run build-prod - # dev RUN npm run build - FROM nginx:alpine COPY --from=builder /usr/src/app/dist/frontend /usr/share/nginx/html diff --git a/frontend/src/environments/environment.ts b/frontend/src/environments/environment.ts index fe0fc0be..ab301034 100644 --- a/frontend/src/environments/environment.ts +++ b/frontend/src/environments/environment.ts @@ -4,9 +4,8 @@ export const environment = { production: false, - apiUrl: 'http://backend:3000', - // apiUrl: 'http://159.203.86.45:3001', - // apiUrl: "https://mips-api.makerdao.com", + // apiUrl: 'http://backend:3000', + apiUrl: "https://mips-api-staging.makerdao.com", repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xzbyjjnb', githubURL: 'https://github.com/', diff --git a/backend/helm/staging/values.yaml b/helm/staging/backend.yaml similarity index 100% rename from backend/helm/staging/values.yaml rename to helm/staging/backend.yaml diff --git a/helm/staging/frontend.yaml b/helm/staging/frontend.yaml new file mode 100644 index 00000000..6d6dbd77 --- /dev/null +++ b/helm/staging/frontend.yaml @@ -0,0 +1,64 @@ +replicaCount: 1 +service: + enabled: true + name: mips-frontend + port: 8000 + type: ClusterIP + containerPort: 80 + tls: + enabled: true + issuerName: letsencrypt + +image: + repository: ${ECR_REGISTRY}/mips-frontend-staging + pullPolicy: Always + tag: latest + +serviceAccount: + create: false + +ingress: + enabled: true + hosts: + - mips-staging.makerdao.com + annotations: + external-dns.alpha.kubernetes.io/cloudflare-proxied: "false" + +httpBasicAuth: + enabled: false + # usersList: "$${USERS_LIST}" + +# If enabled will create Traefik Middleware and apply to Ingress +# to redirect http to https and www to non-www +httpWwwRedirect: + enabled: false + +podAnnotations: + reloader.stakater.com/auto: "true" + +resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 250m + memory: 256Mi + +env: + NODE_ENV: + type: kv + value: development + +externalSecrets: + clusterSecretStoreName: maker-staging + +livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + tcpSocket: + port: 80 +readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + tcpSocket: + port: 80 From 9b904d2aec7519d60da3a9e3f6a215307e7bb952 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Thu, 18 Apr 2024 13:39:37 +1000 Subject: [PATCH 27/32] prod --- .github/workflows/angular-dev.yml | 2 +- .github/workflows/node.js-dev.yml | 2 +- frontend/Dockerfile | 4 ++-- frontend/src/environments/environment.fullstack.ts | 5 ++--- frontend/src/environments/environment.prod.ts | 3 ++- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/angular-dev.yml b/.github/workflows/angular-dev.yml index fad2fcfc..6cc73923 100644 --- a/.github/workflows/angular-dev.yml +++ b/.github/workflows/angular-dev.yml @@ -78,7 +78,7 @@ jobs: cluster-name: ${{ env.CLUSTER_NAME }} config-files: ${{ env.HELM_FILE }} chart-path: techops-services/common - namespace: ${{ env.SERVICE_NAME }} + namespace: mips timeout: 5m0s name: ${{ env.SERVICE_NAME }} chart-repository: https://techops-services.github.io/helm-charts diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index 91a31219..f8ff1bfb 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -98,7 +98,7 @@ jobs: cluster-name: ${{ env.CLUSTER_NAME }} config-files: ${{ env.HELM_FILE }} chart-path: techops-services/common - namespace: ${{ env.SERVICE_NAME }} + namespace: mips timeout: 5m0s name: ${{ env.SERVICE_NAME }} chart-repository: https://techops-services.github.io/helm-charts diff --git a/frontend/Dockerfile b/frontend/Dockerfile index ef568080..e8a0b419 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -7,9 +7,9 @@ COPY . . RUN npm install # prod -# RUN npm run build-prod +RUN npm run build-prod # dev -RUN npm run build +# RUN npm run build FROM nginx:alpine diff --git a/frontend/src/environments/environment.fullstack.ts b/frontend/src/environments/environment.fullstack.ts index f8feee24..17f44b86 100644 --- a/frontend/src/environments/environment.fullstack.ts +++ b/frontend/src/environments/environment.fullstack.ts @@ -4,9 +4,8 @@ export const environment = { production: false, - apiUrl: 'http://backend:3000', - //apiUrl: 'http://159.203.86.45:3001', - // apiUrl: "https://mips-api.makerdao.com", + // apiUrl: 'http://backend:3000', + apiUrl: "https://mips-api-staging.makerdao.com", repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xyybvgej', githubURL: 'https://github.com/', diff --git a/frontend/src/environments/environment.prod.ts b/frontend/src/environments/environment.prod.ts index 08e54bb4..3403f99c 100644 --- a/frontend/src/environments/environment.prod.ts +++ b/frontend/src/environments/environment.prod.ts @@ -1,6 +1,7 @@ export const environment = { production: true, - apiUrl: 'https://mips-api.makerdao.com', + apiUrl: 'https://mips-api-staging.makerdao.com', + // apiUrl: 'https://mips-api.makerdao.com', repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xvodvpze', githubURL: 'https://github.com/', From 2f71f6b5e642787d7b11ec61fdb1b070e451b4ff Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Thu, 18 Apr 2024 13:53:52 +1000 Subject: [PATCH 28/32] ns --- .github/workflows/angular-dev.yml | 2 +- .github/workflows/node.js-dev.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/angular-dev.yml b/.github/workflows/angular-dev.yml index 6cc73923..49192bc9 100644 --- a/.github/workflows/angular-dev.yml +++ b/.github/workflows/angular-dev.yml @@ -78,7 +78,7 @@ jobs: cluster-name: ${{ env.CLUSTER_NAME }} config-files: ${{ env.HELM_FILE }} chart-path: techops-services/common - namespace: mips + namespace: mips-frontend timeout: 5m0s name: ${{ env.SERVICE_NAME }} chart-repository: https://techops-services.github.io/helm-charts diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index f8ff1bfb..379882cd 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -98,7 +98,7 @@ jobs: cluster-name: ${{ env.CLUSTER_NAME }} config-files: ${{ env.HELM_FILE }} chart-path: techops-services/common - namespace: mips + namespace: mips-backend timeout: 5m0s name: ${{ env.SERVICE_NAME }} chart-repository: https://techops-services.github.io/helm-charts From 91650f6384a0e93ccfef8344fee95f2d9fb2da59 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Thu, 18 Apr 2024 14:02:39 +1000 Subject: [PATCH 29/32] ns --- .github/workflows/angular-dev.yml | 2 +- .github/workflows/node.js-dev.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/angular-dev.yml b/.github/workflows/angular-dev.yml index 49192bc9..6cc73923 100644 --- a/.github/workflows/angular-dev.yml +++ b/.github/workflows/angular-dev.yml @@ -78,7 +78,7 @@ jobs: cluster-name: ${{ env.CLUSTER_NAME }} config-files: ${{ env.HELM_FILE }} chart-path: techops-services/common - namespace: mips-frontend + namespace: mips timeout: 5m0s name: ${{ env.SERVICE_NAME }} chart-repository: https://techops-services.github.io/helm-charts diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index 379882cd..f8ff1bfb 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -98,7 +98,7 @@ jobs: cluster-name: ${{ env.CLUSTER_NAME }} config-files: ${{ env.HELM_FILE }} chart-path: techops-services/common - namespace: mips-backend + namespace: mips timeout: 5m0s name: ${{ env.SERVICE_NAME }} chart-repository: https://techops-services.github.io/helm-charts From e7882d8f1e585de0bfc18be1010473bedc9f547b Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Thu, 18 Apr 2024 14:03:07 +1000 Subject: [PATCH 30/32] api --- frontend/src/environments/environment.prod.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/environments/environment.prod.ts b/frontend/src/environments/environment.prod.ts index 3403f99c..75918b22 100644 --- a/frontend/src/environments/environment.prod.ts +++ b/frontend/src/environments/environment.prod.ts @@ -1,7 +1,7 @@ export const environment = { production: true, - apiUrl: 'https://mips-api-staging.makerdao.com', - // apiUrl: 'https://mips-api.makerdao.com', + // apiUrl: 'https://mips-api-staging.makerdao.com', + apiUrl: 'https://mips-api.makerdao.com', repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xvodvpze', githubURL: 'https://github.com/', From a8c8e0ccf63c3083834fe4a1d310733c0e4b378d Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Fri, 19 Apr 2024 09:16:12 +1000 Subject: [PATCH 31/32] apiUrl --- frontend/src/environments/environment.prod.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/environments/environment.prod.ts b/frontend/src/environments/environment.prod.ts index 75918b22..3403f99c 100644 --- a/frontend/src/environments/environment.prod.ts +++ b/frontend/src/environments/environment.prod.ts @@ -1,7 +1,7 @@ export const environment = { production: true, - // apiUrl: 'https://mips-api-staging.makerdao.com', - apiUrl: 'https://mips-api.makerdao.com', + apiUrl: 'https://mips-api-staging.makerdao.com', + // apiUrl: 'https://mips-api.makerdao.com', repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xvodvpze', githubURL: 'https://github.com/', From ccb5180792ada7c29bd61439954d7303e2f4e548 Mon Sep 17 00:00:00 2001 From: Simon KP <“simon@techops.services”> Date: Mon, 29 Apr 2024 17:14:57 +1000 Subject: [PATCH 32/32] comments --- .github/workflows/node.js-dev.yml | 2 -- docker-compose.yml | 7 +++---- frontend/helm/staging/values.yaml | 0 frontend/src/environments/environment.prod.ts | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 frontend/helm/staging/values.yaml diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index f8ff1bfb..c87b446e 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -84,10 +84,8 @@ jobs: if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # USERS_LIST: ${{ secrets.STAGING_USERS_LIST }} run: | sed -i 's/${ECR_REGISTRY}/'$ECR_REGISTRY/ $HELM_FILE - sed -i 's/${USERS_LIST}/'$USERS_LIST/ $HELM_FILE - name: Deploying Service to Kubernetes with Helm id: deploy diff --git a/docker-compose.yml b/docker-compose.yml index 88e861a8..9d058e67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,10 +5,9 @@ networks: services: backend: - # build: - # context: backend - # dockerfile: Dockerfile - image: 068992353948.dkr.ecr.us-east-2.amazonaws.com/mips-backend-staging + build: + context: backend + dockerfile: Dockerfile platform: linux/amd64 ports: - 3000:3000 diff --git a/frontend/helm/staging/values.yaml b/frontend/helm/staging/values.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/frontend/src/environments/environment.prod.ts b/frontend/src/environments/environment.prod.ts index 3403f99c..75918b22 100644 --- a/frontend/src/environments/environment.prod.ts +++ b/frontend/src/environments/environment.prod.ts @@ -1,7 +1,7 @@ export const environment = { production: true, - apiUrl: 'https://mips-api-staging.makerdao.com', - // apiUrl: 'https://mips-api.makerdao.com', + // apiUrl: 'https://mips-api-staging.makerdao.com', + apiUrl: 'https://mips-api.makerdao.com', repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xvodvpze', githubURL: 'https://github.com/',