diff --git a/.github/workflows/angular-dev.yml b/.github/workflows/angular-dev.yml index 6d562350..6cc73923 100644 --- a/.github/workflows/angular-dev.yml +++ b/.github/workflows/angular-dev.yml @@ -1,52 +1,86 @@ -# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions - -name: Build DEV Angular +name: Build DEV Angular - Frontend on: push: - branches: [ develop ] + branches: [ docker ] paths: - "frontend/**" - # pull_request: - # branches: [ develop ] - # paths: - # - "frontend/**" - workflow_dispatch: + - ".github/workflows/angular-dev.yml" jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - node-version: [12.x] + env: + REGION: us-east-2 + CLUSTER_NAME: maker-staging + SERVICE_NAME: mips-frontend + AWS_ECR_NAME: mips-frontend-staging + ENVIRONMENT_TAG: staging + HELM_FILE: helm/staging/frontend.yaml steps: - - uses: actions/checkout@v1 - - - name: Cache node modules - uses: actions/cache@v3 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - name: Node ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - name: npm install and npm run build - run: | - npm i - npm run build --prod - working-directory: frontend - - name: Copy file via scp - uses: appleboy/scp-action@v0.1.3 - env: - HOST: ${{ secrets.SSH_HOST }} - USERNAME: ${{ secrets.SSH_USER }} - PORT: ${{ secrets.SSH_PORT }} - PASSWORD: ${{ secrets.SSH_PASS }} - with: - source: "./frontend/dist" - target: "/var/warehouse/makerdao-mips-develop" + - uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.REGION }} + + - name: Login to AWS ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Extract commit hash + id: vars + if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} + shell: bash + run: | + echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + + - name: Build, tag, and push image to ECR + id: build-image + if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} + working-directory: frontend + env: + SHA_TAG: ${{ steps.vars.outputs.sha_short }} + LATEST_TAG: latest + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }} + docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true + docker build -t $AWS_ECR_NAME \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$SHA_TAG \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \ + -f Dockerfile \ + . + + docker push $ECR_REGISTRY/$AWS_ECR_NAME --all-tags + + - name: Replace variables in the Helm values file + id: replace-vars + if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # USERS_LIST: ${{ secrets.STAGING_USERS_LIST }} + run: | + sed -i 's/${ECR_REGISTRY}/'$ECR_REGISTRY/ $HELM_FILE + sed -i 's/${USERS_LIST}/'$USERS_LIST/ $HELM_FILE + + - name: Deploying Service to Kubernetes with Helm + id: deploy + if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} + uses: bitovi/github-actions-deploy-eks-helm@v1.2.8 + with: + values: image.repository=${{ steps.login-ecr.outputs.registry }}/${{ env.AWS_ECR_NAME }},image.tag=${{ steps.vars.outputs.sha_short }} + cluster-name: ${{ env.CLUSTER_NAME }} + config-files: ${{ env.HELM_FILE }} + chart-path: techops-services/common + namespace: mips + timeout: 5m0s + name: ${{ env.SERVICE_NAME }} + chart-repository: https://techops-services.github.io/helm-charts + version: 0.0.31 + atomic: true diff --git a/.github/workflows/node.js-dev.yml b/.github/workflows/node.js-dev.yml index 2c120de5..c87b446e 100644 --- a/.github/workflows/node.js-dev.yml +++ b/.github/workflows/node.js-dev.yml @@ -1,76 +1,104 @@ -# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions - -name: Node.js develop CI +name: Node.js develop CI - Backend on: push: - branches: [ develop ] + branches: [ docker ] paths: - "backend/**" - ".github/workflows/node.js-dev.yml" jobs: - build: + build-deploy: runs-on: ubuntu-latest - strategy: - matrix: - node-version: [14.x] - # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ + env: + REGION: us-east-2 + CLUSTER_NAME: maker-staging + SERVICE_NAME: mips-backend + AWS_ECR_NAME: mips-backend-staging + ENVIRONMENT_TAG: staging + HELM_FILE: helm/staging/backend.yaml steps: - - uses: actions/checkout@v3 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} + - name: Checkout + uses: actions/checkout@v3 + - name: Running the Test Suit env: REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }} + GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }} MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }} - GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} run: | cp '.env example' .env npm install npm run pre-start-develop npm test - working-directory: backend + working-directory: backend - deploy: - needs: [build] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Copy file via scp - uses: appleboy/scp-action@master + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - port: ${{ secrets.SSH_PORT }} - password: ${{ secrets.SSH_PASS }} - source: "./backend" - target: "/var/warehouse/makerdao-mips-develop" + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.REGION }} + + - name: Login to AWS ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Extract commit hash + id: vars + if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} + shell: bash + run: | + echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" - - name: Executing remote command - uses: appleboy/ssh-action@master + - name: Build, tag, and push image to ECR + id: build-image + if: ${{ !contains(github.event.head_commit.message , '[skip build]') }} + working-directory: backend env: + SHA_TAG: ${{ steps.vars.outputs.sha_short }} + LATEST_TAG: latest + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} REQUEST_GITHUB_URL_API_ENDPOINT: ${{ secrets.REQUEST_GITHUB_URL_API_ENDPOINT }} + GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} MIP_GITHUB_REPOSITORY: ${{ secrets.MIP_GITHUB_REPOSITORY_DEV }} MIP_GITHUB_REPOSITORY_OWNER: ${{ secrets.MIP_GITHUB_REPOSITORY_OWNER_DEV }} - GIT_ACCESS_API_TOKEN: ${{ secrets.GIT_ACCESS_API_TOKEN }} + run: | + # Build Docker containers and push them to ECR ${{ env.AWS_ECR_NAME }} + docker pull $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG || true + docker build -t $AWS_ECR_NAME \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$SHA_TAG \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$LATEST_TAG \ + -t $ECR_REGISTRY/$AWS_ECR_NAME:$ENVIRONMENT_TAG \ + --build-arg REQUEST_GITHUB_URL_API_ENDPOINT=$REQUEST_GITHUB_URL_API_ENDPOINT \ + --build-arg GIT_ACCESS_API_TOKEN=$GIT_ACCESS_API_TOKEN \ + -f Dockerfile \ + . + + docker push $ECR_REGISTRY/$AWS_ECR_NAME --all-tags + + - name: Replace variables in the Helm values file + id: replace-vars + if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + sed -i 's/${ECR_REGISTRY}/'$ECR_REGISTRY/ $HELM_FILE + + - name: Deploying Service to Kubernetes with Helm + id: deploy + if: ${{ !contains(github.event.head_commit.message , '[skip deploy]') }} + uses: bitovi/github-actions-deploy-eks-helm@v1.2.8 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - port: ${{ secrets.SSH_PORT }} - password: ${{ secrets.SSH_PASS }} - script: | - cd /var/warehouse/makerdao-mips-develop/backend - pm2 stop dist/main.js - npm install - npm run build - npm run pre-start-develop - npx nestjs-command drop:db - npx nestjs-command parse:mips - pm2 start dist/main.js --name "api-dev" + values: image.repository=${{ steps.login-ecr.outputs.registry }}/${{ env.AWS_ECR_NAME }},image.tag=${{ steps.vars.outputs.sha_short }} + cluster-name: ${{ env.CLUSTER_NAME }} + config-files: ${{ env.HELM_FILE }} + chart-path: techops-services/common + namespace: mips + timeout: 5m0s + name: ${{ env.SERVICE_NAME }} + chart-repository: https://techops-services.github.io/helm-charts + version: 0.0.31 + atomic: true diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..6c4400ca --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,39 @@ +FROM node:18.8.0-alpine3.16 as appbuild +WORKDIR /usr/src/backend +COPY ./package*.json ./ +RUN npm install +COPY . . +RUN npm run build + +FROM mongo:4.4.8-focal + +ARG DEBIAN_FRONTEND=noninteractive +ARG REQUEST_GITHUB_URL_API_ENDPOINT +ARG GIT_ACCESS_API_TOKEN +ENV MONGODB_URI=mongodb://127.0.0.1:27017/ + +# Install dependencies +RUN apt-get update && apt-get install -y \ + curl gnupg software-properties-common git + +# Install Node.js +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ + apt-get install -y nodejs + +WORKDIR /usr/src/backend +COPY . . +COPY --from=appbuild /usr/src/backend/dist ./dist +COPY --from=appbuild /usr/src/backend/package*.json ./ +RUN npm install + +EXPOSE 3000 + +# parse the mips entries +RUN mongod --fork --logpath=/dev/null && \ + npm run pre-start && \ + npx nestjs-command drop:db && \ + npx nestjs-command parse:mips + +# start mongo and the app +CMD mongod --fork --logpath=/dev/null && \ + npm run start:prod diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml deleted file mode 100644 index 0410881c..00000000 --- a/backend/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.3" - -volumes: - mongo: - -services: - mongodb: - image: mongo:4.4.4-bionic - volumes: - - mongo:/var/lib/mongo - ports: - - 27018:27017 - environment: - MONGO_INITDB_ROOT_USERNAME: admin - MONGO_INITDB_ROOT_PASSWORD: password - restart: always diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..9d058e67 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3.3" + +networks: + default: + +services: + backend: + build: + context: backend + dockerfile: Dockerfile + platform: linux/amd64 + ports: + - 3000:3000 + environment: + - MONGODB_URI=${MONGODB_URI} + - FOLDER_REPOSITORY_NAME=${FOLDER_REPOSITORY_NAME} + - FOLDER_PATTERN=${FOLDER_PATTERN} + - WEBHOOKS_SECRET_TOKEN=${WEBHOOKS_SECRET_TOKEN} + - GIT_ACCESS_API_TOKEN=${GIT_ACCESS_API_TOKEN} + - GITHUB_REPOSITORY_ID=${GITHUB_REPOSITORY_ID} + - MIP_GITHUB_REPOSITORY=${MIP_GITHUB_REPOSITORY} + - MIP_GITHUB_REPOSITORY_OWNER=${MIP_GITHUB_REPOSITORY_OWNER} + - REPO_BRANCH=${REPO_BRANCH} + - REPO_PATH=${REPO_PATH} + - GITHUB_LINKS=${GITHUB_LINKS} + restart: no + + frontend: + depends_on: + - backend + build: + context: frontend + dockerfile: Dockerfile + ports: + - 8080:80 diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..e8a0b419 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,19 @@ +FROM node:14.16.1-alpine3.13 as builder + +WORKDIR /usr/src/app + +COPY . . + +RUN npm install + +# prod +RUN npm run build-prod +# dev +# RUN npm run build + +FROM nginx:alpine + +COPY --from=builder /usr/src/app/dist/frontend /usr/share/nginx/html +COPY --from=builder /usr/src/app/nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 00000000..b3be85a2 --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,20 @@ +server { + listen 80; + sendfile on; + default_type application/octet-stream; + + gzip on; + gzip_http_version 1.1; + gzip_disable "MSIE [1-6]\."; + gzip_min_length 256; + gzip_vary on; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_comp_level 9; + + root /usr/share/nginx/html; + + location / { + try_files $uri $uri/ /index.html =404; + } +} diff --git a/frontend/src/environments/environment.fullstack.ts b/frontend/src/environments/environment.fullstack.ts index e7046e86..17f44b86 100644 --- a/frontend/src/environments/environment.fullstack.ts +++ b/frontend/src/environments/environment.fullstack.ts @@ -4,9 +4,8 @@ export const environment = { production: false, - apiUrl: 'http://localhost:3001', - //apiUrl: 'http://159.203.86.45:3001', - // apiUrl: "https://mips-api.makerdao.com", + // apiUrl: 'http://backend:3000', + apiUrl: "https://mips-api-staging.makerdao.com", repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xyybvgej', githubURL: 'https://github.com/', diff --git a/frontend/src/environments/environment.prod.ts b/frontend/src/environments/environment.prod.ts index 08e54bb4..75918b22 100644 --- a/frontend/src/environments/environment.prod.ts +++ b/frontend/src/environments/environment.prod.ts @@ -1,5 +1,6 @@ export const environment = { production: true, + // apiUrl: 'https://mips-api-staging.makerdao.com', apiUrl: 'https://mips-api.makerdao.com', repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xvodvpze', diff --git a/frontend/src/environments/environment.ts b/frontend/src/environments/environment.ts index 563f3108..ab301034 100644 --- a/frontend/src/environments/environment.ts +++ b/frontend/src/environments/environment.ts @@ -4,9 +4,8 @@ export const environment = { production: false, - // apiUrl: 'http://localhost:3001', - apiUrl: 'http://159.203.86.45:3001', - // apiUrl: "https://mips-api.makerdao.com", + // apiUrl: 'http://backend:3000', + apiUrl: "https://mips-api-staging.makerdao.com", repoUrl: 'https://github.com/makerdao/mips/blob/master', feedBackFormUrl: 'https://formspree.io/f/xzbyjjnb', githubURL: 'https://github.com/', diff --git a/helm/staging/backend.yaml b/helm/staging/backend.yaml new file mode 100644 index 00000000..a5af24b3 --- /dev/null +++ b/helm/staging/backend.yaml @@ -0,0 +1,68 @@ +replicaCount: 1 +service: + enabled: true + name: mips-backend + port: 3000 + type: ClusterIP + containerPort: 3000 + tls: + enabled: true + issuerName: letsencrypt + +image: + repository: ${ECR_REGISTRY}/mips-backend-staging + pullPolicy: Always + tag: latest + +serviceAccount: + create: false + +ingress: + enabled: true + hosts: + - mips-api-staging.makerdao.com + annotations: + external-dns.alpha.kubernetes.io/cloudflare-proxied: "false" + +httpBasicAuth: + enabled: false + # usersList: "$${USERS_LIST}" + +# If enabled will create Traefik Middleware and apply to Ingress +# to redirect http to https and www to non-www +httpWwwRedirect: + enabled: false + +podAnnotations: + reloader.stakater.com/auto: "true" + +resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 250m + memory: 256Mi + +env: + NODE_ENV: + type: kv + value: development + WEBHOOKS_SECRET_TOKEN: + type: parameterStore + name: webhooks-secret-token + parameter_name: /eks/maker-staging/mips-backend/webhooks-secret-token + +externalSecrets: + clusterSecretStoreName: maker-staging + +livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + tcpSocket: + port: 3000 +readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + tcpSocket: + port: 3000 diff --git a/helm/staging/frontend.yaml b/helm/staging/frontend.yaml new file mode 100644 index 00000000..6d6dbd77 --- /dev/null +++ b/helm/staging/frontend.yaml @@ -0,0 +1,64 @@ +replicaCount: 1 +service: + enabled: true + name: mips-frontend + port: 8000 + type: ClusterIP + containerPort: 80 + tls: + enabled: true + issuerName: letsencrypt + +image: + repository: ${ECR_REGISTRY}/mips-frontend-staging + pullPolicy: Always + tag: latest + +serviceAccount: + create: false + +ingress: + enabled: true + hosts: + - mips-staging.makerdao.com + annotations: + external-dns.alpha.kubernetes.io/cloudflare-proxied: "false" + +httpBasicAuth: + enabled: false + # usersList: "$${USERS_LIST}" + +# If enabled will create Traefik Middleware and apply to Ingress +# to redirect http to https and www to non-www +httpWwwRedirect: + enabled: false + +podAnnotations: + reloader.stakater.com/auto: "true" + +resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 250m + memory: 256Mi + +env: + NODE_ENV: + type: kv + value: development + +externalSecrets: + clusterSecretStoreName: maker-staging + +livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + tcpSocket: + port: 80 +readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 30 + tcpSocket: + port: 80