Initial

li10labs · May 16, 2024 · c71fe85 · c71fe85
commit c71fe85
Show file tree

Hide file tree

Showing 54 changed files with 3,795 additions and 0 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,38 @@
+name: Build Docker Container
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - infra/cloudcustodian/docker/**
+      - policies/schedule-based/**
+      - .github/workflows/docker.yml
+
+jobs:
+  docker:
+    runs-on: [ self-hosted, ubuntu ]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-skip-session-tagging: true
+          role-to-assume: <<CI_ROLE_HERE>>
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          mask-password: "true" # see: https://github.com/aws-actions/amazon-ecr-login#docker-credentials
+
+      - name: Build, tag, and push docker image to Amazon ECR
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: governance
+          IMAGE_TAG: latest
+        run: |
+          docker build --platform=linux/amd64 -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f infra/cloudcustodian/docker/Dockerfile .
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml
@@ -0,0 +1,71 @@
+name: CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - "!infra/cloudcustodian/docker/**"
+      - infra/cloudcustodian/**
+      - .github/workflows/infra.yml
+      - policies/**
+  pull_request:
+    branches: [ "main" ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build:
+    # The type of runner that the job will run on
+    runs-on: [ self-hosted, ubuntu ]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+        env:
+          PIP_ROOT_USER_ACTION: ignore
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-skip-session-tagging: true
+          role-to-assume: <<CI_ROLE_HERE>>
+          aws-region: us-east-1
+
+      - name: Install Python dependencies and CDK
+        run: |
+          python -m pip install --upgrade pip
+          npm install -g aws-cdk
+
+      - name: Deploy to AWS
+        run: |
+          cd infra/cloudcustodian
+
+          echo "sha: ${{ github.sha }}"
+          echo "url: ${{ github.server_url }}//${{ github.repository }}"
+
+          python -m pip install -r requirements.txt
+          python transform.py > lambda-code/config.json
+          pip install c7n==0.9.35 -t ./tmp/lambda-code/ --no-deps --no-compile
+          cp -r ./tmp/lambda-code/c7n ./lambda-code/c7n/
+
+          pip install -r ./lambda-mailer-code/requirements-mailer.txt -t ./lambda-mailer-code/  --no-deps --no-compile
+          mv ./lambda-mailer-code/main-modified.py ./lambda-mailer-code/main.py
+          mv ./lambda-mailer-code/handle-modified.py ./lambda-mailer-code/c7n_mailer/handle.py
+          mv ./lambda-mailer-code/slack_delivery-modified.py ./lambda-mailer-code/c7n_mailer/slack_delivery.py
+          mv ./lambda-mailer-code/sqs_queue_processor-modified.py ./lambda-mailer-code/c7n_mailer/sqs_queue_processor.py
+          mv ./lambda-mailer-code/target-modified.py ./lambda-mailer-code/c7n_mailer/target.py
+          mv ./lambda-mailer-code/email_delivery-modified.py ./lambda-mailer-code/c7n_mailer/email_delivery.py
+          mv ../../policies/templates/*.j2 ./lambda-mailer-code/c7n_mailer/msg-templates/
+
+          export SHA="${{ github.sha }}"
+          export GIT="${{ github.server_url }}/${{ github.repository }}"
+          export JOB="${{ github.run_id }}"
+          export TEAM="IE"
+
+          cdk deploy --require-approval=never --all
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,17 @@
+.tfcache
+.env
+.venv
+.python-version
+.venv/**
+infra/cloudcustodian/tmp/**
+out/
+.DS_STORE
+.tox/
+.idea/
+.vscode/
+!.vscode/*.sample
+.pytest_cache/
+pip-wheel-metadata/
+pyvenv.cfg
+junit
+/.vs
diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml
@@ -0,0 +1,95 @@
+stages:
+- deploy-infra
+- docker
+
+.common:
+  before_script:
+    - echo "common script"
+    - yum install -y python pip
+    - python --version
+    - pip --version
+    - yum install -y aws-cli
+    - aws --version
+    - yum install -y nodejs
+    - node --version
+    - npm install -g aws-cdk
+    - cdk --version
+
+    - eval aws configure set aws_access_key_id \$${MyEnv}_AWS_ACCESS_KEY_ID
+    - eval aws configure set aws_secret_access_key \$${MyEnv}_AWS_SECRET_ACCESS_KEY
+    - aws sts get-caller-identity
+
+    # - aws codeartifact login --tool npm --domain greenify --domain-owner $ACCOUNT_ID --repository npmjs --region $AWS_DEFAULT_REGION
+
+deploy-infra:
+  stage: deploy-infra
+  image: amazonlinux:2023
+  extends: .common
+  variables:
+    MyEnv: dev
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+      #when: manual
+      allow_failure: true
+    #   changes:
+    #     - template.yaml
+    #     - .gitlab.yml
+  script:
+    - cd infra/cloudcustodian
+    - echo sha= ${CI_COMMIT_SHA}
+    - echo url= ${CI_PROJECT_ROOT_NAMESPACE} ${CI_PROJECT_NAME}
+    - echo id=  ${CI_JOB_ID}
+
+    - python -m pip install -r requirements.txt
+    - python transform.py > lambda-code/config.json
+    - pip install c7n==0.9.35 -t ./tmp/lambda-code/ --no-deps --no-compile
+    - cp -r ./tmp/lambda-code/c7n ./lambda-code/c7n/
+
+    - pip install -r ./lambda-mailer-code/requirements-mailer.txt -t ./lambda-mailer-code/  --no-deps --no-compile
+    - mv ./lambda-mailer-code/main-modified.py ./lambda-mailer-code/main.py
+    - mv ./lambda-mailer-code/handle-modified.py ./lambda-mailer-code/c7n_mailer/handle.py
+    - mv ./lambda-mailer-code/slack_delivery-modified.py ./lambda-mailer-code/c7n_mailer/slack_delivery.py
+    - mv ./lambda-mailer-code/sqs_queue_processor-modified.py ./lambda-mailer-code/c7n_mailer/sqs_queue_processor.py
+    - mv ./lambda-mailer-code/target-modified.py ./lambda-mailer-code/c7n_mailer/target.py
+    - mv ./lambda-mailer-code/email_delivery-modified.py ./lambda-mailer-code/c7n_mailer/email_delivery.py
+    - mv ./lambda-mailer-code/mongodb_delivery.py ./lambda-mailer-code/c7n_mailer/mongodb_delivery.py
+    - mv ../../policies/templates/*.j2 ./lambda-mailer-code/c7n_mailer/msg-templates/
+
+    - export SHA=${CI_COMMIT_SHA}
+    - export GIT=${CI_PROJECT_ROOT_NAMESPACE}/${CI_PROJECT_NAME}
+    - export JOB=${CI_JOB_ID}
+
+    - cdk deploy --require-approval=never --all
+
+docker:
+  stage: docker
+  image: docker
+  services:
+    - docker:dind
+  # extends: .common
+  variables:
+    MyEnv: dev
+    REGISTRY: 123456789123.dkr.ecr.us-east-1.amazonaws.com
+    REPOSITORY: governance
+    IMAGE_TAG: latest
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main"
+      #when: manual
+      allow_failure: true
+    #   changes:
+    #     - template.yaml
+    #     - .gitlab.yml
+  script:
+    # - yum install -y docker
+    # - systemctl start docker
+    - apk add --no-cache aws-cli
+    - eval aws configure set aws_access_key_id \$${MyEnv}_AWS_ACCESS_KEY_ID
+    - eval aws configure set aws_secret_access_key \$${MyEnv}_AWS_SECRET_ACCESS_KEY
+    - aws sts get-caller-identity
+    - docker --help
+    - whoami
+    # - apt-get update && apt-get install -y aws-cli
+    - aws --version
+    - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 123456789123.dkr.ecr.us-east-1.amazonaws.com
+    - docker build --platform=linux/amd64 -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f infra/cloudcustodian/docker/Dockerfile .
+    - docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,45 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+      # Git style
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: check-vcs-permalinks
+      - id: forbid-new-submodules
+      #- id: no-commit-to-branch
+      # Common errors
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+        exclude: CHANGELOG.md
+      - id: check-yaml
+        exclude: aws-cf/.*\.(yml|yaml)$|.*\.(yml|yaml)
+      - id: check-merge-conflict
+      - id: check-executables-have-shebangs
+      - id: check-ast
+
+      # Cross platform
+      - id: check-case-conflict
+      - id: mixed-line-ending
+        args: [--fix=lf]
+
+      # Security
+      - id: detect-aws-credentials
+        args: ["--allow-missing-credentials"]
+      - id: detect-private-key
+
+  - repo: https://github.com/awslabs/cfn-python-lint
+    rev: v0.72.10
+    hooks:
+    -   id: cfn-python-lint
+        files: aws-cf/.*\.(yml|yaml)$|sam/.*\.(yml|yaml)
+
+  # - repo: https://github.com/pre-commit/mirrors-eslint
+  #   rev: 'v8.38.0'  # Use the sha / tag you want to point at
+  #   hooks:
+  #   -   id: eslint
+  #       args: ["-c", "./.eslintrc.yml"]
+  #       additional_dependencies:
+  #       -   [email protected]
+  #       -   eslint-plugin@latest
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -0,0 +1,6 @@
+# See syntax guidelines for owners file
+# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
+
+# These owners will be the default owners for everything in the repo.
+* @matthiasbuchner
+* @ntmggr