From 814a66426f89c287edd085519af5d272747789f6 Mon Sep 17 00:00:00 2001 From: Harris Tzovanakis Date: Fri, 12 Jul 2024 14:23:12 +0000 Subject: [PATCH] global: fix Dockerfile and gh-actions --- .github/workflows/build.yml | 52 +++++++++++++++++ .github/workflows/lint.yml | 21 +++++++ .github/workflows/pull-request-main.yml | 16 +++++ .github/workflows/push-main.yml | 31 ++++++++++ .github/workflows/test.yml | 59 +++++++++++++++++++ .github/workflows/test_and_build.yml | 77 ------------------------- Dockerfile | 12 +--- docker-compose.yaml | 2 +- 8 files changed, 183 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pull-request-main.yml create mode 100644 .github/workflows/push-main.yml create mode 100644 .github/workflows/test.yml delete mode 100644 .github/workflows/test_and_build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..fefda61 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,52 @@ +name: Build + +on: + workflow_call: + inputs: + ref: + description: The reference to build + type: string + required: true + image: + description: The name of the image to build + type: string + required: true + context: + description: The context used to build the image + type: string + required: true + dockerfile: + description: The path to the Dockerfile + type: string + required: false + outputs: + image-id: + description: The ID of image that has been built + value: ${{ jobs.build.outputs.image-id }} + +jobs: + build: + runs-on: ubuntu-latest + outputs: + image-id: ${{ steps.build.outputs.image-id }} + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - id: build + name: Build and push + uses: cern-sis/gh-workflows/.github/actions/docker-build@v6.2.0 + with: + image: ${{ inputs.image }} + context: ${{ inputs.context }} + dockerfile: ${{ inputs.dockerfile }} + registry: registry.cern.ch + cache: false + tags: | + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + username: ${{ secrets.HARBOR_USERNAME }} + password: ${{ secrets.HARBOR_PASSWORD }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..895f043 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,21 @@ +name: Lint + +on: + workflow_call: + inputs: + ref: + description: The reference to build + type: string + required: true + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Pre-commit check + uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/pull-request-main.yml b/.github/workflows/pull-request-main.yml new file mode 100644 index 0000000..8a045c7 --- /dev/null +++ b/.github/workflows/pull-request-main.yml @@ -0,0 +1,16 @@ +name: Pull request main + +on: + pull_request_target: + branches: [main] + +jobs: + lint: + uses: ./.github/workflows/lint.yml + with: + ref: ${{ github.ref }} + test: + uses: ./.github/workflows/test.yml + with: + ref: ${{ github.event.pull_request.head.sha }} + secrets: inherit diff --git a/.github/workflows/push-main.yml b/.github/workflows/push-main.yml new file mode 100644 index 0000000..d634ba8 --- /dev/null +++ b/.github/workflows/push-main.yml @@ -0,0 +1,31 @@ +name: Push main + +on: + push: + branches: [main] + +defaults: + run: + shell: bash + +jobs: + lint: + uses: ./.github/workflows/lint.yml + with: + ref: ${{ github.ref }} + test: + uses: ./.github/workflows/test.yml + with: + ref: ${{ github.ref }} + secrets: inherit + deploy: + needs: [lint, test] + runs-on: ubuntu-latest + steps: + - name: send event + uses: cern-sis/gh-workflows/.github/actions/kubernetes-project-new-images@v6.2.0 + with: + event-type: update + images: | + cern-sis/inspire/workflows@${{ needs.test.outputs.image-id }} + token: ${{ secrets.PAT_FIRE_EVENTS_ON_CERN_SIS_KUBERNETES }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..55ee03d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,59 @@ +name: Tests + +on: + workflow_call: + inputs: + ref: + description: The reference to build + type: string + required: true + outputs: + image-id: + description: The ID of image that has been built + value: ${{ jobs.build.outputs.image-id }} + +defaults: + run: + shell: bash + +jobs: + build: + uses: ./.github/workflows/build.yml + with: + ref: ${{ inputs.ref }} + image: cern-sis/inspire/backoffice + context: . + dockerfile: ./compose/production/django/Dockerfile + secrets: inherit + test: + needs: build + runs-on: ubuntu-latest + services: + redis: + image: redis + ports: + - 6379:6379 + postgres: + image: postgres:13 + env: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow + ports: + - 5431:5431 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Test + run: > + docker run + -v "$(pwd)"/tests:/opt/airflow/tests + -v "$(pwd)"/dags:/opt/airflow/dags + -v "$(pwd)"/airflow.cfg:/opt/airflow/airflow.cfg + -v "$(pwd)"/requirements-test.txt:/opt/airflow/requirements-test.txt + -v "$(pwd)"/data:/opt/airflow/data + ${{ inputs.image }} + bash -c "pip install -r requirements-text.txt && airflow db init && pytest /opt/airflow/tests" diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml deleted file mode 100644 index 358a769..0000000 --- a/.github/workflows/test_and_build.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Build and Test -on: - push: - branches: - - main - pull_request_target: - branches: - - main - -env: - REGISTRY: registry.cern.ch - IMAGE: cern-sis/inspire/workflows -jobs: - build_test: - name: Build and Test - runs-on: ubuntu-latest - steps: - - name: Checkout - if: ${{ github.event_name == 'push' }} - uses: actions/checkout@v3 - - - name: Checkout PR - if: ${{ github.event_name == 'pull_request_target' }} - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install Python 3 - uses: actions/setup-python@v4 - with: - python-version: 3.11.9 - - - name: Run services on docker compose - run: docker compose up -d postgres redis - - - name: List services for tests - run: docker ps - - - name: Build Image - id: build - uses: cern-sis/gh-workflows/.github/actions/docker-build@v6 - with: - registry: ${{ env.REGISTRY }} - image: ${{ env.IMAGE }} - tags: | - type=ref,event=branch - type=ref,event=pr - type=ref,event=tag - cache: false - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_PASSWORD }} - - - name: Run tests with pytest and generate report - run: | - docker run \ - --name airflowdags \ - --network=host \ - -v "$(pwd)"/tests:/opt/airflow/tests \ - -v "$(pwd)"/dags:/opt/airflow/dags \ - -v "$(pwd)"/airflow.cfg:/opt/airflow/airflow.cfg \ - -v "$(pwd)"/data:/opt/airflow/data \ - $REGISTRY/$IMAGE@${{ steps.build.outputs.image-digest }} \ - bash -c "airflow db init && \ - airflow webserver -D && \ - airflow scheduler -D && \ - airflow triggerer -D && \ - airflow celery worker -D && \ - airflow celery flower -D && \ - pytest /opt/airflow/tests" - - - name: Deploy QA - if: ${{ github.event_name == 'push' }} - uses: cern-sis/gh-workflows/.github/actions/kubernetes-project-new-images@v6 - with: - event-type: update - images: ${{ env.REGISTRY }}/${{ env.IMAGE }}@${{ steps.build.outputs.image-digest }} - token: ${{ secrets.PAT_FIRE_EVENTS_ON_CERN_SIS_KUBERNETES }} diff --git a/Dockerfile b/Dockerfile index b721835..e113789 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,6 @@ FROM apache/airflow:2.8.3-python3.11 -USER root +COPY --chown=airflow:root dags /opt/airflow/dags/ +COPY --chown=airflow:root requirements.txt ./requirements.txt -RUN mkdir -p ${AIRFLOW_HOME} && chown -R airflow: ${AIRFLOW_HOME} - -USER airflow - -RUN airflow db init -COPY requirements.txt ./requirements.txt -COPY requirements-test.txt ./requirements-test.txt -RUN pip install --no-cache-dir --upgrade --user -r requirements.txt -r requirements-test.txt +RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" -r requirements.txt diff --git a/docker-compose.yaml b/docker-compose.yaml index 83bd999..898358b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -115,7 +115,7 @@ services: start_period: 5s restart: always ports: - - 5431:5432 + - 5431:5431 inspire-db: image: postgres:13