diff --git a/.github/workflows/.ci.yml b/.github/workflows/.ci.yml index ce1abe2..d13168a 100644 --- a/.github/workflows/.ci.yml +++ b/.github/workflows/.ci.yml @@ -1,6 +1,11 @@ name: CI on: workflow_dispatch: + inputs: + push-docker-image-to-harbor: + description: 'Push Docker Image to Harbor' + type: boolean + default: false pull_request: push: branches: @@ -56,7 +61,9 @@ jobs: run: cp object_storage_api/logging.example.ini object_storage_api/logging.ini - name: Run unit tests - run: pytest -c test/pytest.ini test/unit/ --cov + run: | + docker build --target unit-test -t object-storage-api:unit-test . + docker run object-storage-api:unit-test - name: Upload coverage reports to Codecov if: success() @@ -90,14 +97,16 @@ jobs: # Sleep 10 seconds to give time for containers to start - name: Start MongoDB and MinIO run: | - docker compose up -d mongo-db minio minio_create_buckets + docker compose -f docker-compose.test.yml up -d mongo-db minio minio_create_buckets sleep 10 - name: Create MinIO buckets run: | - docker compose up minio_create_buckets + docker compose -f docker-compose.test.yml up minio_create_buckets - name: Run e2e tests - run: pytest -c test/pytest.ini test/e2e/ --cov + run: | + docker build --target e2e-test -t object-storage-api:e2e-test . + docker run object-storage-api:e2e-test - name: Output docker logs (mongodb) if: failure() @@ -106,3 +115,36 @@ jobs: - name: Output docker logs (minio) if: failure() run: docker logs object-storage-api-minio-1 + docker: + # This job triggers only if all the other jobs succeed. It builds the Docker image + # and if run manually from Github Actions, it pushes to Harbor. + needs: [linting, unit-tests, e2e-tests] + name: Docker + runs-on: ubuntu-latest + env: + PUSH_DOCKER_IMAGE_TO_HARBOR: ${{ inputs.push-docker-image-to-harbor != null && inputs.push-docker-image-to-harbor || 'false' }} + steps: + - name: Check out repo + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + + - name: Login to Harbor + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ secrets.HARBOR_URL }} + username: ${{ secrets.HARBOR_USERNAME }} + password: ${{ secrets.HARBOR_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ secrets.HARBOR_URL }}/object-storage-api + + - name: ${{ fromJSON(env.PUSH_DOCKER_IMAGE_TO_HARBOR) && 'Build and push Docker image to Harbor' || 'Build Docker image' }} + uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 + with: + context: . + push: ${{ fromJSON(env.PUSH_DOCKER_IMAGE_TO_HARBOR) }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + target: prod \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 145cae9..b898c41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,54 @@ -FROM python:3.12.8-alpine3.20@sha256:0c4f778362f30cc50ff734a3e9e7f3b2ae876d8386f470e0c3ee1ab299cec21b +FROM python:3.12.8-alpine3.20@sha256:0c4f778362f30cc50ff734a3e9e7f3b2ae876d8386f470e0c3ee1ab299cec21b as dev WORKDIR /object-storage-api-run -COPY requirements.txt ./ +COPY pyproject.toml ./ COPY object_storage_api/ object_storage_api/ RUN --mount=type=cache,target=/root/.cache \ set -eux; \ \ - python3 -m pip install -r requirements.txt; + python3 -m pip install -r pyproject.toml; CMD ["fastapi", "dev", "object_storage_api/main.py", "--host", "0.0.0.0", "--port", "8000"] EXPOSE 8000 + +FROM dev as unit-test + +WORKDIR /object-storage-api-run + +COPY test/ test/ + +CMD ["pytest", "--config-file", "test/pytest.ini", "test/unit", "--cov"] + +FROM unit-test as e2e-test + +WORKDIR /object-storage-api-run + +CMD ["pytest", "--config-file", "test/pytest.ini", "test/e2e", "--cov"] + +FROM unit-test as test + +WORKDIR /object-storage-api-run + +CMD ["pytest", "--config-file", "test/pytest.ini", "test/", "--cov"] + +FROM python:3.12.8-alpine3.20@sha256:0c4f778362f30cc50ff734a3e9e7f3b2ae876d8386f470e0c3ee1ab299cec21b as prod + +WORKDIR /object-storage-api-run + +COPY requirements.txt ./ +COPY object_storage_api/ object_storage_api/ + +RUN --mount=type=cache,target=/root/.cache \ + set -eux; \ + \ + python3 -m pip install --no-cache-dir -r requirements.txt; \ + # Create a non-root user to run as \ + addgroup -S object-storage-api; \ + adduser -S -D -G object-storage-api -H -h /object-storage-api-run object-storage-api; + +USER object-storage-api + +CMD ["fastapi", "run", "object_storage_api/main.py", "--host", "0.0.0.0", "--port", "8000"] +EXPOSE 8000 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.dev.yml similarity index 97% rename from docker-compose.yml rename to docker-compose.dev.yml index 0323df8..1d54245 100644 --- a/docker-compose.yml +++ b/docker-compose.dev.yml @@ -1,7 +1,9 @@ services: object-storage-api: container_name: object_storage_api_container - build: . + build: + context: . + target: dev volumes: - ./object_storage_api:/object-storage-api-run/object_storage_api - ./keys:/object-storage-api-run/keys diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..89de156 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,64 @@ +services: + object-storage-api-test: + container_name: object_storage_api_container_test + build: + context: . + target: test + volumes: + - ./object_storage_api:/object-storage-api-run/object_storage_api + - ./keys:/object-storage-api-run/keys + restart: on-failure + ports: + - 8002:8000 + depends_on: + - mongo-db + - minio + environment: + DATABASE__HOST_AND_OPTIONS: object_storage_api_mongodb_container:27017/?authMechanism=SCRAM-SHA-256&authSource=admin + extra_hosts: + # Want to use localhost for MinIO connection so the presigned URLs are correct but also want to avoid using host + # networking + - "localhost:host-gateway" + + mongo-db: + image: mongo:7.0-jammy + container_name: object_storage_api_mongodb_container + volumes: + - ./mongodb/data:/data/db + restart: always + ports: + - 27018:27017 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: example + + minio: + image: minio/minio:RELEASE.2024-09-13T20-26-02Z + container_name: object_storage_minio_container + command: minio server /data + volumes: + - ./minio/data:/data + ports: + - 9000:9000 + - 9001:9001 + environment: + MINIO_ROOT_USER: root + MINIO_ROOT_PASSWORD: example_password + MINIO_ADDRESS: ":9000" + MINIO_CONSOLE_ADDRESS: ":9001" + network_mode: "host" + + # From https://stackoverflow.com/questions/66412289/minio-add-a-public-bucket-with-docker-compose + minio_create_buckets: + image: minio/mc + container_name: object_storage_minio_mc_container + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set object-storage http://localhost:9000 root example_password; + /usr/bin/mc mb object-storage/object-storage; + /usr/bin/mc mb object-storage/test-object-storage; + exit 0; + " + network_mode: "host"