From 934cdd778b41d6d906e2291fb482e0a8bea6eb7a Mon Sep 17 00:00:00 2001 From: Shubhada Date: Tue, 1 Oct 2024 14:50:20 -0700 Subject: [PATCH] fixed the vpn issue --- .pipelines/ci.yml | 190 ++++++++++++++++++------------------ Dockerfile.vpn | 7 +- Dockerfile.vpn.dockerignore | 2 + Makefile | 51 ++-------- docker-compose.yml | 123 +++++++++++++++++++---- 5 files changed, 215 insertions(+), 158 deletions(-) create mode 100644 Dockerfile.vpn.dockerignore diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml index 0203ee84cfa..9a81740c2b2 100644 --- a/.pipelines/ci.yml +++ b/.pipelines/ci.yml @@ -11,8 +11,6 @@ trigger: include: - v2* -# PR triggers are overridden in the ADO UI - resources: containers: - container: golang @@ -28,147 +26,153 @@ resources: variables: - template: vars.yml -jobs: - - job: Build_Test_And_Push_Az_ARO_Extension - pool: - name: 1es-aro-ci-pool - steps: - - template: ./templates/template-checkout.yml - - # Build and test the Az ARO Extension - - script: | - set -xe - DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(Build.BuildId) - displayName: 🛠 Build & Test Az ARO Extension - - # Push the image to ACR - - template: ./templates/template-acr-push.yml - parameters: - acrFQDN: 'arosvcdev.azurecr.io' - repository: 'azext-aro' - pushLatest: true - - - job: Build_And_Test_RP_And_Portal - pool: - name: 1es-aro-ci-pool - steps: - - template: ./templates/template-checkout.yml - - # Build and test RP and Portal - - script: | - set -xe - DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(Build.BuildId) - displayName: 🛠 Build & Test RP and Portal - - # Publish test results - - task: PublishTestResults@2 - displayName: 📊 Publish tests results - inputs: - testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml - condition: succeededOrFailed() - - # Publish code coverage results - - task: PublishCodeCoverageResults@2 - displayName: 📈 Publish code coverage - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml - failIfCoverageEmpty: false - condition: succeededOrFailed() +stages: + - stage: Containerized + jobs: + - job: Build_Test_And_Push_Az_ARO_Extension + pool: + name: 1es-aro-ci-pool + steps: + - template: ./templates/template-checkout.yml + + # Build and test the Az ARO Extension + - script: | + set -xe + DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(Build.BuildId) + displayName: 🛠 Build & Test Az ARO Extension # Push the image to ACR - template: ./templates/template-acr-push.yml parameters: acrFQDN: 'arosvcdev.azurecr.io' - repository: 'aro' + repository: 'azext-aro' pushLatest: true - - job: Lint_Az_ARO_Extension + - job: Build_And_Test_RP_And_Portal pool: name: 1es-aro-ci-pool - variables: - HOME: $(Agent.BuildDirectory) steps: - template: ./templates/template-checkout.yml + + # Build and test RP and Portal - script: | set -xe - export AZDEV_CONFIG_DIR=$(Agent.BuildDirectory)/azdev-config - make test-python - [[ -z "$(git status -s)" ]] - target: python + DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(Build.BuildId) + displayName: 🛠 Build & Test RP and Portal + + # Publish test results + - task: PublishTestResults@2 + displayName: 📊 Publish tests results + inputs: + testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml + condition: succeededOrFailed() + + # Publish code coverage results + - task: PublishCodeCoverageResults@2 + displayName: 📈 Publish code coverage + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml + failIfCoverageEmpty: false + condition: succeededOrFailed() - # New E2E Stage with Docker Compose - - stage: E2E + # Push the image to ACR + - template: ./templates/template-acr-push.yml + parameters: + acrFQDN: 'arosvcdev.azurecr.io' + repository: 'aro' + pushLatest: true + + - stage: E2E # E2E Stage using Docker Compose dependsOn: Containerized jobs: - job: Run_E2E_Tests pool: name: 1es-aro-ci-pool steps: - # Checkout the code + # Step 1: Checkout the code - template: ./templates/template-checkout.yml - # Install Docker Compose and pull the RP image + # Step 2: Pull the RP image from ACR - template: ./templates/e2e-pipeline-template.yml parameters: rpImageACR: 'arosvcdev.azurecr.io' acrCredentialsJSON: $(acr-credentials) - # Install OpenVPN (the command will depend on the agent OS) + # Step 3: Install Docker Compose and Docker - script: | - set -xe - sudo apt-get update && sudo apt-get install -y openvpn || \ - sudo tdnf install -y openvpn || \ - sudo yum install -y openvpn - displayName: Install OpenVPN - - # AZ CLI Login using the existing secret as in the old pipeline + sudo apt-get update + sudo apt-get install -y docker.io docker-compose-plugin make + sudo apt-get install -y openvpn + sudo systemctl start docker + sudo systemctl enable docker + displayName: Install OpenVPN, Docker, and Docker Compose + + # Step 4: AZ CLI Login using the existing secret as in the old pipeline - template: ./templates/template-az-cli-login.yml parameters: azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn) + # Step 5: Download and extract VPN secrets with `make secrets` - script: | - set -xe az account set -s $AZURE_SUBSCRIPTION_ID - - # Download secrets tarball containing the VPN certificates and config - az storage blob download --account-name $(SECRET_SA_ACCOUNT_NAME) --container-name secrets --name vpn-secrets.tar.gz --file vpn-secrets.tar.gz - - # Extract the secrets tarball + #command to download the secrets tarball and set the environment + SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets + . secrets/env sudo mkdir -p /etc/openvpn - sudo tar -xzf vpn-secrets.tar.gz -C /etc/openvpn + ls -R ./secrets + sudo cp ./secrets/* /etc/openvpn + ls -R /etc/openvpn - # Ensure the extracted files contain the required certificates - ls /etc/openvpn - displayName: Download and Extract VPN Secrets + displayName: Download and Setup VPN Secrets - # Setup Azure and source secrets/env + # Step 6: Setup environment variables for Docker Compose - script: | - set -xe export RP_IMAGE_ACR=arosvcdev.azurecr.io export VERSION=${BUILD_BUILDID} export E2E_FLAGS="--flag1 --flag2" export E2E_LABEL="test-label" - export E2E_DELETE_CLUSTER="false" echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env echo "VERSION=$VERSION" >> .env echo "E2E_FLAGS=$E2E_FLAGS" >> .env echo "E2E_LABEL=$E2E_LABEL" >> .env - + . secrets/env cat .env - sudo openvpn --config /etc/openvpn/vpn.conf & - docker-compose --env-file .env -f docker-compose.yml up -d - displayName: Start OpenVPN and Run Docker Compose for E2E Services + displayName: Set Up Environment Variables - # Log the output from the e2e container in case of failure + # Step 7: Start services using Docker Compose - script: | - set -xe - docker-compose logs e2e - displayName: Log E2E Test Output + docker compose --env-file .env -f docker-compose.yml up -d + docker ps + displayName: Start Services using Docker Compose + + # Step 8: Execute E2E tests + - script: | + docker compose exec e2e-container make test-e2e + displayName: Execute E2E Tests + + # Step 9: Log the output from the services in case of failure + - script: | + docker compose logs vpn + docker compose logs selenium + docker compose logs rp + docker compose logs e2e + displayName: Log Service Output condition: failed() - # Clean up Docker Compose + # Step 10: Clean up Docker Compose services - script: | - docker-compose down - displayName: Cleanup Docker Compose + docker compose down + displayName: Cleanup Docker Compose Services condition: always() + + # Step 11: Cleanup Azure resources + - script: | + export CI=true + . ./hack/e2e/run-rp-and-e2e.sh + clean_e2e_db + displayName: Cleanup (Azure) + condition: always() + + # Step 12: AZ CLI Logout + - template: ./templates/template-az-cli-logout.yml diff --git a/Dockerfile.vpn b/Dockerfile.vpn index d57e3739e1c..a2ab858b62f 100644 --- a/Dockerfile.vpn +++ b/Dockerfile.vpn @@ -2,10 +2,7 @@ FROM mcr.microsoft.com/azure-cli:2.61.0 AS base # Install OpenVPN +USER root RUN apk add --no-cache openvpn || tdnf install -y openvpn || dnf install -y openvpn -# Create the config directory and generate a basic vpn.conf file -RUN mkdir -p /etc/openvpn && echo "client\nremote vpn-server-address 1194\nproto udp\ndev tun\nresolv-retry infinite\nnobind\npersist-key\npersist-tun\nca ca.crt\ncert client.crt\nkey client.key\ncomp-lzo\nverb 3" > /etc/openvpn/vpn.conf - -# Run OpenVPN when the container starts -CMD ["openvpn", "--config", "/etc/openvpn/vpn.conf"] +ENTRYPOINT openvpn \ No newline at end of file diff --git a/Dockerfile.vpn.dockerignore b/Dockerfile.vpn.dockerignore new file mode 100644 index 00000000000..4ee7adaddb8 --- /dev/null +++ b/Dockerfile.vpn.dockerignore @@ -0,0 +1,2 @@ +# ignore everything +* diff --git a/Makefile b/Makefile index e3943579ab2..b9f653cc574 100644 --- a/Makefile +++ b/Makefile @@ -379,7 +379,6 @@ LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro LOCAL_TUNNEL_IMAGE ?= aro-tunnel -LOCAL_VPN_IMAGE ?= vpn_image ############################################################################### # Targets @@ -541,46 +540,12 @@ run-rp: ci-rp podman-secrets --secret proxy.crt,target=/app/secrets/proxy.crt \ $(LOCAL_ARO_RP_IMAGE):$(VERSION) rp -# Run selenium using Docker -.PHONY: run-selenium -run-selenium: - docker run -d --name selenium-container selenium/standalone-chrome - -# Run RP using Docker -.PHONY: run-rp-docker -run-rp: run-selenium - docker run -d --name rp-container $(ARO_IMAGE_BASE):$(VERSION) - -# Run E2E Tests using Docker -.PHONY: run-e2e -run-e2e: e2e.test - docker-compose run --rm e2e /usr/local/bin/e2e.test $(E2E_FLAGS) --ginkgo.label-filter=$(E2E_LABEL) - -# Clean up containers after E2E tests -.PHONY: e2e-cluster-clean -e2e-cluster-clean: - docker stop selenium-container rp-container e2e-container || true - docker rm selenium-container rp-container e2e-container || true - -# Build the VPN Docker image -.PHONY: build-vpn -build-vpn: - @echo "Building VPN image with VERSION: $(VERSION)" - docker build . $(DOCKER_BUILD_CI_ARGS) \ - -f Dockerfile.vpn \ - -t $(LOCAL_VPN_IMAGE):$(VERSION) - -# Push the VPN image to ACR -.PHONY: push-vpn -push-vpn: build-vpn - @echo "Pushing VPN image to ACR: $(RP_IMAGE_ACR)" - @echo "VERSION is: $(VERSION)" - if [ -z "$(RP_IMAGE_ACR)" ]; then \ - echo "Error: RP_IMAGE_ACR is not set"; \ - exit 1; \ - fi - # Tag the VPN image with the ACR registry and version - docker tag $(LOCAL_VPN_IMAGE):$(VERSION) $(RP_IMAGE_ACR)/vpn_image:$(VERSION) - # Push the VPN image to ACR - docker push $(RP_IMAGE_ACR)/vpn_image:$(VERSION) +.PHONY: vpn +vpn: + docker-compose build vpn +.PHONY: run-vpn +run-vpn: + docker-compose rm -sf vpn + docker-compose up vpn + docker-compose logs --follow vpn \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 93a8eb2eb27..7ea196446b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,22 +2,27 @@ version: '3.8' services: vpn: - # Use an image with OpenVPN installed (or you can use the previous image and install OpenVPN at runtime) - image: kylemanna/openvpn - container_name: vpn-container - privileged: true # Required for OpenVPN - network_mode: host # Use host network + image: vpn:${VERSION} + build: + context: . + dockerfile: Dockerfile.vpn + container_name: vpn + privileged: true volumes: - - /dev/shm:/dev/shm # Shared memory mount - - /etc/openvpn:/etc/openvpn # Mount the extracted VPN secrets - devices: - - /dev/net/tun # Required for VPN to access tunnel - command: ["openvpn", "--config", "/etc/openvpn/vpn.conf"] # Start OpenVPN with existing config + - ${PWD}/secrets:/secrets:z + entrypoint: "openvpn" + command: ["/secrets/vpn-eastus.ovpn"] + healthcheck: + test: ["CMD", "pidof", "openvpn"] + start_period: 20s + interval: 20s + timeout: 3s + retries: 3 selenium: image: selenium/standalone-chrome container_name: selenium-container - network_mode: host # Use host network (no need for port mappings) + network_mode: host healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4444"] interval: 30s @@ -26,21 +31,90 @@ services: rp: image: ${RP_IMAGE_ACR}/aro:${VERSION} - container_name: rp-container - network_mode: host # Use host network + build: + context: . + dockerfile: Dockerfile.ci-rp + args: + - REGISTRY=${REGISTRY} + - ARO_VERSION=${VERSION} + ulimits: + nofile: + soft: 4096 + hard: 4096 + container_name: aro-rp depends_on: vpn: condition: service_healthy + command: ["rp"] + secrets: + - source: proxy-client-key + target: /app/secrets/proxy-client.key + - source: proxy-client-crt + target: /app/secrets/proxy-client.crt + - source: proxy-crt + target: /app/secrets/proxy.crt + - source: hive-kubeconfig + target: /app/secrets/aks.kubeconfig + environment: + # inherit from host + - ADMIN_OBJECT_ID + - ARO_IMAGE + - AZURE_ARM_CLIENT_ID + - AZURE_CLIENT_ID + - AZURE_CLIENT_SECRET + - AZURE_DBTOKEN_CLIENT_ID + - AZURE_ENVIRONMENT + - AZURE_FP_CLIENT_ID + - AZURE_FP_SERVICE_PRINCIPAL_ID + - AZURE_GATEWAY_CLIENT_ID + - AZURE_GATEWAY_CLIENT_SECRET + - AZURE_GATEWAY_SERVICE_PRINCIPAL_ID + - AZURE_PORTAL_ACCESS_GROUP_IDS + - AZURE_PORTAL_CLIENT_ID + - AZURE_RP_CLIENT_ID + - AZURE_RP_CLIENT_SECRET + - AZURE_SERVICE_PRINCIPAL_ID + - AZURE_SUBSCRIPTION_ID + - AZURE_TENANT_ID + - DATABASE_ACCOUNT_NAME + - DATABASE_NAME + - DOMAIN_NAME + - KEYVAULT_PREFIX + - LOCATION + - MOCK_MSI_CERT + - MOCK_MSI_CLIENT_ID + - MOCK_MSI_TENANT_ID + - OIDC_STORAGE_ACCOUNT_NAME + - PARENT_DOMAIN_NAME + - PARENT_DOMAIN_RESOURCEGROUP + - PROXY_HOSTNAME + - PULL_SECRET + - RESOURCEGROUP + - SECRET_SA_ACCOUNT_NAME + - STORAGE_ACCOUNT_DOMAIN + + # override + - ARO_ADOPT_BY_HIVE=true + - ARO_CHECKOUT_PATH=/app + - ARO_INSTALL_VIA_HIVE=true + - HIVE_KUBE_CONFIG_PATH=/app/secrets/aks.kubeconfig + - KUBECONFIG=/app/secrets/aks.kubeconfig + - RP_MODE=development + expose: + - "8443" + ports: + - "127.0.0.1:8443:8443" healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8443/healthz"] + test: ["CMD", "curl", "-k", "http://localhost:8443/healthz"] interval: 30s - timeout: 10s + timeout: 30s retries: 3 + restart: on-failure:3 e2e: image: ${RP_IMAGE_ACR}/aro:${VERSION} container_name: e2e-container - network_mode: host # Use host network + network_mode: host depends_on: rp: condition: service_healthy @@ -50,7 +124,22 @@ services: - E2E_LABEL=${E2E_LABEL} entrypoint: ["/usr/local/bin/e2e.test"] command: ["${E2E_FLAGS}", "--ginkgo.label-filter=${E2E_LABEL}"] + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/healthz || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + +secrets: + proxy-client-key: + file: ./secrets/proxy-client.key + proxy-client-crt: + file: ./secrets/proxy-client.crt + proxy-crt: + file: ./secrets/proxy.crt + hive-kubeconfig: + file: ./aks.kubeconfig networks: default: - external: true + external: false