Evaluation tests #1151

Workflow file for this run

.github/workflows/pull-evaluation-tests.yaml at d075cb9

	name: "Evaluation tests"
	run-name: "Evaluation tests"

	on:
	pull_request_target:
	types: [labeled, opened, synchronize, reopened, ready_for_review]
	branches:
	- main
	paths-ignore:
	- ".github/**"
	- ".reuse/**"
	- "LICENSES/**"
	- "config/**"
	- "data/**"
	- "docs/**"
	- "scripts/**"
	- "tests/**"
	- "*/.md"
	- "doc_indexer/**"
	- CODEOWNERS
	- LICENSE

	# global env variables.
	env:
	DOCKER_TIMEOUT: 30
	K3D_VERSION: "v5.7.2" # Required K3d version.
	REPOSITORY_FULL_NAME: "${{ github.repository }}" # <owner>/<repository-name>.
	GITHUB_EVENT_ACTION: "${{ github.event.action }}"
	IMAGE_NAME: "europe-docker.pkg.dev/kyma-project/dev/kyma-companion:PR-${{ github.event.number }}"

	jobs:
	wait-for-build:
	name: Wait for image build job
	runs-on: ubuntu-latest
	if: contains(github.event.pull_request.labels.*.name, 'evaluation requested')
	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	ref: main

	- name: Setup python
	uses: actions/setup-python@v5
	with:
	python-version: "3.10"
	cache: "pip"

	- name: Install requirements
	run: \|
	pip install -r ./scripts/python/wait-for-commit-check/requirements.txt

	- name: wait for build
	env:
	GITHUB_TOKEN: ${{ github.token }}
	# Note: 'github.event.pull_request.head.sha' is not same as 'github.sha' on pull requests.
	GIT_REF: ${{ github.event.pull_request.head.sha }}
	REPOSITORY_FULL_NAME: "${{ github.repository }}"
	# The re-useable image-builder workflow from neighbors appends the "Build image" suffix to the check run name.
	GIT_CHECK_RUN_NAME: "build / Build image"
	INTERVAL: 60
	TIMEOUT: 900
	run: \|
	python ./scripts/python/wait-for-commit-check/run.py

	evaluation-tests:
	name: Run evaluation tests
	needs: wait-for-build
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	ref: ${{ github.event.pull_request.head.ref }}
	repository: ${{ github.event.pull_request.head.repo.full_name }}

	- name: K3d Setup - Install Kubectl CLI
	run: \|
	curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
	chmod +x ./kubectl
	mv ./kubectl /usr/local/bin/kubectl
	kubectl version --client

	- name: K3d Setup - Install K3d CLI
	id: install-k3d
	run: \|
	curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh \| TAG=${K3D_VERSION} bash
	k3d --version
	k3d_version=$(k3d --version \| cut -d " " -f 3 \| head -1 \| xargs)
	if [ "$k3d_version" != "${K3D_VERSION}" ]; then
	echo "K3d version is not correct. Expected: ${K3D_VERSION}, got: $k3d_version"
	exit 1
	fi
	echo "K3d version is correct: $k3d_version"

	- name: K3d Setup - Provision K3d cluster
	run: \|
	echo "::group::K3d - Provision"
	k3d cluster create test-cluster --verbose --agents=1 -p 9080:80@loadbalancer -p 9443:443@loadbalancer -p "32000:32000@agent:0"
	echo "::endgroup::"

	- name: K3d Setup - Verify K3d context
	run: \|
	echo "::group::K3d - Context"
	context_name=$(kubectl config current-context)
	if [ "$context_name" != "k3d-test-cluster" ]; then
	echo "K3d context is not correct. Expected: k3d-test-cluster, got: $context_name"
	exit 1
	fi
	echo "K3d context is correct"
	echo "::endgroup::"

	- name: Redis Setup - Deploy Redis using Helm
	run: \|
	kubectl create namespace redis
	helm repo add redis-stack https://redis-stack.github.io/helm-redis-stack/
	echo "Installing Redis using Helm..."
	helm install --wait redis-stack redis-stack/redis-stack --set auth.enabled=false -n redis

	- name: Companion Deploy - Create secret
	env:
	COMPANION_CONFIG_BASE64: ${{ secrets.EVALUATION_TESTS_CONFIG }}
	run: \|
	kubectl create namespace ai-system
	./scripts/k8s/create-secret.sh

	- name: Companion Deploy - Apply companion manifests
	run: \|
	kubectl apply -f scripts/k8s/companion-k3d-manifest.yaml

	- name: Companion Deploy - Update image name in Deployment
	run: \|
	echo "Updating companion image to ${IMAGE_NAME}..."
	kubectl -n ai-system set image deployment/companion companion=${IMAGE_NAME}

	- name: Companion Deploy - Wait for deployment
	run: \|
	echo "Waiting for companion deployment to be ready..."
	kubectl wait --for=condition=Available deployment companion -n ai-system --timeout=300s
	sleep 30

	- name: Companion Deploy - Test reachability through NodePort
	run: \|
	curl http://localhost:32000/readyz

	- name: Companion Deploy - Debug information
	if: failure()
	run: \|
	kubectl get deploy -n ai-system
	kubectl get pod -n ai-system
	kubectl logs -n ai-system -l app.kubernetes.io/name=companion

	- name: Evaluation Tests Setup - Extract project Python version
	working-directory: tests/blackbox
	id: python-version
	run: \|
	./../../scripts/shell/extract-python-version.sh

	- name: Evaluation Tests Setup - Install Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Evaluation Tests Setup - Install Poetry
	working-directory: tests/blackbox
	run: \|
	curl -sSL https://install.python-poetry.org \| python3 -
	echo "$HOME/.local/bin" >> $GITHUB_PATH

	- name: Evaluation Tests Setup - Install dependencies
	working-directory: tests/blackbox
	run: poetry install

	- name: Run Evaluation Tests
	working-directory: tests/blackbox
	env:
	LOG_LEVEL: "DEBUG"
	TEST_DATA_PATH: "./data"
	COMPANION_API_URL: "http://localhost:32000"
	run: \|
	export CONFIG_PATH=$GITHUB_WORKSPACE/config/config.json
	echo "${{ secrets.EVALUATION_TESTS_CONFIG }}" \| base64 --decode \| jq > $CONFIG_PATH
	echo "saved config to $CONFIG_PATH!"
	poetry run python src/run_evaluation.py

	- name: Evaluation Tests - Debug information
	if: failure()
	run: \|
	kubectl get deploy -n ai-system
	kubectl get pod -n ai-system
	kubectl logs -n ai-system -l app.kubernetes.io/name=companion

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Evaluation tests #1151

Workflow file

Evaluation tests #1151

Jobs

Run details

Workflow file for this run