diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 405a9375..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Krkn -on: - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - name: Create multi-node KinD cluster - uses: redhat-chaos/actions/kind@main - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - architecture: 'x64' - - name: Install environment - run: | - sudo apt-get install build-essential python3-dev - pip install --upgrade pip - pip install -r requirements.txt - - name: Run unit tests - run: python -m coverage run -a -m unittest discover -s tests -v - - name: Run CI - run: | - ./CI/run.sh - cat ./CI/results.markdown >> $GITHUB_STEP_SUMMARY - echo >> $GITHUB_STEP_SUMMARY - - name: Upload CI logs - uses: actions/upload-artifact@v3 - with: - name: ci-logs - path: CI/out - if-no-files-found: error - - name: Collect coverage report - run: | - python -m coverage html - - name: Publish coverage report to job summary - run: | - pip install html2text - html2text --ignore-images --ignore-links -b 0 htmlcov/index.html >> $GITHUB_STEP_SUMMARY - - name: Upload coverage data - uses: actions/upload-artifact@v3 - with: - name: coverage - path: htmlcov - if-no-files-found: error - - name: Check CI results - run: grep Fail CI/results.markdown && false || true - diff --git a/.github/workflows/functional_tests.yaml b/.github/workflows/functional_tests.yaml deleted file mode 100644 index 9a05e6c7..00000000 --- a/.github/workflows/functional_tests.yaml +++ /dev/null @@ -1,129 +0,0 @@ -on: issue_comment - -jobs: - check_user: - # This job only runs for pull request comments - name: Check User Authorization - env: - USERS: ${{vars.USERS}} - if: contains(github.event.comment.body, '/funtest') && contains(github.event.comment.html_url, '/pull/') - runs-on: ubuntu-latest - steps: - - name: Check User - run: | - for name in `echo $USERS` - do - name="${name//$'\r'/}" - name="${name//$'\n'/}" - if [ $name == "${{github.event.sender.login}}" ] - then - echo "user ${{github.event.sender.login}} authorized, action started..." - exit 0 - fi - done - echo "user ${{github.event.sender.login}} is not allowed to run functional tests Action" - exit 1 - pr_commented: - # This job only runs for pull request comments containing /functional - name: Functional Tests - if: contains(github.event.comment.body, '/funtest') && contains(github.event.comment.html_url, '/pull/') - runs-on: ubuntu-latest - needs: - - check_user - steps: - - name: Check out Kraken - uses: actions/checkout@v3 - - name: Checkout Pull Request - run: gh pr checkout ${{ github.event.issue.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Install OC CLI - uses: redhat-actions/oc-installer@v1 - with: - oc_version: latest - - name: Install python 3.9 - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Setup kraken dependencies - run: pip install -r requirements.txt - - name: Create Workdir & export the path - run: | - mkdir workdir - echo "WORKDIR_PATH=`pwd`/workdir" >> $GITHUB_ENV - - name: Generate run id - run: | - echo "RUN_ID=`date +%s`" > $GITHUB_ENV - echo "Run Id: ${RUN_ID}" - - name: Write Pull Secret - env: - PULLSECRET_BASE64: ${{ secrets.PS_64 }} - run: | - echo "$PULLSECRET_BASE64" | base64 --decode > pullsecret.txt - - name: Write Boot Private Key - env: - BOOT_KEY: ${{ secrets.CRC_KEY_FILE }} - run: | - echo -n "$BOOT_KEY" > key.txt - - name: Teardown CRC (Post Action) - uses: webiny/action-post-run@3.0.0 - id: post-run-command - with: - run: podman run --rm -v "${{ github.workspace }}:/workspace:z" -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" -e AWS_DEFAULT_REGION=us-west-2 quay.io/crcont/crc-cloud:v0.0.2 destroy --project-name "chaos-funtest-${{ env.RUN_ID }}" --backed-url "s3://krkn-crc-state/${{ env.RUN_ID }}" --provider "aws" - - name: Create cluster - run: | - podman run --name crc-cloud-create --rm \ - -v ${PWD}:/workspace:z \ - -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" \ - -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \ - -e AWS_DEFAULT_REGION="us-west-2" \ - quay.io/crcont/crc-cloud:v0.0.2 \ - create aws \ - --project-name "chaos-funtest-${RUN_ID}" \ - --backed-url "s3://krkn-crc-state/${RUN_ID}" \ - --output "/workspace" \ - --aws-ami-id "ami-00f5eaf98cf42ef9f" \ - --pullsecret-filepath /workspace/pullsecret.txt \ - --key-filepath /workspace/key.txt - - - name: Setup kubeconfig - continue-on-error: true - run: | - ssh -o StrictHostKeyChecking=no -i id_rsa core@$(cat host) "cat /opt/kubeconfig" > kubeconfig - sed -i "s/https:\/\/api.crc.testing:6443/https:\/\/`cat host`.nip.io:6443/g" kubeconfig - echo "KUBECONFIG=${PWD}/kubeconfig" > $GITHUB_ENV - - - name: Example deployment, GitHub Action env init - env: - NAMESPACE: test-namespace - DEPLOYMENT_NAME: test-nginx - run: ./CI/CRC/init_github_action.sh - - name: Setup test suite - run: | - yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml - yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml - yq -i '.kraken.kubeconfig_path="'${KUBECONFIG}'"' CI/config/common_test_config.yaml - echo "test_app_outages_gh" > ./CI/tests/my_tests - echo "test_container" >> ./CI/tests/my_tests - echo "test_namespace" >> ./CI/tests/my_tests - echo "test_net_chaos" >> ./CI/tests/my_tests - echo "test_time" >> ./CI/tests/my_tests - - - name: Print affected config files - run: | - echo -e "## CI/config/common_test_config.yaml\n\n" - cat CI/config/common_test_config.yaml - - - name: Running test suite - run: | - ./CI/run.sh - - name: Print test output - run: cat CI/out/* - - name: Create coverage report - run: | - echo "# Test results" > $GITHUB_STEP_SUMMARY - cat CI/results.markdown >> $GITHUB_STEP_SUMMARY - echo "# Test coverage" >> $GITHUB_STEP_SUMMARY - python -m coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - - diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..a195a95d --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,102 @@ +name: Functional & Unit Tests +on: + pull_request: +jobs: + tests: + name: Functional & Unit Tests + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + - name: Create multi-node KinD cluster + uses: redhat-chaos/actions/kind@main + - name: Install Helm & add repos + run: | + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo add stable https://charts.helm.sh/stable + helm repo update + - name: Deploy prometheus & Port Forwarding + run: | + kubectl create namespace prometheus-k8s + helm install \ + --wait --timeout 360s \ + kind-prometheus \ + prometheus-community/kube-prometheus-stack \ + --namespace prometheus-k8s \ + --set prometheus.service.nodePort=30000 \ + --set prometheus.service.type=NodePort \ + --set grafana.service.nodePort=31000 \ + --set grafana.service.type=NodePort \ + --set alertmanager.service.nodePort=32000 \ + --set alertmanager.service.type=NodePort \ + --set prometheus-node-exporter.service.nodePort=32001 \ + --set prometheus-node-exporter.service.type=NodePort + + SELECTOR=`kubectl -n prometheus-k8s get service kind-prometheus-kube-prome-prometheus -o wide --no-headers=true | awk '{ print $7 }'` + POD_NAME=`kubectl -n prometheus-k8s get pods --selector="$SELECTOR" --no-headers=true | awk '{ print $1 }'` + kubectl -n prometheus-k8s port-forward $POD_NAME 9090:9090 & + sleep 5 + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + architecture: 'x64' + - name: Install environment + run: | + sudo apt-get install build-essential python3-dev + pip install --upgrade pip + pip install -r requirements.txt +# - name: Run unit tests +# run: python -m coverage run -a -m unittest discover -s tests -v + - name: Deploy test workloads + run: | + kubectl apply -f CI/templates/container_scenario_pod.yaml + kubectl wait --for=condition=ready pod -l scenario=container --timeout=300s + kubectl create namespace namespace-scenario + kubectl apply -f CI/templates/time_pod.yaml + kubectl wait --for=condition=ready pod -l scenario=time-skew --timeout=300s + - name: Get Kind nodes + run: | + kubectl get nodes --show-labels=true + + - name: Setup Functional Tests + run: | + yq -i '.kraken.distribution="kubernetes"' CI/config/common_test_config.yaml + yq -i '.kraken.port="8081"' CI/config/common_test_config.yaml + yq -i '.kraken.signal_address="0.0.0.0"' CI/config/common_test_config.yaml + yq -i '.kraken.performance_monitoring="localhost:9090"' CI/config/common_test_config.yaml + # echo "test_app_outages_gh" > ./CI/tests/my_tests + echo "test_container" >> ./CI/tests/my_tests + echo "test_namespace" >> ./CI/tests/my_tests + echo "test_net_chaos" >> ./CI/tests/my_tests + echo "test_time" >> ./CI/tests/my_tests + - name: Run Functional tests + run: | + ./CI/run.sh + cat ./CI/results.markdown >> $GITHUB_STEP_SUMMARY + echo >> $GITHUB_STEP_SUMMARY + - name: Run Unit tests + run: python -m coverage run -a -m unittest discover -s tests -v + - name: Upload CI logs + uses: actions/upload-artifact@v3 + with: + name: ci-logs + path: CI/out + if-no-files-found: error + - name: Collect coverage report + run: | + python -m coverage html + - name: Publish coverage report to job summary + run: | + pip install html2text + html2text --ignore-images --ignore-links -b 0 htmlcov/index.html >> $GITHUB_STEP_SUMMARY + - name: Upload coverage data + uses: actions/upload-artifact@v3 + with: + name: coverage + path: htmlcov + if-no-files-found: error + - name: Check CI results + run: grep Fail CI/results.markdown && false || true + diff --git a/CI/CRC/deployment.yaml b/CI/CRC/deployment.yaml deleted file mode 100644 index 89eb1d6d..00000000 --- a/CI/CRC/deployment.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: $NAMESPACE - ---- - -apiVersion: v1 -kind: Service -metadata: - name: $DEPLOYMENT_NAME-service - namespace: $NAMESPACE -spec: - selector: - app: $DEPLOYMENT_NAME - ports: - - name: http - port: 80 - targetPort: 8080 - type: ClusterIP - ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: $NAMESPACE - name: $DEPLOYMENT_NAME-deployment -spec: - replicas: 3 - selector: - matchLabels: - app: $DEPLOYMENT_NAME - template: - metadata: - labels: - app: $DEPLOYMENT_NAME - spec: - containers: - - name: $DEPLOYMENT_NAME - image: nginxinc/nginx-unprivileged:stable-alpine - ports: - - name: http - containerPort: 8080 diff --git a/CI/CRC/init_github_action.sh b/CI/CRC/init_github_action.sh deleted file mode 100755 index d5427902..00000000 --- a/CI/CRC/init_github_action.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -SCRIPT_PATH=./CI/CRC -DEPLOYMENT_PATH=$SCRIPT_PATH/deployment.yaml - -[[ ! -f $DEPLOYMENT_PATH ]] && echo "[ERROR] please run $0 from GitHub action root directory" && exit 1 -[[ -z $DEPLOYMENT_NAME ]] && echo "[ERROR] please set \$DEPLOYMENT_NAME environment variable" && exit 1 -[[ -z $NAMESPACE ]] && echo "[ERROR] please set \$NAMESPACE environment variable" && exit 1 - - -OPENSSL=`which openssl 2>/dev/null` -[[ $? != 0 ]] && echo "[ERROR]: openssl missing, please install it and try again" && exit 1 -OC=`which oc 2>/dev/null` -[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1 -SED=`which sed 2>/dev/null` -[[ $? != 0 ]] && echo "[ERROR]: sed missing, please install it and try again" && exit 1 -JQ=`which jq 2>/dev/null` -[[ $? != 0 ]] && echo "[ERROR]: jq missing, please install it and try again" && exit 1 -ENVSUBST=`which envsubst 2>/dev/null` -[[ $? != 0 ]] && echo "[ERROR]: envsubst missing, please install it and try again" && exit 1 - - -API_PORT="6443" -API_ADDRESS="https://api.`cat host`.nip.io:${API_PORT}" -FQN=$DEPLOYMENT_NAME.apps.$API_ADDRESS - - -echo "[INF] deploying example deployment: $DEPLOYMENT_NAME in namespace: $NAMESPACE" -$ENVSUBST < $DEPLOYMENT_PATH | $OC apply -f - > /dev/null 2>&1 - -echo "[INF] creating SSL self-signed certificates for route https://$FQN" -$OPENSSL genrsa -out servercakey.pem > /dev/null 2>&1 -$OPENSSL req -new -x509 -key servercakey.pem -out serverca.crt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1 -$OPENSSL genrsa -out server.key > /dev/null 2>&1 -$OPENSSL req -new -key server.key -out server_reqout.txt -subj "/CN=$FQN/O=Red Hat Inc./C=US" > /dev/null 2>&1 -$OPENSSL x509 -req -in server_reqout.txt -days 3650 -sha256 -CAcreateserial -CA serverca.crt -CAkey servercakey.pem -out server.crt > /dev/null 2>&1 -echo "[INF] creating deployment: $DEPLOYMENT_NAME public route: https://$FQN" -$OC create route --namespace $NAMESPACE edge --service=$DEPLOYMENT_NAME-service --cert=server.crt --key=server.key --ca-cert=serverca.crt --hostname="$FQN" > /dev/null 2>&1 - - -echo "[INF] setting github action environment variables" - -NODE_NAME="`$OC get nodes -o json | $JQ -r '.items[0].metadata.name'`" -COVERAGE_FILE="`pwd`/coverage.md" -echo "DEPLOYMENT_NAME=$DEPLOYMENT_NAME" >> $GITHUB_ENV -echo "DEPLOYMENT_FQN=$FQN" >> $GITHUB_ENV -echo "API_ADDRESS=$API_ADDRESS" >> $GITHUB_ENV -echo "API_PORT=$API_PORT" >> $GITHUB_ENV -echo "NODE_NAME=$NODE_NAME" >> $GITHUB_ENV -echo "NAMESPACE=$NAMESPACE" >> $GITHUB_ENV -echo "COVERAGE_FILE=$COVERAGE_FILE" >> $GITHUB_ENV - -echo "[INF] deployment fully qualified name will be available in \${{ env.DEPLOYMENT_NAME }} with value $DEPLOYMENT_NAME" -echo "[INF] deployment name will be available in \${{ env.DEPLOYMENT_FQN }} with value $FQN" -echo "[INF] OCP API address will be available in \${{ env.API_ADDRESS }} with value $API_ADDRESS" -echo "[INF] OCP API port will be available in \${{ env.API_PORT }} with value $API_PORT" -echo "[INF] OCP node name will be available in \${{ env.NODE_NAME }} with value $NODE_NAME" -echo "[INF] coverage file will ve available in \${{ env.COVERAGE_FILE }} with value $COVERAGE_FILE" - diff --git a/CI/run.sh b/CI/run.sh index 9c00832b..bea719a4 100755 --- a/CI/run.sh +++ b/CI/run.sh @@ -1,15 +1,14 @@ #!/bin/bash -set -x MAX_RETRIES=60 -OC=`which oc 2>/dev/null` -[[ $? != 0 ]] && echo "[ERROR]: oc missing, please install it and try again" && exit 1 +KUBECTL=`which kubectl 2>/dev/null` +[[ $? != 0 ]] && echo "[ERROR]: kubectl missing, please install it and try again" && exit 1 wait_cluster_become_ready() { COUNT=1 - until `$OC get namespace > /dev/null 2>&1` + until `$KUBECTL get namespace > /dev/null 2>&1` do - echo "[INF] waiting OpenShift to become ready, after $COUNT check" + echo "[INF] waiting Kubernetes to become ready, after $COUNT check" sleep 3 [[ $COUNT == $MAX_RETRIES ]] && echo "[ERR] max retries exceeded, failing" && exit 1 ((COUNT++)) @@ -20,7 +19,7 @@ wait_cluster_become_ready() { ci_tests_loc="CI/tests/my_tests" -echo "running test suit consisting of ${ci_tests}" +echo -e "********* Running Functional Tests Suite *********\n\n" rm -rf CI/out @@ -37,9 +36,31 @@ echo 'Test | Result | Duration' >> $results echo '-----------------------|--------|---------' >> $results # Run each test +failed_tests=() for test_name in `cat CI/tests/my_tests` do wait_cluster_become_ready - ./CI/run_test.sh $test_name $results + return_value=`./CI/run_test.sh $test_name $results` + if [[ $return_value == 1 ]] + then + echo "Failed" + failed_tests+=("$test_name") + fi wait_cluster_become_ready done + +if (( ${#failed_tests[@]}>0 )) +then + echo -e "\n\n======================================================================" + echo -e "\n FUNCTIONAL TESTS FAILED ${failed_tests[*]} ABORTING" + echo -e "\n======================================================================\n\n" + + for test in "${failed_tests[@]}" + do + echo -e "\n********** $test KRKN RUN OUTPUT **********\n" + cat "CI/out/$test.out" + echo -e "\n********************************************\n\n\n\n" + done + + exit 1 +fi diff --git a/CI/run_test.sh b/CI/run_test.sh index 90fc999e..7d0d516b 100755 --- a/CI/run_test.sh +++ b/CI/run_test.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x readonly SECONDS_PER_HOUR=3600 readonly SECONDS_PER_MINUTE=60 function get_time_format() { @@ -14,9 +13,7 @@ ci_test=`echo $1` results_file=$2 -echo -e "\n======================================================================" -echo -e " CI test for ${ci_test} " -echo -e "======================================================================\n" +echo -e "test: ${ci_test}" >&2 ci_results="CI/out/$ci_test.out" # Test ci @@ -28,13 +25,16 @@ then # if the test passes update the results and complete duration=$SECONDS duration=$(get_time_format $duration) - echo "$ci_test: Successful" + echo -e "> $ci_test: Successful\n" >&2 echo "$ci_test | Pass | $duration" >> $results_file count=$retries + # return value for run.sh + echo 0 else duration=$SECONDS duration=$(get_time_format $duration) - echo "$ci_test: Failed" + echo -e "> $ci_test: Failed\n" >&2 echo "$ci_test | Fail | $duration" >> $results_file - echo "Logs for "$ci_test + # return value for run.sh + echo 1 fi diff --git a/CI/scenarios/container_scenario.yml b/CI/scenarios/container_scenario.yml index 9cc2a731..d0a6595c 100755 --- a/CI/scenarios/container_scenario.yml +++ b/CI/scenarios/container_scenario.yml @@ -1,8 +1,8 @@ scenarios: -- name: "kill machine config container" - namespace: "openshift-machine-config-operator" - label_selector: "k8s-app=machine-config-server" - container_name: "hello-openshift" - action: "kill 1" +- name: "kill test container" + namespace: "default" + label_selector: "scenario=container" + container_name: "fedtools" + action: 1 count: 1 retry_wait: 60 diff --git a/CI/scenarios/network_chaos.yaml b/CI/scenarios/network_chaos.yaml index df527d02..bfe5a226 100644 --- a/CI/scenarios/network_chaos.yaml +++ b/CI/scenarios/network_chaos.yaml @@ -1,6 +1,8 @@ network_chaos: # Scenario to create an outage by simulating random variations in the network. duration: 10 # seconds instance_count: 1 + node_name: kind-worker2 execution: serial egress: bandwidth: 100mbit + diff --git a/CI/scenarios/network_diagnostics_namespace.yaml b/CI/scenarios/network_diagnostics_namespace.yaml index b8705e62..4277fa16 100644 --- a/CI/scenarios/network_diagnostics_namespace.yaml +++ b/CI/scenarios/network_diagnostics_namespace.yaml @@ -1,6 +1,6 @@ scenarios: - action: delete - namespace: "^$openshift-network-diagnostics$" + namespace: "^namespace-scenario$" label_selector: runs: 1 sleep: 15 diff --git a/CI/scenarios/time_scenarios.yml b/CI/scenarios/time_scenarios.yml index 5572ff77..367af83b 100644 --- a/CI/scenarios/time_scenarios.yml +++ b/CI/scenarios/time_scenarios.yml @@ -1,5 +1,5 @@ time_scenarios: - action: skew_time object_type: pod - label_selector: k8s-app=etcd + label_selector: scenario=time-skew container_name: "" diff --git a/CI/templates/container_scenario_pod.yaml b/CI/templates/container_scenario_pod.yaml new file mode 100644 index 00000000..444ff056 --- /dev/null +++ b/CI/templates/container_scenario_pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: container + labels: + scenario: container +spec: + hostNetwork: true + containers: + - name: fedtools + image: docker.io/fedora/tools + command: + - /bin/sh + - -c + - | + sleep infinity \ No newline at end of file diff --git a/CI/templates/time_pod.yaml b/CI/templates/time_pod.yaml new file mode 100644 index 00000000..14ae2d4f --- /dev/null +++ b/CI/templates/time_pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: time-skew + labels: + scenario: time-skew +spec: + hostNetwork: true + containers: + - name: fedtools + image: docker.io/fedora/tools + command: + - /bin/sh + - -c + - | + sleep infinity \ No newline at end of file diff --git a/CI/tests/common.sh b/CI/tests/common.sh index f25f0550..a9aa2608 100644 --- a/CI/tests/common.sh +++ b/CI/tests/common.sh @@ -13,6 +13,6 @@ function error { } function get_node { - worker_node=$(oc get nodes --no-headers | grep worker | head -n 1) + worker_node=$(kubectl get nodes --no-headers | grep worker | head -n 1) export WORKER_NODE=$worker_node }