Skip to content

Commit

Permalink
tests: Re-enable K8S deploy test in CI
Browse files Browse the repository at this point in the history
  • Loading branch information
@michaljurecko
michaljurecko committed Jun 20, 2024
1 parent c0eff20 commit 4cadbca
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 111 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test-k8s-service-apps-proxy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ defaults:

jobs:
test:
name: test
name: "K8S test: Data Apps Proxy"
runs-on: ubuntu-latest
steps:
- name: Checkout
Expand Down
214 changes: 105 additions & 109 deletions .github/workflows/test-k8s-service-stream.yml
View file
Original file line number Diff line number Diff line change
@@ -1,39 +1,36 @@
name: 'K8S: Buffer Service'
name: 'K8S: Stream Service'

on:
# This workflow is also part of the release pipeline,
# in that case, the actual version is deployed twice.
workflow_call:
# TMP DISABLED
# # In PR the previous version is deployed first and then the actual version.
# pull_request:
# paths:
# - .github/workflows/test-k8s-service-buffer.yml
# - provisioning/stream/**
# - provisioning/common/**
# In PR the previous version is deployed first and then the actual version.
pull_request:
paths:
- .github/workflows/test-k8s-service-stream.yml
- provisioning/stream/**
- provisioning/common/**

env:
MINIKUBE_PROFILE: buffer
MINIKUBE_PROFILE: stream
MINIKUBE_DRIVER: docker

KUBERNETES_NAMESPACE: buffer
KUBERNETES_NAMESPACE: stream
KUBERNETES_ROLLOUT_WAIT: 120s
REMOVE_RESOURCES_LIMITS: true

SERVICE_NAME: Buffer
API_RELEASE_NAME: buffer-api
WORKER_RELEASE_NAME: buffer-worker
ETCD_RELEASE_NAME: buffer-etcd
ETCD_ENDPOINT: buffer-etcd-headless.buffer.svc.cluster.local:2379
SERVICE_NAME: Stream
ETCD_RELEASE_NAME: stream-etcd
ETCD_ENDPOINT: stream-etcd-headless.stream.svc.cluster.local:2379
METRICS_PORT: 9000

defaults:
run:
working-directory: provisioning/buffer
working-directory: provisioning/stream

jobs:
test:
name: test
name: "K8S test: Stream Service"
runs-on: ubuntu-latest
steps:
- name: Checkout
Expand Down Expand Up @@ -61,18 +58,18 @@ jobs:
- name: Set Kubernetes namespace
run: kubectl config set-context --current "--namespace=$KUBERNETES_NAMESPACE"

- name: Checkout BASE branch (or HEAD if it is not a pull request)
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.base.ref }}

- name: Deploy the old version, from the BASE branch
continue-on-error: true
run: ./deploy_local.sh

- name: Dump the old version (for diff)
continue-on-error: true
run: sleep 10 && /tmp/latest-scripts/k8s/dump.sh /tmp/artifacts/test-k8s-state.old.json
# - name: Checkout BASE branch (or HEAD if it is not a pull request)
# uses: actions/checkout@v4
# with:
# ref: ${{ github.event.pull_request.base.ref }}
#
# - name: Deploy the old version, from the BASE branch
# continue-on-error: true
# run: ./deploy_local.sh
#
# - name: Dump the old version (for diff)
# continue-on-error: true
# run: sleep 10 && /tmp/latest-scripts/k8s/dump.sh /tmp/artifacts/test-k8s-state.old.json

- name: Checkout HEAD branch
uses: actions/checkout@v4
Expand Down Expand Up @@ -103,77 +100,68 @@ jobs:

- name: Check deployment of the API nodes
if: always()
run: kubectl rollout status "deployment/$API_RELEASE_NAME" --timeout=10s
run: kubectl rollout status "deployment/stream-api" --timeout=10s

- name: Check deployment of the Worker nodes
- name: Check deployment of the HTTP source nodes
if: always()
run: kubectl rollout status "deployment/$WORKER_RELEASE_NAME" --timeout=10s
run: kubectl rollout status "deployment/stream-http-source" --timeout=10s

- name: Check access to the API metrics from the DataDog Agent
- name: Check deployment of the storage writer/reader nodes
if: always()
run: |
set -Eeuo pipefail
kubectl create namespace datadog || true
export POD_IP=`kubectl get pod -l app=buffer-api -o=jsonpath='{.items[0].status.podIP}'`
echo "Pod IP: $POD_IP"
kubectl run --attach --rm --restart=Never check-api-datadog \
--namespace datadog \
--image docker.io/alpine/curl \
--labels="app=datadog-agent" \
--env="POD_IP=$POD_IP" \
--env="METRICS_PORT=$METRICS_PORT" \
--command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$POD_IP:$METRICS_PORT/metrics" | tail"
run: kubectl rollout status "sts/stream-storage-writer-reader" --timeout=10s

- name: Check forbidden access to the API metrics from other places
- name: Check deployment of the storage coordinator nodes
if: always()
run: |
set -Eeuo pipefail
export POD_IP=`kubectl get pod -l app=buffer-api -o=jsonpath='{.items[0].status.podIP}'`
echo "Pod IP: $POD_IP"
if kubectl run --attach --rm --restart=Never check-api-other \
--image docker.io/alpine/curl \
--env="POD_IP=$POD_IP" \
--env="METRICS_PORT=$METRICS_PORT" \
--command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$POD_IP:$METRICS_PORT/metrics" | tail"; then
echo "The command did not fail, but it should have."
exit 1
else
echo "The command failed, OK."
exit 0
fi
run: kubectl rollout status "deployment/stream-storage-coordinator" --timeout=10s

- name: Check access to the Worker metrics from the DataDog Agent
- name: Check access to the metrics from the DataDog Agent
if: always()
run: |
set -Eeuo pipefail
kubectl create namespace datadog || true
export POD_IP=`kubectl get pod -l app=buffer-worker -o=jsonpath='{.items[0].status.podIP}'`
echo "Pod IP: $POD_IP"
kubectl run --attach --rm --restart=Never check-worker-datadog \
--namespace datadog \
--image docker.io/alpine/curl \
--labels="app=datadog-agent" \
--env="POD_IP=$POD_IP" \
--env="METRICS_PORT=$METRICS_PORT" \
--command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$POD_IP:$METRICS_PORT/metrics" | tail"
for APP_WITH_METRICS_PORT in "stream-api:9000" "stream-http-source:9000" "stream-storage-writer-reader:9001" "stream-storage-writer-reader:9002" "stream-storage-coordinator:9000"
do
IFS=: read -r APP METRICS_PORT <<< "$APP_WITH_METRICS_PORT"
echo "---------------------------------"
echo "Checking $APP:$METRICS_PORT"
export POD_IP=`kubectl get pod -l "app=$APP" -o=jsonpath='{.items[0].status.podIP}'`
echo "Pod IP: $POD_IP"
kubectl run --attach --rm --restart=Never check-api-datadog \
--namespace datadog \
--image docker.io/alpine/curl \
--labels="app=datadog-agent" \
--env="POD_IP=$POD_IP" \
--env="METRICS_PORT=$METRICS_PORT" \
--command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$POD_IP:$METRICS_PORT/metrics" | tail"
echo "---------------------------------"
done
- name: Check forbidden access to the Worker metrics from other places
- name: Check forbidden access to the metrics from other places
if: always()
run: |
set -Eeuo pipefail
export POD_IP=`kubectl get pod -l app=buffer-worker -o=jsonpath='{.items[0].status.podIP}'`
echo "Pod IP: $POD_IP"
if kubectl run --attach --rm --restart=Never check-worker-other \
--image docker.io/alpine/curl \
--env="POD_IP=$POD_IP" \
--env="METRICS_PORT=$METRICS_PORT" \
--command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$POD_IP:$METRICS_PORT/metrics" | tail"; then
echo "The command did not fail, but it should have."
exit 1
else
echo "The command failed, OK."
exit 0
fi
for APP_WITH_METRICS_PORT in "stream-api:9000" "stream-http-source:9000" "stream-storage-writer-reader:9001" "stream-storage-writer-reader:9002" "stream-storage-coordinator:9000"
do
IFS=: read -r APP METRICS_PORT <<< "$APP_WITH_METRICS_PORT"
echo "---------------------------------"
echo "Checking $APP:$METRICS_PORT"
export POD_IP=`kubectl get pod -l "app=$APP" -o=jsonpath='{.items[0].status.podIP}'`
echo "Pod IP: $POD_IP"
if kubectl run --attach --rm --restart=Never check-api-other \
--image docker.io/alpine/curl \
--env="POD_IP=$POD_IP" \
--env="METRICS_PORT=$METRICS_PORT" \
--command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$POD_IP:$METRICS_PORT/metrics" | tail"; then
echo "The command did not fail, but it should have."
exit 1
else
echo "The command failed, OK."
exit 0
fi
echo "---------------------------------"
done
- name: Check access to the etcd from a client
if: always()
Expand Down Expand Up @@ -216,7 +204,15 @@ jobs:
- name: Check API response
if: always()
run: curl --fail -L -s --max-time 5 "$(minikube service --url $API_RELEASE_NAME --namespace $KUBERNETES_NAMESPACE)"
run: |
kubectl port-forward --address 0.0.0.0 --namespace $KUBERNETES_NAMESPACE service/stream-api 30001:80 &
curl --fail -s --max-time 5 --retry 5 --retry-connrefused "localhost:30001/health-check"
- name: Check HTTP source response
if: always()
run: |
kubectl port-forward --address 0.0.0.0 --namespace $KUBERNETES_NAMESPACE service/stream-http-source 30002:80 &
curl --fail -s --max-time 5 --retry 5 --retry-connrefused "localhost:30002/health-check"
- name: Test etcd defragmentation cron job
if: always()
Expand All @@ -235,23 +231,23 @@ jobs:
exit 1
fi
- name: Diff the old and the new Kubernetes state
if: always()
run: |
set -Eeuo pipefail
# Diff JSON states
/tmp/latest-scripts/k8s/diff.sh \
/tmp/artifacts/test-k8s-state.old.json \
/tmp/artifacts/test-k8s-state.new.json \
/tmp/artifacts/test-k8s-state.diff
# Remove ANSI sequences
sed -e 's/\x1b\[[0-9;]*m//g' -i /tmp/artifacts/test-k8s-state.diff || true
# Prepare PR comment message
echo -e "### ${{ env.SERVICE_NAME }} Kubernetes Diff [CI]\n\n" >> /tmp/artifacts/test-k8s-state.diff.message
echo -e "Between \`base\` ${{ github.event.pull_request.base.sha }} :arrow_left: \`head\` ${{ github.event.pull_request.head.sha }}.\n\n" >> /tmp/artifacts/test-k8s-state.diff.message
echo -e "<details>\n<summary>Expand</summary>\n\n\`\`\`diff\n" >> /tmp/artifacts/test-k8s-state.diff.message
head -c 50000 /tmp/artifacts/test-k8s-state.diff >> /tmp/artifacts/test-k8s-state.diff.message
echo -e "\n\n(see artifacts in the Github Action for more information)\n\`\`\`\n</details>" >> /tmp/artifacts/test-k8s-state.diff.message
# - name: Diff the old and the new Kubernetes state
# if: always()
# run: |
# set -Eeuo pipefail
# # Diff JSON states
# /tmp/latest-scripts/k8s/diff.sh \
# /tmp/artifacts/test-k8s-state.old.json \
# /tmp/artifacts/test-k8s-state.new.json \
# /tmp/artifacts/test-k8s-state.diff
# # Remove ANSI sequences
# sed -e 's/\x1b\[[0-9;]*m//g' -i /tmp/artifacts/test-k8s-state.diff || true
# # Prepare PR comment message
# echo -e "### ${{ env.SERVICE_NAME }} Kubernetes Diff [CI]\n\n" >> /tmp/artifacts/test-k8s-state.diff.message
# echo -e "Between \`base\` ${{ github.event.pull_request.base.sha }} :arrow_left: \`head\` ${{ github.event.pull_request.head.sha }}.\n\n" >> /tmp/artifacts/test-k8s-state.diff.message
# echo -e "<details>\n<summary>Expand</summary>\n\n\`\`\`diff\n" >> /tmp/artifacts/test-k8s-state.diff.message
# head -c 50000 /tmp/artifacts/test-k8s-state.diff >> /tmp/artifacts/test-k8s-state.diff.message
# echo -e "\n\n(see artifacts in the Github Action for more information)\n\`\`\`\n</details>" >> /tmp/artifacts/test-k8s-state.diff.message

- name: Dump logs
if: always()
Expand All @@ -267,9 +263,9 @@ jobs:
path: /tmp/artifacts
if-no-files-found: error

- name: Send PR comment
uses: marocchino/sticky-pull-request-comment@v2
with:
header: "${{ env.KUBERNETES_NAMESPACE }}-kubernetes-state-diff"
recreate: true
path: /tmp/artifacts/test-k8s-state.diff.message
# - name: Send PR comment
# uses: marocchino/sticky-pull-request-comment@v2
# with:
# header: "${{ env.KUBERNETES_NAMESPACE }}-kubernetes-state-diff"
# recreate: true
# path: /tmp/artifacts/test-k8s-state.diff.message
2 changes: 1 addition & 1 deletion .github/workflows/test-k8s-service-templates.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ defaults:

jobs:
test:
name: test
name: "K8S test: Templates Service"
runs-on: ubuntu-latest
steps:
- name: Checkout
Expand Down

0 comments on commit 4cadbca

Please sign in to comment.