chore(operations): Kubernetes test procedure

.github/workflows/code.yml

@@ -164,6 +164,34 @@ jobs:
       - uses: actions/checkout@v1
       - run: make test-integration-splunk
 
+  integration-kubernetes:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        kubernetes:
+          - v1.18.2
+          - v1.17.5
+          - v1.16.9
+          - v1.15.11
+          - v1.14.10
+      fail-fast: false
+    steps:
+      - name: Setup Minikube
+        uses: manusa/[email protected]
+        with:
+          minikube version: 'v1.9.2'
+          kubernetes version: '${{ matrix.kubernetes }}'
+          github token: '${{ secrets.GITHUB_TOKEN }}'
+      - name: Login to DockerHub Registry
+        run: docker login -u "${{ secrets.K8S_INTEGRATION_CI_DOCKER_USERNAME }}" -p "${{ secrets.K8S_INTEGRATION_CI_DOCKER_PASSWORD }}"
+      - name: Checkout
+        uses: actions/checkout@v1
+      - run: make slim-builds
+      - run: make test-integration-kubernetes
+        env:
+          CONTAINER_IMAGE_REPO: ${{ secrets.K8S_INTEGRATION_CI_DOCKER_REPO }}
+          PACKAGE_DEB_USE_CONTAINER: docker
+
   test-unit:
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -88,6 +88,10 @@ test-integration-pulsar: ## Runs Kafka integration tests
 test-integration-splunk: ## Runs Kafka integration tests
 	$(RUN) test-integration-splunk
 
+PACKAGE_DEB_USE_CONTAINER ?= "$(USE_CONTAINER)"
+test-integration-kubernetes: ## Runs Kubernetes integration tests
+	PACKAGE_DEB_USE_CONTAINER="$(PACKAGE_DEB_USE_CONTAINER)" USE_CONTAINER=none $(RUN) test-integration-kubernetes
+
 test-unit: ## Runs unit tests, tests which do not require additional services to be present
 	$(RUN) test-unit
 

distribution/kubernetes/README.md

@@ -0,0 +1,4 @@
+# Kubernetes Deployment Configuration
+
+Make sure to use the API versions of the Minimal Supported Kubernetes Version
+for cluster object definitions, otherwise older cluster will fail.

distribution/kubernetes/vector-global.yaml

@@ -0,0 +1,14 @@
+# Permissions to use Kubernetes API.
+# Requires that RBAC authorization is enabled.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: vector
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: vector
+roleRef:
+  kind: ClusterRole
+  name: view
+  apiGroup: rbac.authorization.k8s.io

distribution/kubernetes/vector-namespaced.yaml

@@ -0,0 +1,94 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-daemonset-managed-config
+data:
+  managed.toml: |
+    # Configuration for vector.
+    # Docs: https://vector.dev/docs/
+
+    # Configure the controlled by the deployment.
+    data_dir = "/vector-data-dir"
+
+    # Ingest logs from Kubernetes.
+    [sources.kubernetes]
+        type = "kubernetes"
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: vector
+  labels:
+    k8s-app: vector
+spec:
+  selector:
+    matchLabels:
+      name: vector
+  template:
+    metadata:
+      labels:
+        name: vector
+    spec:
+      containers:
+        - name: vector
+          image: timberio/vector:latest-alpine
+          imagePullPolicy: Always
+          args:
+            - --config
+            - /etc/vector/*.toml
+          env:
+            - name: VECTOR_SELF_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: VECTOR_SELF_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: VECTOR_SELF_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          volumeMounts:
+            - name: var-log
+              mountPath: /var/log/
+              readOnly: true
+            - name: var-lib
+              mountPath: /var/lib/
+              readOnly: true
+            - name: data-dir
+              mountPath: /vector-data-dir/
+            - name: config-dir
+              mountPath: /etc/vector/
+              readOnly: true
+      tolerations:
+        # This toleration is to have the daemonset runnable on master nodes.
+        # Remove it if your masters can't run pods.
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      volumes:
+        # Log directory.
+        - name: var-log
+          hostPath:
+            path: /var/log/
+        # Docker and containerd log files in Kubernetes are symlinks to this folder.
+        - name: var-lib
+          hostPath:
+            path: /var/lib/
+        # Vector will store it's data here.
+        - name: data-dir
+          hostPath:
+            path: /var/lib/vector/
+        # Vector config files.
+        - name: config-dir
+          projected:
+            sources:
+              - configMap:
+                  name: vector-daemonset-managed-config
+                  optional: true
+              - configMap:
+                  name: vector-config
+                  optional: true
+              - secret:
+                  name: vector-config
+                  optional: true

scripts/build-docker.sh

@@ -15,14 +15,18 @@ VERSION="${VERSION:-"$(scripts/version.sh)"}"
 DATE="${DATE:-"$(date -u +%Y-%m-%d)"}"
 PUSH="${PUSH:-}"
 PLATFORM="${PLATFORM:-}"
+REPO="${REPO:-"timberio/vector"}"
 
 #
 # Functions
 #
 
 build() {
-  BASE="$1"
-  VERSION="$2"
+  local BASE="$1"
+  local VERSION="$2"
+
+  local TAG="$REPO:$VERSION-$BASE"
+  local DOCKERFILE="distribution/docker/$BASE/Dockerfile"
 
   if [ -n "$PLATFORM" ]; then
     export DOCKER_CLI_EXPERIMENTAL=enabled
@@ -33,17 +37,17 @@ build() {
 
     docker buildx build \
       --platform="$PLATFORM" \
-      --tag "timberio/vector:$VERSION-$BASE" \
+      --tag "$TAG" \
       target/artifacts \
-      -f "distribution/docker/$BASE/Dockerfile" ${PUSH:+--push}
+      -f "$DOCKERFILE" ${PUSH:+--push}
   else
     docker build \
-      --tag "timberio/vector:$VERSION-$BASE" \
+      --tag "$TAG" \
       target/artifacts \
-      -f "distribution/docker/$BASE/Dockerfile"
+      -f "$DOCKERFILE"
 
     if [ -n "$PUSH" ]; then
-      docker push "timberio/vector:$VERSION-$BASE"
+      docker push "$TAG"
     fi
   fi
 }
@@ -52,7 +56,7 @@ build() {
 # Build
 #
 
-echo "Building timberio/vector:* Docker images"
+echo "Building $REPO:* Docker images"
 
 if [[ "$CHANNEL" == "latest" ]]; then
   VERSION_EXACT="$VERSION"
@@ -70,4 +74,6 @@ elif [[ "$CHANNEL" == "nightly" ]]; then
     build alpine "$VERSION_TAG"
     build debian "$VERSION_TAG"
   done
+elif [[ "$CHANNEL" == "test" ]]; then
+  build "${BASE:-"alpine"}" "${TAG:-"test"}"
 fi

scripts/deploy-kubernetes-test.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# deploy-kubernetes-test.sh
+#
+# SUMMARY
+#
+#   Deploys Vector into Kubernetes for testing purposes.
+#   Uses the same installation method our users would use.
+#
+#   This script impements cli interface required by the kubernetes integration
+#   tests.
+#
+# USAGE
+#
+#   Deploy:
+#
+#   $ CONTAINER_IMAGE=timberio/vector:alpine-latest scripts/deploy-kubernetes-test.sh up vector-test-qwerty
+#
+#   Teardown:
+#
+#   $ scripts/deploy-kubernetes-test.sh down vector-test-qwerty
+#
+
+# Command to perform.
+COMMAND="${1:?"Specify the command (up/down) as the first argument"}"
+
+# A Kubernetes namespace to deploy to.
+NAMESPACE="${2:?"Specify the namespace as the second argument"}"
+
+# Allow overriding kubectl with somethingl like `minikube kubectl --`.
+VECTOR_TEST_KUBECTL="${VECTOR_TEST_KUBECTL:-"kubectl"}"
+
+# Allow optionally installing custom resource configs.
+CUSTOM_RESOURCE_CONIFGS_FILE="${CUSTOM_RESOURCE_CONIFGS_FILE:-""}"
+
+
+# TODO: replace with `helm template | kubectl apply -f -` when Helm Chart is
+# available.
+
+templated-config-global() {
+  sed "s|^    namespace: vector|    namespace: $NAMESPACE|" < "distribution/kubernetes/vector-global.yaml" \
+    | sed "s|^  name: vector|  name: $NAMESPACE|"
+}
+
+up() {
+  # A Vector container image to use.
+  CONTAINER_IMAGE="${CONTAINER_IMAGE:?"You must assing CONTAINER_IMAGE variable with the Vector container image name"}"
+
+  templated-config-global | $VECTOR_TEST_KUBECTL create -f -
+
+  $VECTOR_TEST_KUBECTL create namespace "$NAMESPACE"
+
+  if [[ -n "$CUSTOM_RESOURCE_CONIFGS_FILE" ]]; then
+    $VECTOR_TEST_KUBECTL create --namespace "$NAMESPACE" -f "$CUSTOM_RESOURCE_CONIFGS_FILE"
+  fi
+
+  sed "s|timerio/vector:latest|$CONTAINER_IMAGE|" < "distribution/kubernetes/vector-namespaced.yaml" \
+    | $VECTOR_TEST_KUBECTL create --namespace "$NAMESPACE" -f -
+}
+
+down() {
+  $VECTOR_TEST_KUBECTL delete --namespace "$NAMESPACE" -f - < "distribution/kubernetes/vector-namespaced.yaml"
+
+  if [[ -n "$CUSTOM_RESOURCE_CONIFGS_FILE" ]]; then
+    $VECTOR_TEST_KUBECTL delete --namespace "$NAMESPACE" -f "$CUSTOM_RESOURCE_CONIFGS_FILE"
+  fi
+
+  $VECTOR_TEST_KUBECTL delete namespace "$NAMESPACE"
+
+  templated-config-global | $VECTOR_TEST_KUBECTL delete -f -
+}
+
+case "$COMMAND" in
+  up|down)
+    "$COMMAND" "$@"
+    ;;
+  *)
+    echo "Invalid command: $COMMAND" >&2
+    exit 1
+esac

scripts/start-docker-registry.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# start-docker-registry.sh
+#
+# SUMMARY
+#
+#   Starts a Docker Distribution instance in docker and prints it's IP address
+#   to the stdout.
+#
+#   Useful for conducting tests involing Vector docker images.
+#
+
+CONTAINER_NAME="${1:-"vector-docker-registry"}"
+PORT="${2:-"5000"}"
+
+IS_ALREADY_RUNNING="$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || true)"
+if [ "${IS_ALREADY_RUNNING}" != 'true' ]; then
+  docker run \
+    -d \
+    --restart=always \
+    -p "$PORT:5000" \
+    --name "$CONTAINER_NAME" \
+    registry:2 > /dev/null
+fi
+
+IP_ADDRESS="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "$CONTAINER_NAME")"
+echo "$IP_ADDRESS:$PORT"

scripts/test-integration-kubernetes.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# test-integration-kubernetes.sh
+#
+# SUMMARY
+#
+#   Run integration tests for Kubernetes components only.
+
+random-string() {
+  local CHARS="abcdefghijklmnopqrstuvwxyz0123456789"
+  # shellcheck disable=SC2034
+  for i in {1..8}; do
+    echo -n "${CHARS:RANDOM%${#CHARS}:1}"
+  done
+  echo
+}
+
+# Require a repo to put image at.
+#
+# Hint: you can use `scripts/start-docker-registry.sh`, but it requires
+# manually preparing the environment to allow insecure registries, and it can
+# also not work if you k8s cluster doesn't have network connectivity to the
+# registry.
+CONTAINER_IMAGE_REPO="${CONTAINER_IMAGE_REPO:?"You have to specify CONTAINER_IMAGE_REPO to upload the test image to."}"
+
+# Assign a default test run ID if none is provided by the user.
+TEST_RUN_ID="${TEST_RUN_ID:-"test-$(date +%s)-$(random-string)"}"
+
+if [[ -z "${SKIP_PACKAGE_DEB:-}" ]]; then
+  make package-deb-x86_64 USE_CONTAINER="${PACKAGE_DEB_USE_CONTAINER:-"docker"}"
+fi
+
+# Build docker image with Vector - the same way it's done for releses - and push
+# it into the docker configured repo.
+REPO="$CONTAINER_IMAGE_REPO" CHANNEL="test" TAG="$TEST_RUN_ID" PUSH=1 scripts/build-docker.sh
+
+# Set the deployment command for integration tests.
+export KUBE_TEST_DEPLOY_COMMAND="scripts/deploy-kubernetes-test.sh"
+
+# Configure the deploy command to use our repo file.
+export CONTAINER_IMAGE="$CONTAINER_IMAGE_REPO:$TEST_RUN_ID"
+
+# TODO: enable kubernetes tests when they're implemented
+exit 0 # disable the test and make them pass
+
+# Run the tests.
+cargo test --no-default-features --features kubernetes-integration-tests