diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 8b8b43dd95381..33b58d746660f 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -164,6 +164,34 @@ jobs: - uses: actions/checkout@v1 - run: make test-integration-splunk + integration-kubernetes: + runs-on: ubuntu-latest + strategy: + matrix: + kubernetes: + - v1.18.2 + - v1.17.5 + - v1.16.9 + - v1.15.11 + - v1.14.10 + fail-fast: false + steps: + - name: Setup Minikube + uses: manusa/actions-setup-minikube@v1.0.2 + with: + minikube version: 'v1.9.2' + kubernetes version: '${{ matrix.kubernetes }}' + github token: '${{ secrets.GITHUB_TOKEN }}' + - name: Login to DockerHub Registry + run: docker login -u "${{ secrets.K8S_INTEGRATION_CI_DOCKER_USERNAME }}" -p "${{ secrets.K8S_INTEGRATION_CI_DOCKER_PASSWORD }}" + - name: Checkout + uses: actions/checkout@v1 + - run: make slim-builds + - run: make test-integration-kubernetes + env: + CONTAINER_IMAGE_REPO: ${{ secrets.K8S_INTEGRATION_CI_DOCKER_REPO }} + PACKAGE_DEB_USE_CONTAINER: docker + test-unit: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 9b2b3e0692a95..7d65ebb3864ff 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,10 @@ test-integration-pulsar: ## Runs Kafka integration tests test-integration-splunk: ## Runs Kafka integration tests $(RUN) test-integration-splunk +PACKAGE_DEB_USE_CONTAINER ?= "$(USE_CONTAINER)" +test-integration-kubernetes: ## Runs Kubernetes integration tests + PACKAGE_DEB_USE_CONTAINER="$(PACKAGE_DEB_USE_CONTAINER)" USE_CONTAINER=none $(RUN) test-integration-kubernetes + test-unit: ## Runs unit tests, tests which do not require additional services to be present $(RUN) test-unit diff --git a/distribution/kubernetes/README.md b/distribution/kubernetes/README.md new file mode 100644 index 0000000000000..df480c6fedc22 --- /dev/null +++ b/distribution/kubernetes/README.md @@ -0,0 +1,4 @@ +# Kubernetes Deployment Configuration + +Make sure to use the API versions of the Minimal Supported Kubernetes Version +for cluster object definitions, otherwise older cluster will fail. diff --git a/distribution/kubernetes/vector-global.yaml b/distribution/kubernetes/vector-global.yaml new file mode 100644 index 0000000000000..e391bce43829e --- /dev/null +++ b/distribution/kubernetes/vector-global.yaml @@ -0,0 +1,14 @@ +# Permissions to use Kubernetes API. +# Requires that RBAC authorization is enabled. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: vector +subjects: + - kind: ServiceAccount + name: default + namespace: vector +roleRef: + kind: ClusterRole + name: view + apiGroup: rbac.authorization.k8s.io diff --git a/distribution/kubernetes/vector-namespaced.yaml b/distribution/kubernetes/vector-namespaced.yaml new file mode 100644 index 0000000000000..dec678f2bb6a9 --- /dev/null +++ b/distribution/kubernetes/vector-namespaced.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-daemonset-managed-config +data: + managed.toml: | + # Configuration for vector. + # Docs: https://vector.dev/docs/ + + # Configure the controlled by the deployment. + data_dir = "/vector-data-dir" + + # Ingest logs from Kubernetes. + [sources.kubernetes] + type = "kubernetes" +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: vector + labels: + k8s-app: vector +spec: + selector: + matchLabels: + name: vector + template: + metadata: + labels: + name: vector + spec: + containers: + - name: vector + image: timberio/vector:latest-alpine + imagePullPolicy: Always + args: + - --config + - /etc/vector/*.toml + env: + - name: VECTOR_SELF_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: VECTOR_SELF_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VECTOR_SELF_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: var-log + mountPath: /var/log/ + readOnly: true + - name: var-lib + mountPath: /var/lib/ + readOnly: true + - name: data-dir + mountPath: /vector-data-dir/ + - name: config-dir + mountPath: /etc/vector/ + readOnly: true + tolerations: + # This toleration is to have the daemonset runnable on master nodes. + # Remove it if your masters can't run pods. + - key: node-role.kubernetes.io/master + effect: NoSchedule + volumes: + # Log directory. + - name: var-log + hostPath: + path: /var/log/ + # Docker and containerd log files in Kubernetes are symlinks to this folder. + - name: var-lib + hostPath: + path: /var/lib/ + # Vector will store it's data here. + - name: data-dir + hostPath: + path: /var/lib/vector/ + # Vector config files. + - name: config-dir + projected: + sources: + - configMap: + name: vector-daemonset-managed-config + optional: true + - configMap: + name: vector-config + optional: true + - secret: + name: vector-config + optional: true diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index f73098c8c2f55..3dcafed7e8375 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -15,14 +15,18 @@ VERSION="${VERSION:-"$(scripts/version.sh)"}" DATE="${DATE:-"$(date -u +%Y-%m-%d)"}" PUSH="${PUSH:-}" PLATFORM="${PLATFORM:-}" +REPO="${REPO:-"timberio/vector"}" # # Functions # build() { - BASE="$1" - VERSION="$2" + local BASE="$1" + local VERSION="$2" + + local TAG="$REPO:$VERSION-$BASE" + local DOCKERFILE="distribution/docker/$BASE/Dockerfile" if [ -n "$PLATFORM" ]; then export DOCKER_CLI_EXPERIMENTAL=enabled @@ -33,17 +37,17 @@ build() { docker buildx build \ --platform="$PLATFORM" \ - --tag "timberio/vector:$VERSION-$BASE" \ + --tag "$TAG" \ target/artifacts \ - -f "distribution/docker/$BASE/Dockerfile" ${PUSH:+--push} + -f "$DOCKERFILE" ${PUSH:+--push} else docker build \ - --tag "timberio/vector:$VERSION-$BASE" \ + --tag "$TAG" \ target/artifacts \ - -f "distribution/docker/$BASE/Dockerfile" + -f "$DOCKERFILE" if [ -n "$PUSH" ]; then - docker push "timberio/vector:$VERSION-$BASE" + docker push "$TAG" fi fi } @@ -52,7 +56,7 @@ build() { # Build # -echo "Building timberio/vector:* Docker images" +echo "Building $REPO:* Docker images" if [[ "$CHANNEL" == "latest" ]]; then VERSION_EXACT="$VERSION" @@ -70,4 +74,6 @@ elif [[ "$CHANNEL" == "nightly" ]]; then build alpine "$VERSION_TAG" build debian "$VERSION_TAG" done +elif [[ "$CHANNEL" == "test" ]]; then + build "${BASE:-"alpine"}" "${TAG:-"test"}" fi diff --git a/scripts/deploy-kubernetes-test.sh b/scripts/deploy-kubernetes-test.sh new file mode 100755 index 0000000000000..f7e9a73755bc4 --- /dev/null +++ b/scripts/deploy-kubernetes-test.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +# deploy-kubernetes-test.sh +# +# SUMMARY +# +# Deploys Vector into Kubernetes for testing purposes. +# Uses the same installation method our users would use. +# +# This script impements cli interface required by the kubernetes integration +# tests. +# +# USAGE +# +# Deploy: +# +# $ CONTAINER_IMAGE=timberio/vector:alpine-latest scripts/deploy-kubernetes-test.sh up vector-test-qwerty +# +# Teardown: +# +# $ scripts/deploy-kubernetes-test.sh down vector-test-qwerty +# + +# Command to perform. +COMMAND="${1:?"Specify the command (up/down) as the first argument"}" + +# A Kubernetes namespace to deploy to. +NAMESPACE="${2:?"Specify the namespace as the second argument"}" + +# Allow overriding kubectl with somethingl like `minikube kubectl --`. +VECTOR_TEST_KUBECTL="${VECTOR_TEST_KUBECTL:-"kubectl"}" + +# Allow optionally installing custom resource configs. +CUSTOM_RESOURCE_CONIFGS_FILE="${CUSTOM_RESOURCE_CONIFGS_FILE:-""}" + + +# TODO: replace with `helm template | kubectl apply -f -` when Helm Chart is +# available. + +templated-config-global() { + sed "s|^ namespace: vector| namespace: $NAMESPACE|" < "distribution/kubernetes/vector-global.yaml" \ + | sed "s|^ name: vector| name: $NAMESPACE|" +} + +up() { + # A Vector container image to use. + CONTAINER_IMAGE="${CONTAINER_IMAGE:?"You must assing CONTAINER_IMAGE variable with the Vector container image name"}" + + templated-config-global | $VECTOR_TEST_KUBECTL create -f - + + $VECTOR_TEST_KUBECTL create namespace "$NAMESPACE" + + if [[ -n "$CUSTOM_RESOURCE_CONIFGS_FILE" ]]; then + $VECTOR_TEST_KUBECTL create --namespace "$NAMESPACE" -f "$CUSTOM_RESOURCE_CONIFGS_FILE" + fi + + sed "s|timerio/vector:latest|$CONTAINER_IMAGE|" < "distribution/kubernetes/vector-namespaced.yaml" \ + | $VECTOR_TEST_KUBECTL create --namespace "$NAMESPACE" -f - +} + +down() { + $VECTOR_TEST_KUBECTL delete --namespace "$NAMESPACE" -f - < "distribution/kubernetes/vector-namespaced.yaml" + + if [[ -n "$CUSTOM_RESOURCE_CONIFGS_FILE" ]]; then + $VECTOR_TEST_KUBECTL delete --namespace "$NAMESPACE" -f "$CUSTOM_RESOURCE_CONIFGS_FILE" + fi + + $VECTOR_TEST_KUBECTL delete namespace "$NAMESPACE" + + templated-config-global | $VECTOR_TEST_KUBECTL delete -f - +} + +case "$COMMAND" in + up|down) + "$COMMAND" "$@" + ;; + *) + echo "Invalid command: $COMMAND" >&2 + exit 1 +esac diff --git a/scripts/start-docker-registry.sh b/scripts/start-docker-registry.sh new file mode 100755 index 0000000000000..19daa3f23ae17 --- /dev/null +++ b/scripts/start-docker-registry.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +# start-docker-registry.sh +# +# SUMMARY +# +# Starts a Docker Distribution instance in docker and prints it's IP address +# to the stdout. +# +# Useful for conducting tests involing Vector docker images. +# + +CONTAINER_NAME="${1:-"vector-docker-registry"}" +PORT="${2:-"5000"}" + +IS_ALREADY_RUNNING="$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || true)" +if [ "${IS_ALREADY_RUNNING}" != 'true' ]; then + docker run \ + -d \ + --restart=always \ + -p "$PORT:5000" \ + --name "$CONTAINER_NAME" \ + registry:2 > /dev/null +fi + +IP_ADDRESS="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "$CONTAINER_NAME")" +echo "$IP_ADDRESS:$PORT" diff --git a/scripts/test-integration-kubernetes.sh b/scripts/test-integration-kubernetes.sh new file mode 100755 index 0000000000000..37ce65c891fac --- /dev/null +++ b/scripts/test-integration-kubernetes.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +# test-integration-kubernetes.sh +# +# SUMMARY +# +# Run integration tests for Kubernetes components only. + +random-string() { + local CHARS="abcdefghijklmnopqrstuvwxyz0123456789" + # shellcheck disable=SC2034 + for i in {1..8}; do + echo -n "${CHARS:RANDOM%${#CHARS}:1}" + done + echo +} + +# Require a repo to put image at. +# +# Hint: you can use `scripts/start-docker-registry.sh`, but it requires +# manually preparing the environment to allow insecure registries, and it can +# also not work if you k8s cluster doesn't have network connectivity to the +# registry. +CONTAINER_IMAGE_REPO="${CONTAINER_IMAGE_REPO:?"You have to specify CONTAINER_IMAGE_REPO to upload the test image to."}" + +# Assign a default test run ID if none is provided by the user. +TEST_RUN_ID="${TEST_RUN_ID:-"test-$(date +%s)-$(random-string)"}" + +if [[ -z "${SKIP_PACKAGE_DEB:-}" ]]; then + make package-deb-x86_64 USE_CONTAINER="${PACKAGE_DEB_USE_CONTAINER:-"docker"}" +fi + +# Build docker image with Vector - the same way it's done for releses - and push +# it into the docker configured repo. +REPO="$CONTAINER_IMAGE_REPO" CHANNEL="test" TAG="$TEST_RUN_ID" PUSH=1 scripts/build-docker.sh + +# Set the deployment command for integration tests. +export KUBE_TEST_DEPLOY_COMMAND="scripts/deploy-kubernetes-test.sh" + +# Configure the deploy command to use our repo file. +export CONTAINER_IMAGE="$CONTAINER_IMAGE_REPO:$TEST_RUN_ID" + +# TODO: enable kubernetes tests when they're implemented +exit 0 # disable the test and make them pass + +# Run the tests. +cargo test --no-default-features --features kubernetes-integration-tests