From fdd1c250c85252d50fa6b751107c3ba45d0ffc7c Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 26 Mar 2024 15:29:09 +0900 Subject: [PATCH 1/2] External EP cicd added --- cicd/k3s-ext-ep/Vagrantfile | 36 +++++++++ cicd/k3s-ext-ep/config.sh | 3 + cicd/k3s-ext-ep/ext-tcp.yml | 21 +++++ cicd/k3s-ext-ep/host.sh | 10 +++ cicd/k3s-ext-ep/k3s.yaml | 19 +++++ cicd/k3s-ext-ep/kube-loxilb.yml | 134 ++++++++++++++++++++++++++++++++ cicd/k3s-ext-ep/master.sh | 54 +++++++++++++ cicd/k3s-ext-ep/nginx.yml | 26 +++++++ cicd/k3s-ext-ep/rmconfig.sh | 5 ++ cicd/k3s-ext-ep/validation.sh | 65 ++++++++++++++++ cicd/k3s-ext-ep/wait_ready.sh | 37 +++++++++ 11 files changed, 410 insertions(+) create mode 100644 cicd/k3s-ext-ep/Vagrantfile create mode 100755 cicd/k3s-ext-ep/config.sh create mode 100644 cicd/k3s-ext-ep/ext-tcp.yml create mode 100755 cicd/k3s-ext-ep/host.sh create mode 100644 cicd/k3s-ext-ep/k3s.yaml create mode 100644 cicd/k3s-ext-ep/kube-loxilb.yml create mode 100644 cicd/k3s-ext-ep/master.sh create mode 100644 cicd/k3s-ext-ep/nginx.yml create mode 100755 cicd/k3s-ext-ep/rmconfig.sh create mode 100755 cicd/k3s-ext-ep/validation.sh create mode 100755 cicd/k3s-ext-ep/wait_ready.sh diff --git a/cicd/k3s-ext-ep/Vagrantfile b/cicd/k3s-ext-ep/Vagrantfile new file mode 100644 index 00000000..f80b8035 --- /dev/null +++ b/cicd/k3s-ext-ep/Vagrantfile @@ -0,0 +1,36 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +workers = (ENV['WORKERS'] || "2").to_i +#box_name = (ENV['VAGRANT_BOX'] || "ubuntu/focal64") +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = "0.7.1" +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + config.vm.define "host" do |host| + host.vm.hostname = 'host' + host.vm.network :private_network, ip: "192.168.82.2", :netmask => "255.255.255.0" + host.vm.provision :shell, :path => "host.sh" + host.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 2048] + vbox.customize ["modifyvm", :id, "--cpus", 2] + end + end + + config.vm.define "master" do |master| + master.vm.hostname = 'master' + master.vm.network :private_network, ip: "192.168.82.128", :netmask => "255.255.255.0" + master.vm.provision :shell, :path => "master.sh" + master.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 8192] + vbox.customize ["modifyvm", :id, "--cpus", 8] + vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + end + end +end diff --git a/cicd/k3s-ext-ep/config.sh b/cicd/k3s-ext-ep/config.sh new file mode 100755 index 00000000..6b8ee48e --- /dev/null +++ b/cicd/k3s-ext-ep/config.sh @@ -0,0 +1,3 @@ +#!/bin/bash +vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f +vagrant up diff --git a/cicd/k3s-ext-ep/ext-tcp.yml b/cicd/k3s-ext-ep/ext-tcp.yml new file mode 100644 index 00000000..57e49990 --- /dev/null +++ b/cicd/k3s-ext-ep/ext-tcp.yml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: ext-tcp-lb +spec: + loadBalancerClass: loxilb.io/loxilb + type: LoadBalancer + ports: + - protocol: TCP + port: 8000 + targetPort: 80 +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: ext-tcp-lb +subsets: + - addresses: + - ip: 192.168.82.2 + ports: + - port: 80 diff --git a/cicd/k3s-ext-ep/host.sh b/cicd/k3s-ext-ep/host.sh new file mode 100755 index 00000000..9ffa2ea0 --- /dev/null +++ b/cicd/k3s-ext-ep/host.sh @@ -0,0 +1,10 @@ +echo "20.20.20.1 k8s-svc" >> /etc/hosts +apt-get update +apt-get install -y software-properties-common lksctp-tools +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce +docker run --cap-add SYS_ADMIN -dit --net=host --name tcp_ep ghcr.io/loxilb-io/nginx:stable +sudo ip route add 20.20.20.1 via 192.168.82.100 +echo "Host is up" diff --git a/cicd/k3s-ext-ep/k3s.yaml b/cicd/k3s-ext-ep/k3s.yaml new file mode 100644 index 00000000..aa22116a --- /dev/null +++ b/cicd/k3s-ext-ep/k3s.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTVRFME16SXpNRGN3SGhjTk1qUXdNekkyTURVMU1UUTNXaGNOTXpRd016STBNRFUxTVRRMwpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTVRFME16SXpNRGN3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTK296V2p3aEJRcHZrSDVCaXJFSVdnRnhNRGR0OFZsaXZLM1Y5Ym56T1EKQmdnUXVkQWxlQ0FYUmk4RjdhWU9ISXUwaGZPbkthRi84cnJTZnlvdWJrSkdvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXYyT0xtZFo5TDFublNTTjExbUhCCmxQWmxjbGt3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnU05IbXNMakh5bkdMRTRFY04yVjVtZFNJaWRNdHNVUTcKSHIwbi9BdUg1NUVDSUVjSGExY3c1RGdOeEZvaDRCWWpQRGdycDIwam9DTVZ6d2ZXRGNzdVhLTUQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + server: https://192.168.82.128:6443 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRlZ0F3SUJBZ0lJV3BvZGdxci9Bb1V3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOekV4TkRNeU16QTNNQjRYRFRJME1ETXlOakExTlRFME4xb1hEVEkxTURNeQpOakExTlRFME4xb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKU1gyRTdwbHNmM0RlM0cKWHZZclJNYVlKaWJmNGpoTlhYSXRYbWEwN01ZV3VXLzB3N1BVTlcyRW9KSVJKdUd0MHVuK0dmbmtvaUd3aG5udwpUWHBJNVlXalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCU1RiQ0l6U0ZGaGdNTklnZitrcHNJMU5Ma2JlekFLQmdncWhrak9QUVFEQWdOSkFEQkcKQWlFQTJBMjlDNktDTDBYazVMZFBXTjBJLzA3cWZjNEpFd3k0OFJrd0QvYTBER29DSVFDeEYvQkMyTjYrTVVWQQowUFNyZTloeFA4WkZpa3Mzd3pCbnFxU0VTbUIrMGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZGpDQ0FSMmdBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwClpXNTBMV05oUURFM01URTBNekl6TURjd0hoY05NalF3TXpJMk1EVTFNVFEzV2hjTk16UXdNekkwTURVMU1UUTMKV2pBak1TRXdId1lEVlFRRERCaHJNM010WTJ4cFpXNTBMV05oUURFM01URTBNekl6TURjd1dUQVRCZ2NxaGtqTwpQUUlCQmdncWhrak9QUU1CQndOQ0FBUlVDY0t3TzJIUEJSNXNBcWUvamx1VC9rc25iSWVVQUtsMWpmbDZLbThjCnBwbm93LzNXNkdGbVhBekErRHpCVmtaODBYU0IzN3J0UUl0cHB3alNrV0hvbzBJd1FEQU9CZ05WSFE4QkFmOEUKQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVrMndpTTBoUllZRERTSUgvcEtiQwpOVFM1RzNzd0NnWUlLb1pJemowRUF3SURSd0F3UkFJZ0djeTQ3dmhJT3lqdHgvMjkva2VTV2p5OFUrd3pmN1ROCkhraWhER21aS3pBQ0lGdEp0YnlGTmU0enRrSHZQRDdBKzcrUDYwaWZ3eGhDUGJlQW81Ny9UYXJnCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUxkOVk0WUl3bjhyZzZ4NFAxb1ZZWE83dEZqRm8ydGRqZGp2bVhSMkhlL0VvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFbEpmWVR1bVd4L2NON2NaZTlpdEV4cGdtSnQvaU9FMWRjaTFlWnJUc3hoYTViL1REczlRMQpiWVNna2hFbTRhM1M2ZjRaK2VTaUliQ0dlZkJOZWtqbGhRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/cicd/k3s-ext-ep/kube-loxilb.yml b/cicd/k3s-ext-ep/kube-loxilb.yml new file mode 100644 index 00000000..0a62bfc1 --- /dev/null +++ b/cicd/k3s-ext-ep/kube-loxilb.yml @@ -0,0 +1,134 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + - --loxiURL=http://172.17.0.2:11111 + - --externalCIDR=20.20.20.1/32 + #- --externalSecondaryCIDRs=124.124.124.1/24,125.125.125.1/24 + #- --monitor + #- --setBGP=64511 + #- --extBGPPeers=50.50.50.1:65101,51.51.51.1:65102 + #- --setRoles=0.0.0.0 + #- --monitor + #- --setBGP + - --setLBMode=2 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/k3s-ext-ep/master.sh b/cicd/k3s-ext-ep/master.sh new file mode 100644 index 00000000..a0db823c --- /dev/null +++ b/cicd/k3s-ext-ep/master.sh @@ -0,0 +1,54 @@ +export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.82' | awk '{print $2}' | cut -f1 -d '/') + +apt-get update +apt-get install -y software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce + +## Set promisc mode for mac-vlan to work +sudo ifconfig eth1 promisc + +sudo docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged --entrypoint /root/loxilb-io/loxilb/loxilb -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest + +# Create mac-vlan on top of underlying eth1 interface +docker network create -d macvlan -o parent=eth1 --subnet 192.168.82.0/24 --gateway 192.168.82.1 --aux-address 'host=192.168.82.252' llbnet + +# Assign mac-vlan to loxilb docker with specified IP (which will be used as LB VIP) +docker network connect llbnet loxilb --ip=192.168.82.100 + +# Add iptables rule to allow traffic from source IP(192.168.82.1) to loxilb +sudo iptables -A DOCKER -s 192.168.82.1 -j ACCEPT + + +#K3s installation +curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik --disable servicelb --disable-cloud-controller \ +--flannel-backend=none \ +--disable-network-policy" sh - + +#Install Cilium +CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) +CLI_ARCH=amd64 +if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi +curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} +sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum +sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin +rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} +mkdir -p ~/.kube/ +sudo cat /etc/rancher/k3s/k3s.yaml > ~/.kube/config +cilium install + +echo $MASTER_IP > /vagrant/master-ip +sudo cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token +sudo cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml +sudo sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /vagrant/k3s.yaml +\ +#Add route for service IP towards loxilb +sudo ip route add 20.20.20.1/32 via 172.17.0.2 + +/vagrant/wait_ready.sh +sudo kubectl apply -f /vagrant/kube-loxilb.yml +sudo kubectl apply -f /vagrant/nginx.yml +sudo kubectl apply -f /vagrant/ext-tcp.yml +/vagrant/wait_ready.sh diff --git a/cicd/k3s-ext-ep/nginx.yml b/cicd/k3s-ext-ep/nginx.yml new file mode 100644 index 00000000..5bcf72fd --- /dev/null +++ b/cicd/k3s-ext-ep/nginx.yml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-lb1 +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: nginx-test + ports: + - port: 55002 + targetPort: 80 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx-test + labels: + what: nginx-test +spec: + containers: + - name: nginx-test + image: nginx:stable + ports: + - containerPort: 80 diff --git a/cicd/k3s-ext-ep/rmconfig.sh b/cicd/k3s-ext-ep/rmconfig.sh new file mode 100755 index 00000000..7f1bb180 --- /dev/null +++ b/cicd/k3s-ext-ep/rmconfig.sh @@ -0,0 +1,5 @@ +#!/bin/bash +vagrant destroy -f master +vagrant destroy -f host +rm master-ip node-token extIP + diff --git a/cicd/k3s-ext-ep/validation.sh b/cicd/k3s-ext-ep/validation.sh new file mode 100755 index 00000000..ab72154a --- /dev/null +++ b/cicd/k3s-ext-ep/validation.sh @@ -0,0 +1,65 @@ +#!/bin/bash +source ../common.sh +echo k3s-ext-ip + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +set -eo pipefail +# Set space as the delimiter +IFS=' ' + +for((i=0; i<120; i++)) +do + extLB=$(vagrant ssh master -c 'sudo kubectl get svc' 2> /dev/null | grep "nginx") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find tcp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +#sleep 30 + +echo Service IP : $extIP +echo $extIP > extIP + +echo -e "\nEnd Points List" +echo "******************************************************************************" +vagrant ssh master -c 'sudo kubectl get endpoints -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nSVC List" +echo "******************************************************************************" +vagrant ssh master -c 'sudo kubectl get svc' 2> /dev/null +echo "******************************************************************************" +echo -e "\nPod List" +echo "******************************************************************************" +vagrant ssh master -c 'sudo kubectl get pods -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nLB List" +echo "******************************************************************************" +vagrant ssh master -c 'sudo sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nEP List" +echo "******************************************************************************" +vagrant ssh master -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" + +echo -e "\nTEST RESULTS" +echo "******************************************************************************" + +echo -e "\n\nCommand: curl --connect-time 10 http://20.20.20.1:55002'\n\n" +vagrant ssh host -c 'curl --connect-time 10 http://20.20.20.1:55002' 2> /dev/null +echo -e "\n\n\nConnecting external EP service from the pod\n\n" +echo "sudo kubectl exec -it nginx-test -- curl 20.20.20.1:8000\n" +vagrant ssh master -c 'sudo kubectl exec -it nginx-test -- curl 20.20.20.1:8000' 2> /dev/null diff --git a/cicd/k3s-ext-ep/wait_ready.sh b/cicd/k3s-ext-ep/wait_ready.sh new file mode 100755 index 00000000..5ff06e37 --- /dev/null +++ b/cicd/k3s-ext-ep/wait_ready.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +function wait_cluster_ready { + Res=$(sudo kubectl get pods -A | + while IFS= read -r line; do + if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then + echo "not ready" + return + fi + done) + if [[ $Res == *"not ready"* ]]; then + return 1 + fi + return 0 +} + +function wait_cluster_ready_full { + i=1 + nr=0 + for ((;;)) do + wait_cluster_ready + nr=$? + if [[ $nr == 0 ]]; then + echo "Cluster is ready" + break + fi + i=$(( $i + 1 )) + if [[ $i -ge 40 ]]; then + echo "Cluster is not ready.Giving up" + exit 1 + fi + echo "Cluster is not ready...." + sleep 10 + done +} + +wait_cluster_ready_full From 5fa620977dd0c27dcc1eb3ddf76c61730d5bf14d Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 26 Mar 2024 15:32:05 +0900 Subject: [PATCH 2/2] sctp test added in k3s-base-sanity --- .github/workflows/k3s-base-sanity.yml | 2 +- cicd/k3s-base-sanity/config.sh | 3 +- cicd/k3s-base-sanity/input | 6 ++++ cicd/k3s-base-sanity/sctp-svc-lb.yml | 33 +++++++++++++++++++ cicd/k3s-base-sanity/validation.sh | 47 ++++++++++++++++++++++++--- 5 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 cicd/k3s-base-sanity/input create mode 100644 cicd/k3s-base-sanity/sctp-svc-lb.yml diff --git a/.github/workflows/k3s-base-sanity.yml b/.github/workflows/k3s-base-sanity.yml index 5070c9f6..cde42293 100644 --- a/.github/workflows/k3s-base-sanity.yml +++ b/.github/workflows/k3s-base-sanity.yml @@ -26,7 +26,7 @@ jobs: with: go-version: '>=1.18.0' - run: sudo apt-get update - - run: sudo apt-get -y install bridge-utils net-tools iperf curl wget clang-10 llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iperf iproute2 nodejs socat ethtool + - run: sudo apt-get -y install bridge-utils net-tools iperf curl lksctp-tools wget clang-10 llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iperf iproute2 nodejs socat ethtool - run: docker pull ghcr.io/loxilb-io/loxilb:latest - run: docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest - run: pwd && ls && sudo mkdir /opt/loxilb/ && sudo -E env "PATH=$PATH" make docker-cp diff --git a/cicd/k3s-base-sanity/config.sh b/cicd/k3s-base-sanity/config.sh index 17be6f4f..430fa8d1 100755 --- a/cicd/k3s-base-sanity/config.sh +++ b/cicd/k3s-base-sanity/config.sh @@ -179,7 +179,8 @@ kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml sleep 10 kubectl $KUBECONFIG apply -f udp-svc-lb.yml sleep 30 - +kubectl $KUBECONFIG apply -f sctp-svc-lb.yml +sleep 10 # External LB service must be created by now kubectl $KUBECONFIG get svc diff --git a/cicd/k3s-base-sanity/input b/cicd/k3s-base-sanity/input new file mode 100644 index 00000000..6fb66a5e --- /dev/null +++ b/cicd/k3s-base-sanity/input @@ -0,0 +1,6 @@ + + + + + + diff --git a/cicd/k3s-base-sanity/sctp-svc-lb.yml b/cicd/k3s-base-sanity/sctp-svc-lb.yml new file mode 100644 index 00000000..7af84843 --- /dev/null +++ b/cicd/k3s-base-sanity/sctp-svc-lb.yml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Service +metadata: + name: sctp-lb1 + annotations: + loxilb.io/lbmode: "fullnat" + loxilb.io/liveness: "yes" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: sctp-test + ports: + - port: 55003 + protocol: SCTP + targetPort: 9999 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: sctp-test + labels: + what: sctp-test +spec: + containers: + - name: sctp-test + image: loxilbio/sctp-darn:latest + imagePullPolicy: Always + #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"] + command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"] + ports: + - containerPort: 9999 diff --git a/cicd/k3s-base-sanity/validation.sh b/cicd/k3s-base-sanity/validation.sh index e742cae4..b4a09769 100755 --- a/cicd/k3s-base-sanity/validation.sh +++ b/cicd/k3s-base-sanity/validation.sh @@ -71,9 +71,9 @@ echo "********************" out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) if [[ ${out} == *"Welcome to nginx"* ]]; then - echo "cluster-k3s (kube-loxilb) tcp [OK]" + echo "cluster-k3s (kube-loxilb) tcp [OK]" else - echo "cluster-k3s (kube-loxilb) tcp [FAILED]" + echo "cluster-k3s (kube-loxilb) tcp [FAILED]" code=1 fi @@ -97,17 +97,54 @@ done out=$($hexec user timeout 30 ../common/udp_client $extIP 55003) if [[ ${out} == *"Client"* ]]; then - echo "cluster-k3s (kube-loxilb) udp [OK]" + echo "cluster-k3s (kube-loxilb) udp [OK]" else - echo "cluster-k3s (kube-loxilb) udp [FAILED]" + echo "cluster-k3s (kube-loxilb) udp [FAILED]" code=1 fi +for((i=0; i<120; i++)) +do + extLB=$(sudo kubectl $KUBECONFIG get svc | grep "sctp-lb") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find sctp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +$hexec user timeout 5 stdbuf -oL sctp_darn -H 0.0.0.0 -h $extIP -p 55003 -s < input > output +sleep 5 +exp="New connection, peer addresses +$extIP:55003" + +res=`cat output | grep -A 1 "New connection, peer addresses"` +#echo "Result" +#echo $res +#echo "Expected" +#echo $exp +sudo rm -rf output +if [[ "$res" == "$exp" ]]; then + #echo $res + echo "cluster-k3s (kube-loxilb) sctp [OK]" +else + echo "cluster-k3s (kube-loxilb) sctp [NOK]" + print_debug_info + code=1 +fi if [[ $code -eq 1 ]]; then print_debug_info echo "cluster-k3s failed" - touch /vagrant/error exit 1 fi + exit