cicd: added k3s-multi-master scenario with service LB and proxy support

cicd/k3s-multi-master-service-proxy/EPconfig.txt

@@ -0,0 +1,34 @@
+{
+   "Attr":[
+      {
+         "hostName":"192.168.80.10",
+         "name":"192.168.80.10_tcp_6443",
+         "inactiveReTries":2,
+         "probeType":"tcp",
+         "probeReq":"",
+         "probeResp":"",
+         "probeDuration":5,
+         "probePort":6443
+      },
+      {
+         "hostName":"192.168.80.11",
+         "name":"192.168.80.11_tcp_6443",
+         "inactiveReTries":2,
+         "probeType":"tcp",
+         "probeReq":"",
+         "probeResp":"",
+         "probeDuration":5,
+         "probePort":6443
+      },
+      {
+         "hostName":"192.168.80.12",
+         "name":"192.168.80.12_tcp_6443",
+         "inactiveReTries":2,
+         "probeType":"tcp",
+         "probeReq":"",
+         "probeResp":"",
+         "probeDuration":5,
+         "probePort":6443
+      }
+   ]
+}

cicd/k3s-multi-master-service-proxy/README

@@ -0,0 +1,7 @@
+## Test Case Description
+
+This scenario will have K3s(2 Master Nodes & 2 Worker Nodes) cluster with flannel CNI. LoxiLB will be running in the in-cluster Active-Backup High Availabity mode(in both the master nodes) but without State Syncronization. Workloads will be spawned in all the cluster nodes.
+
+Client will be connected directly to the cluster with L2 network. Service CIDR will also be a Virtual IP from the K3s cluster network.
+
+In in-cluster scenarios, it is advised to create LB services in either one-arm or fullnat mode for ease of connectivity.

cicd/k3s-multi-master-service-proxy/Vagrantfile

@@ -0,0 +1,85 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+workers = (ENV['WORKERS'] || "2").to_i
+box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s")
+box_version = "0.7.1"
+Vagrant.configure("2") do |config|
+  config.vm.box = "#{box_name}"
+  config.vm.box_version = "#{box_version}"
+
+  if Vagrant.has_plugin?("vagrant-vbguest")
+    config.vbguest.auto_update = false
+  end
+
+  config.vm.define "host" do |host|
+    host.vm.hostname = 'host1'
+    host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
+    host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0"
+    host.vm.provision :shell, :path => "host.sh"
+    host.vm.provider :virtualbox do |vbox|
+        vbox.memory = "2048"
+        vbox.cpus = "2"
+        vbox.default_nic_type = "virtio"
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    end
+  end
+
+  config.vm.define "master1" do |master|
+    master.vm.hostname = 'master1'
+    master.vm.network :private_network, ip: "192.168.90.10", :netmask => "255.255.255.0"
+    master.vm.network :private_network, ip: "192.168.80.10", :netmask => "255.255.255.0"
+    master.vm.provision :shell, :path => "master1.sh"
+    master.vm.provider :virtualbox do |vbox|
+        vbox.memory = "8192"
+        vbox.cpus = "3"
+        vbox.default_nic_type = "virtio"
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    end
+  end
+
+  config.vm.define "master2" do |master|
+    master.vm.hostname = 'master2'
+    master.vm.network :private_network, ip: "192.168.90.11", :netmask => "255.255.255.0"
+    master.vm.network :private_network, ip: "192.168.80.11", :netmask => "255.255.255.0"
+    master.vm.provision :shell, :path => "master2.sh"
+    master.vm.provider :virtualbox do |vbox|
+        vbox.memory = "8192"
+        vbox.cpus = "3"
+        vbox.default_nic_type = "virtio"
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    end
+  end
+
+  config.vm.define "master3" do |master|
+    master.vm.hostname = 'master3'
+    master.vm.network :private_network, ip: "192.168.90.12", :netmask => "255.255.255.0"
+    master.vm.network :private_network, ip: "192.168.80.12", :netmask => "255.255.255.0"
+    master.vm.provision :shell, :path => "master3.sh"
+    master.vm.provider :virtualbox do |vbox|
+        vbox.memory = "8192"
+        vbox.cpus = "3"
+        vbox.default_nic_type = "virtio"
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    end
+  end
+
+  (1..workers).each do |node_number|
+    config.vm.define "worker#{node_number}" do |worker|
+      worker.vm.hostname = "worker#{node_number}"
+      ip = node_number + 100
+      worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
+      worker.vm.provision :shell, :path => "worker.sh"
+      worker.vm.provider :virtualbox do |vbox|
+          vbox.memory = "4096"
+          vbox.cpus = "3"
+          vbox.default_nic_type = "virtio"
+          vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+      end
+    end
+  end
+end

cicd/k3s-multi-master-service-proxy/config.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+vagrant global-status  | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f
+vagrant up
+#vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/tcp-onearm-ds.yml'
+#vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/udp-onearm-ds.yml'
+#vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/sctp-onearm-ds.yml'

cicd/k3s-multi-master-service-proxy/host.sh

@@ -0,0 +1,3 @@
+sudo apt-get install -y lksctp-tools iperf iperf3
+sysctl net.core.netdev_max_backlog=10000
+echo "Host is up"

cicd/k3s-multi-master-service-proxy/host_validation.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+extIP=$(cat /vagrant/extIP)
+
+mode="onearm"
+tcp_port=55001
+udp_port=55002
+sctp_port=55003
+
+code=0
+echo Service IP: $extIP
+
+ip route list match $extIP | grep $extIP -A 2
+
+echo -e "\n*********************************************"
+echo "Testing Service"
+echo "*********************************************"
+for((i=0;i<20;i++))
+do
+
+out=$(curl -s --connect-timeout 10 http://$extIP:$tcp_port)
+if [[ ${out} == *"Welcome to nginx"* ]]; then
+  echo -e "K3s-flannel-incluster-l2 TCP\t($mode)\t[OK]"
+else
+  echo -e "K3s-flannel-incluster-l2 TCP\t($mode)\t[FAILED]"
+  code=1
+fi
+
+out=$(timeout 5 /vagrant/udp_client $extIP $udp_port)
+if [[ ${out} == *"Client"* ]]; then
+  echo -e "K3s-flannel-incluster-l2 UDP\t($mode)\t[OK]"
+else
+  echo -e "K3s-flannel-incluster-l2 UDP\t($mode)\t[FAILED]"
+  code=1
+fi
+
+sctp_darn -H 192.168.80.9 -h $extIP -p $sctp_port -s < /vagrant/input > output
+#sleep 2
+exp="New connection, peer addresses
+192.168.80.200:55003"
+
+res=`cat output | grep -A 1 "New connection, peer addresses"`
+sudo rm -rf output
+if [[ "$res" == "$exp" ]]; then
+    #echo $res
+    echo -e "K3s-flannel-incluster-l2 SCTP\t($mode)\t[OK]"
+else
+    echo -e "K3s-flannel-incluster-l2 SCTP\t($mode)\t[FAILED]"
+    code=1
+fi
+
+
+done
+exit $code

cicd/k3s-multi-master-service-proxy/input

@@ -0,0 +1,6 @@
+
+
+
+
+
+

cicd/k3s-multi-master-service-proxy/iperf-onearm-ds.yml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: iperf-service
+  annotations:
+    loxilb.io/lbmode: "onearm" 
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    app: iperf-app
+  ports:
+    - port: 55001
+      targetPort: 5001
+  type: LoadBalancer
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: iperf-daemonset
+  labels:
+    app: iperf-app
+spec:
+  selector:
+    matchLabels:
+      app: iperf-app
+  template:
+    metadata:
+      labels:
+        app: iperf-app
+    spec:
+      dnsPolicy: ClusterFirstWithHostNet
+      #affinity:
+      #  nodeAffinity:
+      #    requiredDuringSchedulingIgnoredDuringExecution:
+      #      nodeSelectorTerms:
+      #      - matchExpressions:
+      #        - key: "node-role.kubernetes.io/master"
+      #          operator: DoesNotExist
+      #        - key: "node-role.kubernetes.io/control-plane"
+      #          operator: DoesNotExist
+      containers:
+      - name: iperf
+        image: eyes852/ubuntu-iperf-test:0.5
+        command: [ "iperf", "-s"]
+        ports:
+        - containerPort: 5001

cicd/k3s-multi-master-service-proxy/k3s.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTVRRek9EWTBPRGt3SGhjTk1qUXdOREk1TVRBeU9EQTVXaGNOTXpRd05ESTNNVEF5T0RBNQpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTVRRek9EWTBPRGt3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTU29Zc0RSbkVnNGdpeFVDcEY3aWZpUWVWVU5QMDJidmc5bUJ6VmUwbUcKRDZmZjVWV1h2Yk1JTXM4UWdzUDlkMVMrUWNab3JyUGEvZHo3NXJDRnZrM2tvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVVZLSEQwQ0prd2k3SFQ3ejJPUUNSCmZZTC83Y1F3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnWmp1aENUek5sYzMyc1NTa3hTVndEaUxEQ2hHenpaRUkKaGxGSFJtNFIrbllDSVFDaHJja0kzY3F6djB6UTN2VFNjSHM5MzRYSkF6S0dnb0hQZmx4b0tkbVM4QT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    server: https://192.168.80.80:6443
+  name: default
+contexts:
+- context:
+    cluster: default
+    user: default
+  name: default
+current-context: default
+kind: Config
+preferences: {}
+users:
+- name: default
+  user:
+    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrRENDQVRlZ0F3SUJBZ0lJYUR3NFdwek1wRmd3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOekUwTXpnMk5EZzVNQjRYRFRJME1EUXlPVEV3TWpnd09Wb1hEVEkxTURReQpPVEV3TWpnd09Wb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJDdUdLQlZNSUJNYVVrZEcKQlNCalBGNzNWMmVaNHZCaGl0ejUrYWtqRUprNnBWSW1yQzdPTkNVVytjOWowZnFJS0xZVGJyblI3cWZiT0FiMgp2UU9GWTZ1alNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCU21QOXBVR0FoNlpKOFZONUhnUzdHMEd6Y092akFLQmdncWhrak9QUVFEQWdOSEFEQkUKQWlBaEVNcWs3TUtPTXd6TmM4aEc3L04wR3RZZzIzQVgvdmFsdmNEOFAycitqZ0lnZHJIbDZZdmNTbWFoVG1GYgowQTJwWkpxK2hvNkg2ZThBVCtRRTJGSHVKZXM9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUzTVRRek9EWTBPRGt3SGhjTk1qUXdOREk1TVRBeU9EQTVXaGNOTXpRd05ESTNNVEF5T0RBNQpXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUzTVRRek9EWTBPRGt3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTNVZyeHFpaE4yMktlcCtwSTFWalBna21hV09tUXBBek9wY0VrQzNrRVAKTDB3cS9CalVGaVo2ckNxZklLSndzckhnbGgxemtJQzhuRjdhbzU0SjdzbS9vMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXBqL2FWQmdJZW1TZkZUZVI0RXV4CnRCczNEcjR3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnSFVIM3UzR2NsWmxPQWFNMEpNOVRrMTVISUhYUVJ2VVIKcmUzNksvSkNmTWNDSVFEdk5pWmdFaGVmRytrZkdoOHBHK080REFHVVA1ZG5QTXNMU2JvZVlWelhTUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJNL1lLR0svWkkzc3IrUmZjNURQWkF3YzBtY3gwZFBlNFkwdmxpdGhMS3JvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFSzRZb0ZVd2dFeHBTUjBZRklHTThYdmRYWjVuaThHR0szUG41cVNNUW1UcWxVaWFzTHM0MApKUmI1ejJQUitvZ290aE51dWRIdXA5czRCdmE5QTRWanF3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=