diff --git a/.github/workflows/k3s-flannel-ubuntu-22.yml b/.github/workflows/k3s-flannel-ubuntu-22.yml index 63f5ed064..58cbc6b8a 100644 --- a/.github/workflows/k3s-flannel-ubuntu-22.yml +++ b/.github/workflows/k3s-flannel-ubuntu-22.yml @@ -1,8 +1,8 @@ name: K3s-Flannel-Sanity-CI-Ubuntu-22 on: schedule: - # Runs "At 11:00 UTC every day-of-week" - - cron: '0 11 * * *' + # Runs "At 23:00 UTC every day-of-week" + - cron: '0 23 * * *' workflow_dispatch: inputs: testName: @@ -12,28 +12,24 @@ on: jobs: build: name: k3s-flannel-sanity-ubuntu-22 - runs-on: ubuntu-22.04 + runs-on: self-hosted steps: - - run: sudo apt update && sudo apt -y install bridge-utils lksctp-tools - - run: | - echo "KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV - uses: actions/checkout@v2 with: submodules: recursive - - run: curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh - - - run: | - sleep 10 - kubectl "${{ env.KUBECONFIG }}" taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- - sleep 60 - kubectl "${{ env.KUBECONFIG }}" get nodes - kubectl "${{ env.KUBECONFIG }}" get pods -A - wget https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml - kubectl "${{ env.KUBECONFIG }}" apply -f ./loxi-ccm-k3s.yaml - sleep 60 - kubectl "${{ env.KUBECONFIG }}" get pods -A - - run: | + - name: Run k3s-flannel CICD + if: github.repository == 'loxilb-io/loxilb' + run: | + echo "KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV + echo "ubuntu=22.04" >> $GITHUB_ENV cd cicd/k3s-flannel/ - ./config.sh "${{ env.KUBECONFIG }}" - ./validation.sh "${{ env.KUBECONFIG }}" - ./rmconfig.sh "${{ env.KUBECONFIG }}" + ./config.sh "ubuntu" "22.04" + ./validation.sh + ./rmconfig.sh cd - + - name: Clean test-bed + if: success() || failure() + run: | + cd cicd/k3s-flannel/ || true + ./rmconfig.sh + cd - diff --git a/.github/workflows/k3s-flannel.yml b/.github/workflows/k3s-flannel.yml index 47b83fe24..175ac4d1a 100644 --- a/.github/workflows/k3s-flannel.yml +++ b/.github/workflows/k3s-flannel.yml @@ -1,8 +1,8 @@ name: K3s-Flannel-Sanity-CI on: schedule: - # Runs "At 11:00 UTC every day-of-week" - - cron: '0 11 * * *' + # Runs "At 23:00 UTC every day-of-week" + - cron: '0 23 * * *' workflow_dispatch: inputs: testName: @@ -12,28 +12,24 @@ on: jobs: build: name: k3s-flannel-sanity - runs-on: ubuntu-latest + runs-on: self-hosted steps: - - run: sudo apt update && sudo apt -y install bridge-utils lksctp-tools - - run: | - echo "KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV - uses: actions/checkout@v2 with: submodules: recursive - - run: curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh - - - run: | - sleep 10 - kubectl "${{ env.KUBECONFIG }}" taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- - sleep 60 - kubectl "${{ env.KUBECONFIG }}" get nodes - kubectl "${{ env.KUBECONFIG }}" get pods -A - wget https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml - kubectl "${{ env.KUBECONFIG }}" apply -f ./loxi-ccm-k3s.yaml - sleep 60 - kubectl "${{ env.KUBECONFIG }}" get pods -A - - run: | + - name: Run k3s-flannel CICD + if: github.repository == 'loxilb-io/loxilb' + run: | + echo "KUBECONFIG=--kubeconfig=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV + echo "ubuntu=20.04" >> $GITHUB_ENV cd cicd/k3s-flannel/ - ./config.sh "${{ env.KUBECONFIG }}" + ./config.sh "ubuntu" "20.04" ./validation.sh "${{ env.KUBECONFIG }}" ./rmconfig.sh "${{ env.KUBECONFIG }}" cd - + - name: Clean test-bed + if: success() || failure() + run: | + cd cicd/k3s-flannel/ || true + ./rmconfig.sh + cd - diff --git a/cicd/k3s-flannel/Vagrantfile b/cicd/k3s-flannel/Vagrantfile new file mode 100644 index 000000000..3e7ffea67 --- /dev/null +++ b/cicd/k3s-flannel/Vagrantfile @@ -0,0 +1,26 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +workers = (ENV['WORKERS'] || "1").to_i +#box_name = (ENV['VAGRANT_BOX'] || "ubuntu/focal64") +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = (ENV['VAGRANT_BOX_VERSION'] || "0.7.1") +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + config.vm.define "k3s" do |k3s| + k3s.vm.hostname = 'k3s' + k3s.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0" + k3s.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0" + k3s.vm.provision :shell, :path => "config-k3s.sh" + k3s.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 16384] + vbox.customize ["modifyvm", :id, "--cpus", 8] + end + end +end diff --git a/cicd/k3s-flannel/config-k3s.sh b/cicd/k3s-flannel/config-k3s.sh new file mode 100755 index 000000000..78f64c4a2 --- /dev/null +++ b/cicd/k3s-flannel/config-k3s.sh @@ -0,0 +1,203 @@ +#!/bin/bash + +sudo su +cd /home/vagrant/ + +apt-get update +apt-get install -y software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce curl wget iperf bridge-utils net-tools + +sudo systemctl stop unattended-upgrades +sudo apt-get purge -y unattended-upgrades + +git clone --recurse-submodule https://github.com/loxilb-io/loxilb.git +cd loxilb/cicd/k3s-flannel + +source ../common.sh +source ../k3s_common.sh + +echo "#########################################" +echo "Spawning all hosts" +echo "#########################################" + +spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --with-ka in +spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --with-ka in +spawn_docker_host --dock-type host --dock-name ep1 +spawn_docker_host --dock-type host --dock-name ep2 +spawn_docker_host --dock-type host --dock-name ep3 +spawn_docker_host --dock-type host --dock-name r1 --with-bgp yes --bgp-config $(pwd)/r1_config +spawn_docker_host --dock-type host --dock-name r2 --with-bgp yes --bgp-config $(pwd)/r2_config +spawn_docker_host --dock-type host --dock-name user + +echo "#########################################" +echo "Connecting and configuring hosts" +echo "#########################################" + +connect_docker_hosts user r1 +connect_docker_hosts r1 llb1 +connect_docker_hosts r1 llb2 +connect_docker_hosts llb1 r2 +connect_docker_hosts llb2 r2 +connect_docker_hosts r2 ep1 +connect_docker_hosts r2 ep2 +connect_docker_hosts r2 ep3 + +## Make network for k3s connectivity +sudo ip link add ellb1sys type veth peer name esysllb1 +sleep 3 +sudo ip link set ellb1sys netns llb1 +sleep 3 +sudo ip -n llb1 link set ellb1sys up +sudo ip -n llb1 addr add 12.12.12.1/24 dev ellb1sys +sudo ip link set esysllb1 up +#sudo ip addr add 12.12.12.254/24 dev esysllb1 + +sudo ip link add ellb2sys type veth peer name esysllb2 +sleep 3 +sudo ip link set ellb2sys netns llb2 +sleep 3 +sudo ip -n llb2 link set ellb2sys up +sudo ip -n llb2 addr add 12.12.12.2/24 dev ellb2sys +sudo ip link set esysllb2 up +#sudo ip addr add 14.14.14.254/24 dev esysllb2 + +sudo brctl addbr k3sbr +sudo brctl addif k3sbr esysllb2 +sudo brctl addif k3sbr esysllb1 +sudo ip link set k3sbr up +sudo ip addr add 12.12.12.254/24 dev k3sbr + +#node1 config +config_docker_host --host1 user --host2 r1 --ptype phy --addr 1.1.1.1/24 --gw 1.1.1.254 +config_docker_host --host1 r1 --host2 user --ptype phy --addr 1.1.1.254/24 + +create_docker_host_vlan --host1 r1 --host2 llb1 --id 11 --ptype untagged +create_docker_host_vlan --host1 llb1 --host2 r1 --id 11 --ptype untagged + +config_docker_host --host1 r1 --host2 llb1 --ptype vlan --id 11 --addr 11.11.11.254/24 --gw 11.11.11.11 +config_docker_host --host1 llb1 --host2 r1 --ptype vlan --id 11 --addr 11.11.11.1/24 + +create_docker_host_vlan --host1 r1 --host2 llb2 --id 11 --ptype untagged +create_docker_host_vlan --host1 llb2 --host2 r1 --id 11 --ptype untagged +config_docker_host --host1 llb2 --host2 r1 --ptype vlan --id 11 --addr 11.11.11.2/24 + + +create_docker_host_vlan --host1 llb1 --host2 r2 --id 10 --ptype untagged +config_docker_host --host1 llb1 --host2 r2 --ptype vlan --id 10 --addr 10.10.10.1/24 +create_docker_host_vlan --host1 llb2 --host2 r2 --id 10 --ptype untagged +config_docker_host --host1 llb2 --host2 r2 --ptype vlan --id 10 --addr 10.10.10.2/24 + +create_docker_host_vlan --host1 r2 --host2 llb1 --id 10 --ptype untagged +create_docker_host_vlan --host1 r2 --host2 llb2 --id 10 --ptype untagged +create_docker_host_vlan --host1 r2 --host2 ep1 --id 10 --ptype untagged +create_docker_host_vlan --host1 r2 --host2 ep2 --id 10 --ptype untagged +create_docker_host_vlan --host1 r2 --host2 ep3 --id 10 --ptype untagged +config_docker_host --host1 r2 --host2 llb1 --ptype vlan --id 10 --addr 10.10.10.254/24 + +create_docker_host_vlan --host1 r2 --host2 ep1 --id 31 --ptype untagged +config_docker_host --host1 r2 --host2 ep1 --ptype vlan --id 31 --addr 31.31.31.254/24 + +create_docker_host_vlan --host1 ep1 --host2 r2 --id 31 --ptype untagged +config_docker_host --host1 ep1 --host2 r2 --ptype vlan --id 31 --addr 31.31.31.1/24 --gw 31.31.31.254 + +create_docker_host_vlan --host1 r2 --host2 ep2 --id 32 --ptype untagged +config_docker_host --host1 r2 --host2 ep2 --ptype vlan --id 32 --addr 32.32.32.254/24 + +create_docker_host_vlan --host1 ep2 --host2 r2 --id 32 --ptype untagged +config_docker_host --host1 ep2 --host2 r2 --ptype vlan --id 32 --addr 32.32.32.1/24 --gw 32.32.32.254 + +create_docker_host_vlan --host1 r2 --host2 ep3 --id 33 --ptype untagged +config_docker_host --host1 r2 --host2 ep3 --ptype vlan --id 33 --addr 33.33.33.254/24 + +create_docker_host_vlan --host1 ep3 --host2 r2 --id 33 --ptype untagged +config_docker_host --host1 ep3 --host2 r2 --ptype vlan --id 33 --addr 33.33.33.1/24 --gw 33.33.33.254 + +##Pod networks +$hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11 +#add_route llb1 1.1.1.0/24 11.11.11.254 +#add_route llb2 1.1.1.0/24 11.11.11.254 + +## host network +#sudo ip route add 11.11.11.11/32 via 14.14.14.1 +#sudo ip route add 123.123.123.1/32 via 14.14.14.1 + +sleep 1 +##Create LB rule +create_lb_rule llb1 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullnat --bgp +create_lb_rule llb2 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullnat --bgp + +# keepalive will take few seconds to be UP and running with valid states +sleep 60 + +# K3s setup + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# If k3s setup exists, skip installation +if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then + echo "K3s exists" + sleep 10 +else + echo "Start K3s installation" + + # Install k3s without external cloud-manager and disabled servicelb + #curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh - + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh - + + sleep 10 + + # Check kubectl works + kubectl $KUBECONFIG get pods -A + + # Remove taints in k3s if any (usually happens if started without cloud-manager) + kubectl $KUBECONFIG taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- + + # Start loxi-ccm as k3s daemonset + kubectl $KUBECONFIG apply -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml + + echo "End K3s installation" +fi + + +sudo apt-get update + +# Install Bird to work with k3s +sudo apt-get install bird2 --yes + +sleep 5 + +sudo cp -f bird_config/bird.conf /etc/bird/bird.conf +if [ ! -f /var/log/bird.log ]; then + sudo touch /var/log/bird.log +fi +sudo chown bird:bird /var/log/bird.log +sudo systemctl restart bird + +# Wait for cluster readiness +wait_cluster_ready_full + +sleep 10 +# Start nginx pods and services for test +#kubectl $KUBECONFIG apply -f nginx.yml +#kubectl $KUBECONFIG apply -f nginx-svc-lb.yml + +sleep 5 + +# Start nginx pods and services for test(using kube-loxilb) +kubectl $KUBECONFIG apply -f kube-loxilb.yml +sleep 15 +kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml +sleep 10 +kubectl $KUBECONFIG apply -f udp-svc-lb.yml +sleep 30 + +# External LB service must be created by now +kubectl $KUBECONFIG get svc + +# Wait for cluster readiness +wait_cluster_ready_full diff --git a/cicd/k3s-flannel/config.sh b/cicd/k3s-flannel/config.sh index 17be6f4f9..e1c73e8d2 100755 --- a/cicd/k3s-flannel/config.sh +++ b/cicd/k3s-flannel/config.sh @@ -1,187 +1,18 @@ #!/bin/bash -source ../common.sh -source ../k3s_common.sh - -echo "#########################################" -echo "Spawning all hosts" -echo "#########################################" - -spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --with-ka in -spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --with-ka in -spawn_docker_host --dock-type host --dock-name ep1 -spawn_docker_host --dock-type host --dock-name ep2 -spawn_docker_host --dock-type host --dock-name ep3 -spawn_docker_host --dock-type host --dock-name r1 --with-bgp yes --bgp-config $(pwd)/r1_config -spawn_docker_host --dock-type host --dock-name r2 --with-bgp yes --bgp-config $(pwd)/r2_config -spawn_docker_host --dock-type host --dock-name user - -echo "#########################################" -echo "Connecting and configuring hosts" -echo "#########################################" - -connect_docker_hosts user r1 -connect_docker_hosts r1 llb1 -connect_docker_hosts r1 llb2 -connect_docker_hosts llb1 r2 -connect_docker_hosts llb2 r2 -connect_docker_hosts r2 ep1 -connect_docker_hosts r2 ep2 -connect_docker_hosts r2 ep3 - -## Make network for k3s connectivity -sudo ip link add ellb1sys type veth peer name esysllb1 -sleep 3 -sudo ip link set ellb1sys netns llb1 -sleep 3 -sudo ip -n llb1 link set ellb1sys up -sudo ip -n llb1 addr add 12.12.12.1/24 dev ellb1sys -sudo ip link set esysllb1 up -#sudo ip addr add 12.12.12.254/24 dev esysllb1 - -sudo ip link add ellb2sys type veth peer name esysllb2 -sleep 3 -sudo ip link set ellb2sys netns llb2 -sleep 3 -sudo ip -n llb2 link set ellb2sys up -sudo ip -n llb2 addr add 12.12.12.2/24 dev ellb2sys -sudo ip link set esysllb2 up -#sudo ip addr add 14.14.14.254/24 dev esysllb2 - -sudo brctl addbr k3sbr -sudo brctl addif k3sbr esysllb2 -sudo brctl addif k3sbr esysllb1 -sudo ip link set k3sbr up -sudo ip addr add 12.12.12.254/24 dev k3sbr - -#node1 config -config_docker_host --host1 user --host2 r1 --ptype phy --addr 1.1.1.1/24 --gw 1.1.1.254 -config_docker_host --host1 r1 --host2 user --ptype phy --addr 1.1.1.254/24 - -create_docker_host_vlan --host1 r1 --host2 llb1 --id 11 --ptype untagged -create_docker_host_vlan --host1 llb1 --host2 r1 --id 11 --ptype untagged - -config_docker_host --host1 r1 --host2 llb1 --ptype vlan --id 11 --addr 11.11.11.254/24 --gw 11.11.11.11 -config_docker_host --host1 llb1 --host2 r1 --ptype vlan --id 11 --addr 11.11.11.1/24 - -create_docker_host_vlan --host1 r1 --host2 llb2 --id 11 --ptype untagged -create_docker_host_vlan --host1 llb2 --host2 r1 --id 11 --ptype untagged -config_docker_host --host1 llb2 --host2 r1 --ptype vlan --id 11 --addr 11.11.11.2/24 - - -create_docker_host_vlan --host1 llb1 --host2 r2 --id 10 --ptype untagged -config_docker_host --host1 llb1 --host2 r2 --ptype vlan --id 10 --addr 10.10.10.1/24 -create_docker_host_vlan --host1 llb2 --host2 r2 --id 10 --ptype untagged -config_docker_host --host1 llb2 --host2 r2 --ptype vlan --id 10 --addr 10.10.10.2/24 - -create_docker_host_vlan --host1 r2 --host2 llb1 --id 10 --ptype untagged -create_docker_host_vlan --host1 r2 --host2 llb2 --id 10 --ptype untagged -create_docker_host_vlan --host1 r2 --host2 ep1 --id 10 --ptype untagged -create_docker_host_vlan --host1 r2 --host2 ep2 --id 10 --ptype untagged -create_docker_host_vlan --host1 r2 --host2 ep3 --id 10 --ptype untagged -config_docker_host --host1 r2 --host2 llb1 --ptype vlan --id 10 --addr 10.10.10.254/24 - -create_docker_host_vlan --host1 r2 --host2 ep1 --id 31 --ptype untagged -config_docker_host --host1 r2 --host2 ep1 --ptype vlan --id 31 --addr 31.31.31.254/24 - -create_docker_host_vlan --host1 ep1 --host2 r2 --id 31 --ptype untagged -config_docker_host --host1 ep1 --host2 r2 --ptype vlan --id 31 --addr 31.31.31.1/24 --gw 31.31.31.254 - -create_docker_host_vlan --host1 r2 --host2 ep2 --id 32 --ptype untagged -config_docker_host --host1 r2 --host2 ep2 --ptype vlan --id 32 --addr 32.32.32.254/24 - -create_docker_host_vlan --host1 ep2 --host2 r2 --id 32 --ptype untagged -config_docker_host --host1 ep2 --host2 r2 --ptype vlan --id 32 --addr 32.32.32.1/24 --gw 32.32.32.254 - -create_docker_host_vlan --host1 r2 --host2 ep3 --id 33 --ptype untagged -config_docker_host --host1 r2 --host2 ep3 --ptype vlan --id 33 --addr 33.33.33.254/24 - -create_docker_host_vlan --host1 ep3 --host2 r2 --id 33 --ptype untagged -config_docker_host --host1 ep3 --host2 r2 --ptype vlan --id 33 --addr 33.33.33.1/24 --gw 33.33.33.254 - -##Pod networks -$hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11 -#add_route llb1 1.1.1.0/24 11.11.11.254 -#add_route llb2 1.1.1.0/24 11.11.11.254 - -## host network -#sudo ip route add 11.11.11.11/32 via 14.14.14.1 -#sudo ip route add 123.123.123.1/32 via 14.14.14.1 - -sleep 1 -##Create LB rule -create_lb_rule llb1 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullnat --bgp -create_lb_rule llb2 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullnat --bgp - -# keepalive will take few seconds to be UP and running with valid states -sleep 60 - -# K3s setup - -if [ "$1" ]; then - KUBECONFIG="$1" +if [ "$#" -eq 2 ]; then + ubuntu=$2 + echo "version: $ubuntu" fi -# If k3s setup exists, skip installation -if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then - echo "K3s exists" - sleep 10 +if [[ "$ubuntu" == "20.04" ]]; then + export VAGRANT_BOX="sysnet4admin/Ubuntu-k8s" + export VAGRANT_BOX_VERSION="0.7.1" +elif [[ "$ubuntu" == "22.04" ]]; then + export VAGRANT_BOX="bento/ubuntu-22.04" + export VAGRANT_BOX_VERSION=">= 0" else - echo "Start K3s installation" - - # Install k3s without external cloud-manager and disabled servicelb - #curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh - - curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external" K3S_KUBECONFIG_MODE="644" sh - - - sleep 10 - - # Check kubectl works - kubectl $KUBECONFIG get pods -A - - # Remove taints in k3s if any (usually happens if started without cloud-manager) - kubectl $KUBECONFIG taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- - - # Start loxi-ccm as k3s daemonset - kubectl $KUBECONFIG apply -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml - - echo "End K3s installation" -fi - - -sudo apt-get update - -# Install Bird to work with k3s -sudo apt-get install bird2 --yes - -sleep 5 - -sudo cp -f bird_config/bird.conf /etc/bird/bird.conf -if [ ! -f /var/log/bird.log ]; then - sudo touch /var/log/bird.log + export VAGRANT_BOX="sysnet4admin/Ubuntu-k8s" + export VAGRANT_BOX_VERSION="0.7.1" fi -sudo chown bird:bird /var/log/bird.log -sudo systemctl restart bird - -# Wait for cluster readiness -wait_cluster_ready_full - -sleep 10 -# Start nginx pods and services for test -#kubectl $KUBECONFIG apply -f nginx.yml -#kubectl $KUBECONFIG apply -f nginx-svc-lb.yml - -sleep 5 - -# Start nginx pods and services for test(using kube-loxilb) -kubectl $KUBECONFIG apply -f kube-loxilb.yml -sleep 15 -kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml -sleep 10 -kubectl $KUBECONFIG apply -f udp-svc-lb.yml -sleep 30 - -# External LB service must be created by now -kubectl $KUBECONFIG get svc - -# Wait for cluster readiness -wait_cluster_ready_full +vagrant up k3s diff --git a/cicd/k3s-flannel/rmconfig-k3s.sh b/cicd/k3s-flannel/rmconfig-k3s.sh new file mode 100755 index 000000000..28021feb5 --- /dev/null +++ b/cicd/k3s-flannel/rmconfig-k3s.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +echo "#########################################" +echo "Removing testbed" +echo "#########################################" + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +source ../common.sh + +#sudo ip route del 11.11.11.11/32 via 14.14.14.1 +#sudo ip route del 123.123.123.1/32 via 14.14.14.1 + +sudo kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f nginx-svc-lb.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f nginx.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f udp-svc-lb.yml >> /dev/null 2>&1 +sudo kubectl $KUBECONFIG delete -f kube-loxilb.yml >> /dev/null 2>&1 +#sudo kubectl $KUBECONFIG delete -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml >> /dev/null 2>&1 + +disconnect_docker_hosts user r1 +disconnect_docker_hosts r1 llb1 +disconnect_docker_hosts r1 llb2 +disconnect_docker_hosts llb1 r2 +disconnect_docker_hosts llb2 r2 +disconnect_docker_hosts r2 ep1 +disconnect_docker_hosts r2 ep2 +disconnect_docker_hosts r2 ep3 + +delete_docker_host ka_llb1 +delete_docker_host ka_llb2 +delete_docker_host llb1 +delete_docker_host llb2 +delete_docker_host user +delete_docker_host r1 +delete_docker_host r2 +delete_docker_host ep1 +delete_docker_host ep2 +delete_docker_host ep3 +sudo brctl delif k3sbr esysllb2 +sudo brctl delif k3sbr esysllb1 +sudo ip link set k3sbr down +sudo brctl delbr k3sbr +sudo ip link del esysllb1 +sudo ip link del esysllb2 + +# If k3s setup exists, remove it +if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then + /usr/local/bin/k3s-uninstall.sh +fi + +sudo apt-get remove bird2 --yes + +echo "#########################################" +echo "Removed testbed" +echo "#########################################" diff --git a/cicd/k3s-flannel/rmconfig.sh b/cicd/k3s-flannel/rmconfig.sh index 28021feb5..7d197e0bf 100755 --- a/cicd/k3s-flannel/rmconfig.sh +++ b/cicd/k3s-flannel/rmconfig.sh @@ -1,58 +1,3 @@ #!/bin/bash -echo "#########################################" -echo "Removing testbed" -echo "#########################################" - -if [ "$1" ]; then - KUBECONFIG="$1" -fi - -source ../common.sh - -#sudo ip route del 11.11.11.11/32 via 14.14.14.1 -#sudo ip route del 123.123.123.1/32 via 14.14.14.1 - -sudo kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml >> /dev/null 2>&1 -sudo kubectl $KUBECONFIG delete -f nginx-svc-lb.yml >> /dev/null 2>&1 -sudo kubectl $KUBECONFIG delete -f nginx.yml >> /dev/null 2>&1 -sudo kubectl $KUBECONFIG delete -f udp-svc-lb.yml >> /dev/null 2>&1 -sudo kubectl $KUBECONFIG delete -f kube-loxilb.yml >> /dev/null 2>&1 -#sudo kubectl $KUBECONFIG delete -f https://github.com/loxilb-io/loxi-ccm/raw/master/manifests/loxi-ccm-k3s.yaml >> /dev/null 2>&1 - -disconnect_docker_hosts user r1 -disconnect_docker_hosts r1 llb1 -disconnect_docker_hosts r1 llb2 -disconnect_docker_hosts llb1 r2 -disconnect_docker_hosts llb2 r2 -disconnect_docker_hosts r2 ep1 -disconnect_docker_hosts r2 ep2 -disconnect_docker_hosts r2 ep3 - -delete_docker_host ka_llb1 -delete_docker_host ka_llb2 -delete_docker_host llb1 -delete_docker_host llb2 -delete_docker_host user -delete_docker_host r1 -delete_docker_host r2 -delete_docker_host ep1 -delete_docker_host ep2 -delete_docker_host ep3 -sudo brctl delif k3sbr esysllb2 -sudo brctl delif k3sbr esysllb1 -sudo ip link set k3sbr down -sudo brctl delbr k3sbr -sudo ip link del esysllb1 -sudo ip link del esysllb2 - -# If k3s setup exists, remove it -if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then - /usr/local/bin/k3s-uninstall.sh -fi - -sudo apt-get remove bird2 --yes - -echo "#########################################" -echo "Removed testbed" -echo "#########################################" +vagrant destroy -f k3s diff --git a/cicd/k3s-flannel/validation-k3s.sh b/cicd/k3s-flannel/validation-k3s.sh new file mode 100755 index 000000000..e742cae4b --- /dev/null +++ b/cicd/k3s-flannel/validation-k3s.sh @@ -0,0 +1,113 @@ +#!/bin/bash +source ../common.sh +echo cluster-k3s + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +for((i=0; i<120; i++)) +do + extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb1") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find nginx-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +sleep 60 + +echo "ExternalIP $extIP" + +print_debug_info() { + ## Dump some debug info + echo "**** k3s svc info ****" + sudo kubectl $KUBECONFIG get svc + echo "**** k3s pods info ****" + sudo kubectl $KUBECONFIG get pods -A + + echo "**** llb1 lb-info ****" + $dexec llb1 loxicmd get lb -o wide + echo "**** loxilb ep-info ****" + $dexec llb1 loxicmd get ep -o wide + echo "**** llb1 route-info ****" + $dexec llb1 ip route + echo "**** llb1 bfd-info ****" + $dexec llb1 loxicmd get ha + + echo "**** llb2 lb-info ****" + $dexec llb2 loxicmd get lb -o wide + echo "**** loxilb ep-info ****" + $dexec llb1 loxicmd get ep -o wide + echo "**** llb2 route-info ****" + $dexec llb2 ip route + echo "**** llb2 bfd-info ****" + $dexec llb2 loxicmd get ha + + echo "**** r1 route-info ****" + $dexec r1 ip route + + echo "**** sys route-info ****" + ip route +} + +code=0 +echo "********************" +print_debug_info +echo "********************" + +out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) +if [[ ${out} == *"Welcome to nginx"* ]]; then + echo "cluster-k3s (kube-loxilb) tcp [OK]" +else + echo "cluster-k3s (kube-loxilb) tcp [FAILED]" + code=1 +fi + +for((i=0; i<120; i++)) +do + extLB=$(sudo kubectl $KUBECONFIG get svc | grep "udp-lb1") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find udp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +out=$($hexec user timeout 30 ../common/udp_client $extIP 55003) +if [[ ${out} == *"Client"* ]]; then + echo "cluster-k3s (kube-loxilb) udp [OK]" +else + echo "cluster-k3s (kube-loxilb) udp [FAILED]" + code=1 +fi + +if [[ $code -eq 1 ]]; then + print_debug_info + echo "cluster-k3s failed" + touch /vagrant/error + exit 1 +fi + +exit diff --git a/cicd/k3s-flannel/validation.sh b/cicd/k3s-flannel/validation.sh index 3b800ce20..38e7c8d0a 100755 --- a/cicd/k3s-flannel/validation.sh +++ b/cicd/k3s-flannel/validation.sh @@ -1,112 +1,7 @@ #!/bin/bash -source ../common.sh -echo cluster-k3s - -if [ "$1" ]; then - KUBECONFIG="$1" -fi - -# Set space as the delimiter -IFS=' ' - -for((i=0; i<120; i++)) -do - extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb1") - read -a strarr <<< "$extLB" - len=${#strarr[*]} - if [[ $((len)) -lt 6 ]]; then - echo "Can't find nginx-lb service" - sleep 1 - continue - fi - if [[ ${strarr[3]} != *"none"* ]]; then - extIP="$(cut -d'-' -f2 <<<${strarr[3]})" - break - fi - echo "No external LB allocated" - sleep 1 -done - -## Any routing updates ?? -sleep 60 - -echo "ExternalIP $extIP" - -print_debug_info() { - ## Dump some debug info - echo "**** k3s svc info ****" - sudo kubectl $KUBECONFIG get svc - echo "**** k3s pods info ****" - sudo kubectl $KUBECONFIG get pods -A - - echo "**** llb1 lb-info ****" - $dexec llb1 loxicmd get lb -o wide - echo "**** loxilb ep-info ****" - $dexec llb1 loxicmd get ep -o wide - echo "**** llb1 route-info ****" - $dexec llb1 ip route - echo "**** llb1 bfd-info ****" - $dexec llb1 loxicmd get ha - - echo "**** llb2 lb-info ****" - $dexec llb2 loxicmd get lb -o wide - echo "**** loxilb ep-info ****" - $dexec llb1 loxicmd get ep -o wide - echo "**** llb2 route-info ****" - $dexec llb2 ip route - echo "**** llb2 bfd-info ****" - $dexec llb2 loxicmd get ha - - echo "**** r1 route-info ****" - $dexec r1 ip route - - echo "**** sys route-info ****" - ip route -} - -code=0 -echo "********************" -print_debug_info -echo "********************" - -out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) -if [[ ${out} == *"Welcome to nginx"* ]]; then - echo "cluster-k3s (kube-loxilb) tcp [OK]" -else - echo "cluster-k3s (kube-loxilb) tcp [FAILED]" - code=1 -fi - -for((i=0; i<120; i++)) -do - extLB=$(sudo kubectl $KUBECONFIG get svc | grep "udp-lb1") - read -a strarr <<< "$extLB" - len=${#strarr[*]} - if [[ $((len)) -lt 6 ]]; then - echo "Can't find udp-lb service" - sleep 1 - continue - fi - if [[ ${strarr[3]} != *"none"* ]]; then - extIP="$(cut -d'-' -f2 <<<${strarr[3]})" - break - fi - echo "No external LB allocated" - sleep 1 -done - -out=$($hexec user timeout 30 ../common/udp_client $extIP 55003) -if [[ ${out} == *"Client"* ]]; then - echo "cluster-k3s (kube-loxilb) udp [OK]" -else - echo "cluster-k3s (kube-loxilb) udp [FAILED]" - code=1 -fi - -if [[ $code -eq 1 ]]; then - print_debug_info - echo "cluster-k3s failed" +vagrant ssh k3s -c "cd /home/vagrant/loxilb/cicd/k3s-flannel/ && ./validation-k3s.sh" +if [ -f ./error ]; then + rm -f ./error exit 1 fi - exit