From 40d599463bbca82e826a51db5ffae879fb1f4ec6 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Thu, 7 Mar 2024 12:51:37 +0900 Subject: [PATCH 1/6] Removed unwanted files and put a validation check before creating bfd session --- .../operations/delete_config_bfd_remote_ip.go | 58 --- .../delete_config_bfd_remote_ip_parameters.go | 101 ----- .../delete_config_bfd_remote_ip_responses.go | 354 ------------------ .../delete_config_bfd_remote_ip_urlbuilder.go | 113 ------ loxinet/cluster.go | 10 + 5 files changed, 10 insertions(+), 626 deletions(-) delete mode 100644 api/restapi/operations/delete_config_bfd_remote_ip.go delete mode 100644 api/restapi/operations/delete_config_bfd_remote_ip_parameters.go delete mode 100644 api/restapi/operations/delete_config_bfd_remote_ip_responses.go delete mode 100644 api/restapi/operations/delete_config_bfd_remote_ip_urlbuilder.go diff --git a/api/restapi/operations/delete_config_bfd_remote_ip.go b/api/restapi/operations/delete_config_bfd_remote_ip.go deleted file mode 100644 index 00154f4e..00000000 --- a/api/restapi/operations/delete_config_bfd_remote_ip.go +++ /dev/null @@ -1,58 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" -) - -// DeleteConfigBfdRemoteIPHandlerFunc turns a function with the right signature into a delete config bfd remote IP handler -type DeleteConfigBfdRemoteIPHandlerFunc func(DeleteConfigBfdRemoteIPParams) middleware.Responder - -// Handle executing the request and returning a response -func (fn DeleteConfigBfdRemoteIPHandlerFunc) Handle(params DeleteConfigBfdRemoteIPParams) middleware.Responder { - return fn(params) -} - -// DeleteConfigBfdRemoteIPHandler interface for that can handle valid delete config bfd remote IP params -type DeleteConfigBfdRemoteIPHandler interface { - Handle(DeleteConfigBfdRemoteIPParams) middleware.Responder -} - -// NewDeleteConfigBfdRemoteIP creates a new http.Handler for the delete config bfd remote IP operation -func NewDeleteConfigBfdRemoteIP(ctx *middleware.Context, handler DeleteConfigBfdRemoteIPHandler) *DeleteConfigBfdRemoteIP { - return &DeleteConfigBfdRemoteIP{Context: ctx, Handler: handler} -} - -/* - DeleteConfigBfdRemoteIP swagger:route DELETE /config/bfd/{remote_ip} deleteConfigBfdRemoteIp - -# Delete a BFD session - -Delete a BFD session -*/ -type DeleteConfigBfdRemoteIP struct { - Context *middleware.Context - Handler DeleteConfigBfdRemoteIPHandler -} - -func (o *DeleteConfigBfdRemoteIP) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewDeleteConfigBfdRemoteIPParams() - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/api/restapi/operations/delete_config_bfd_remote_ip_parameters.go b/api/restapi/operations/delete_config_bfd_remote_ip_parameters.go deleted file mode 100644 index 20d4bbe4..00000000 --- a/api/restapi/operations/delete_config_bfd_remote_ip_parameters.go +++ /dev/null @@ -1,101 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewDeleteConfigBfdRemoteIPParams creates a new DeleteConfigBfdRemoteIPParams object -// -// There are no default values defined in the spec. -func NewDeleteConfigBfdRemoteIPParams() DeleteConfigBfdRemoteIPParams { - - return DeleteConfigBfdRemoteIPParams{} -} - -// DeleteConfigBfdRemoteIPParams contains all the bound params for the delete config bfd remote IP operation -// typically these are obtained from a http.Request -// -// swagger:parameters DeleteConfigBfdRemoteIP -type DeleteConfigBfdRemoteIPParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Cluster instance name - In: query - */ - Instance *string - /*Remote IP address - Required: true - In: path - */ - RemoteIP string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewDeleteConfigBfdRemoteIPParams() beforehand. -func (o *DeleteConfigBfdRemoteIPParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - qInstance, qhkInstance, _ := qs.GetOK("instance") - if err := o.bindInstance(qInstance, qhkInstance, route.Formats); err != nil { - res = append(res, err) - } - - rRemoteIP, rhkRemoteIP, _ := route.Params.GetOK("remote_ip") - if err := o.bindRemoteIP(rRemoteIP, rhkRemoteIP, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindInstance binds and validates parameter Instance from query. -func (o *DeleteConfigBfdRemoteIPParams) bindInstance(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Instance = &raw - - return nil -} - -// bindRemoteIP binds and validates parameter RemoteIP from path. -func (o *DeleteConfigBfdRemoteIPParams) bindRemoteIP(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.RemoteIP = raw - - return nil -} diff --git a/api/restapi/operations/delete_config_bfd_remote_ip_responses.go b/api/restapi/operations/delete_config_bfd_remote_ip_responses.go deleted file mode 100644 index d9ccb53f..00000000 --- a/api/restapi/operations/delete_config_bfd_remote_ip_responses.go +++ /dev/null @@ -1,354 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/loxilb-io/loxilb/api/models" -) - -// DeleteConfigBfdRemoteIPNoContentCode is the HTTP code returned for type DeleteConfigBfdRemoteIPNoContent -const DeleteConfigBfdRemoteIPNoContentCode int = 204 - -/* -DeleteConfigBfdRemoteIPNoContent OK - -swagger:response deleteConfigBfdRemoteIpNoContent -*/ -type DeleteConfigBfdRemoteIPNoContent struct { -} - -// NewDeleteConfigBfdRemoteIPNoContent creates DeleteConfigBfdRemoteIPNoContent with default headers values -func NewDeleteConfigBfdRemoteIPNoContent() *DeleteConfigBfdRemoteIPNoContent { - - return &DeleteConfigBfdRemoteIPNoContent{} -} - -// WriteResponse to the client -func (o *DeleteConfigBfdRemoteIPNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// DeleteConfigBfdRemoteIPBadRequestCode is the HTTP code returned for type DeleteConfigBfdRemoteIPBadRequest -const DeleteConfigBfdRemoteIPBadRequestCode int = 400 - -/* -DeleteConfigBfdRemoteIPBadRequest Malformed arguments for API call - -swagger:response deleteConfigBfdRemoteIpBadRequest -*/ -type DeleteConfigBfdRemoteIPBadRequest struct { - - /* - In: Body - */ - Payload *models.Error `json:"body,omitempty"` -} - -// NewDeleteConfigBfdRemoteIPBadRequest creates DeleteConfigBfdRemoteIPBadRequest with default headers values -func NewDeleteConfigBfdRemoteIPBadRequest() *DeleteConfigBfdRemoteIPBadRequest { - - return &DeleteConfigBfdRemoteIPBadRequest{} -} - -// WithPayload adds the payload to the delete config bfd remote Ip bad request response -func (o *DeleteConfigBfdRemoteIPBadRequest) WithPayload(payload *models.Error) *DeleteConfigBfdRemoteIPBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the delete config bfd remote Ip bad request response -func (o *DeleteConfigBfdRemoteIPBadRequest) SetPayload(payload *models.Error) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *DeleteConfigBfdRemoteIPBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// DeleteConfigBfdRemoteIPUnauthorizedCode is the HTTP code returned for type DeleteConfigBfdRemoteIPUnauthorized -const DeleteConfigBfdRemoteIPUnauthorizedCode int = 401 - -/* -DeleteConfigBfdRemoteIPUnauthorized Invalid authentication credentials - -swagger:response deleteConfigBfdRemoteIpUnauthorized -*/ -type DeleteConfigBfdRemoteIPUnauthorized struct { - - /* - In: Body - */ - Payload *models.Error `json:"body,omitempty"` -} - -// NewDeleteConfigBfdRemoteIPUnauthorized creates DeleteConfigBfdRemoteIPUnauthorized with default headers values -func NewDeleteConfigBfdRemoteIPUnauthorized() *DeleteConfigBfdRemoteIPUnauthorized { - - return &DeleteConfigBfdRemoteIPUnauthorized{} -} - -// WithPayload adds the payload to the delete config bfd remote Ip unauthorized response -func (o *DeleteConfigBfdRemoteIPUnauthorized) WithPayload(payload *models.Error) *DeleteConfigBfdRemoteIPUnauthorized { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the delete config bfd remote Ip unauthorized response -func (o *DeleteConfigBfdRemoteIPUnauthorized) SetPayload(payload *models.Error) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *DeleteConfigBfdRemoteIPUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(401) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// DeleteConfigBfdRemoteIPForbiddenCode is the HTTP code returned for type DeleteConfigBfdRemoteIPForbidden -const DeleteConfigBfdRemoteIPForbiddenCode int = 403 - -/* -DeleteConfigBfdRemoteIPForbidden Capacity insufficient - -swagger:response deleteConfigBfdRemoteIpForbidden -*/ -type DeleteConfigBfdRemoteIPForbidden struct { - - /* - In: Body - */ - Payload *models.Error `json:"body,omitempty"` -} - -// NewDeleteConfigBfdRemoteIPForbidden creates DeleteConfigBfdRemoteIPForbidden with default headers values -func NewDeleteConfigBfdRemoteIPForbidden() *DeleteConfigBfdRemoteIPForbidden { - - return &DeleteConfigBfdRemoteIPForbidden{} -} - -// WithPayload adds the payload to the delete config bfd remote Ip forbidden response -func (o *DeleteConfigBfdRemoteIPForbidden) WithPayload(payload *models.Error) *DeleteConfigBfdRemoteIPForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the delete config bfd remote Ip forbidden response -func (o *DeleteConfigBfdRemoteIPForbidden) SetPayload(payload *models.Error) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *DeleteConfigBfdRemoteIPForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// DeleteConfigBfdRemoteIPNotFoundCode is the HTTP code returned for type DeleteConfigBfdRemoteIPNotFound -const DeleteConfigBfdRemoteIPNotFoundCode int = 404 - -/* -DeleteConfigBfdRemoteIPNotFound Resource not found - -swagger:response deleteConfigBfdRemoteIpNotFound -*/ -type DeleteConfigBfdRemoteIPNotFound struct { - - /* - In: Body - */ - Payload *models.Error `json:"body,omitempty"` -} - -// NewDeleteConfigBfdRemoteIPNotFound creates DeleteConfigBfdRemoteIPNotFound with default headers values -func NewDeleteConfigBfdRemoteIPNotFound() *DeleteConfigBfdRemoteIPNotFound { - - return &DeleteConfigBfdRemoteIPNotFound{} -} - -// WithPayload adds the payload to the delete config bfd remote Ip not found response -func (o *DeleteConfigBfdRemoteIPNotFound) WithPayload(payload *models.Error) *DeleteConfigBfdRemoteIPNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the delete config bfd remote Ip not found response -func (o *DeleteConfigBfdRemoteIPNotFound) SetPayload(payload *models.Error) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *DeleteConfigBfdRemoteIPNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// DeleteConfigBfdRemoteIPConflictCode is the HTTP code returned for type DeleteConfigBfdRemoteIPConflict -const DeleteConfigBfdRemoteIPConflictCode int = 409 - -/* -DeleteConfigBfdRemoteIPConflict Resource Conflict. BFD session already exists - -swagger:response deleteConfigBfdRemoteIpConflict -*/ -type DeleteConfigBfdRemoteIPConflict struct { - - /* - In: Body - */ - Payload *models.Error `json:"body,omitempty"` -} - -// NewDeleteConfigBfdRemoteIPConflict creates DeleteConfigBfdRemoteIPConflict with default headers values -func NewDeleteConfigBfdRemoteIPConflict() *DeleteConfigBfdRemoteIPConflict { - - return &DeleteConfigBfdRemoteIPConflict{} -} - -// WithPayload adds the payload to the delete config bfd remote Ip conflict response -func (o *DeleteConfigBfdRemoteIPConflict) WithPayload(payload *models.Error) *DeleteConfigBfdRemoteIPConflict { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the delete config bfd remote Ip conflict response -func (o *DeleteConfigBfdRemoteIPConflict) SetPayload(payload *models.Error) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *DeleteConfigBfdRemoteIPConflict) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(409) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// DeleteConfigBfdRemoteIPInternalServerErrorCode is the HTTP code returned for type DeleteConfigBfdRemoteIPInternalServerError -const DeleteConfigBfdRemoteIPInternalServerErrorCode int = 500 - -/* -DeleteConfigBfdRemoteIPInternalServerError Internal service error - -swagger:response deleteConfigBfdRemoteIpInternalServerError -*/ -type DeleteConfigBfdRemoteIPInternalServerError struct { - - /* - In: Body - */ - Payload *models.Error `json:"body,omitempty"` -} - -// NewDeleteConfigBfdRemoteIPInternalServerError creates DeleteConfigBfdRemoteIPInternalServerError with default headers values -func NewDeleteConfigBfdRemoteIPInternalServerError() *DeleteConfigBfdRemoteIPInternalServerError { - - return &DeleteConfigBfdRemoteIPInternalServerError{} -} - -// WithPayload adds the payload to the delete config bfd remote Ip internal server error response -func (o *DeleteConfigBfdRemoteIPInternalServerError) WithPayload(payload *models.Error) *DeleteConfigBfdRemoteIPInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the delete config bfd remote Ip internal server error response -func (o *DeleteConfigBfdRemoteIPInternalServerError) SetPayload(payload *models.Error) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *DeleteConfigBfdRemoteIPInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// DeleteConfigBfdRemoteIPServiceUnavailableCode is the HTTP code returned for type DeleteConfigBfdRemoteIPServiceUnavailable -const DeleteConfigBfdRemoteIPServiceUnavailableCode int = 503 - -/* -DeleteConfigBfdRemoteIPServiceUnavailable Maintanence mode - -swagger:response deleteConfigBfdRemoteIpServiceUnavailable -*/ -type DeleteConfigBfdRemoteIPServiceUnavailable struct { - - /* - In: Body - */ - Payload *models.Error `json:"body,omitempty"` -} - -// NewDeleteConfigBfdRemoteIPServiceUnavailable creates DeleteConfigBfdRemoteIPServiceUnavailable with default headers values -func NewDeleteConfigBfdRemoteIPServiceUnavailable() *DeleteConfigBfdRemoteIPServiceUnavailable { - - return &DeleteConfigBfdRemoteIPServiceUnavailable{} -} - -// WithPayload adds the payload to the delete config bfd remote Ip service unavailable response -func (o *DeleteConfigBfdRemoteIPServiceUnavailable) WithPayload(payload *models.Error) *DeleteConfigBfdRemoteIPServiceUnavailable { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the delete config bfd remote Ip service unavailable response -func (o *DeleteConfigBfdRemoteIPServiceUnavailable) SetPayload(payload *models.Error) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *DeleteConfigBfdRemoteIPServiceUnavailable) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(503) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/api/restapi/operations/delete_config_bfd_remote_ip_urlbuilder.go b/api/restapi/operations/delete_config_bfd_remote_ip_urlbuilder.go deleted file mode 100644 index 68d5b492..00000000 --- a/api/restapi/operations/delete_config_bfd_remote_ip_urlbuilder.go +++ /dev/null @@ -1,113 +0,0 @@ -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// DeleteConfigBfdRemoteIPURL generates an URL for the delete config bfd remote IP operation -type DeleteConfigBfdRemoteIPURL struct { - RemoteIP string - - Instance *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *DeleteConfigBfdRemoteIPURL) WithBasePath(bp string) *DeleteConfigBfdRemoteIPURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *DeleteConfigBfdRemoteIPURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *DeleteConfigBfdRemoteIPURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/config/bfd/{remote_ip}" - - remoteIP := o.RemoteIP - if remoteIP != "" { - _path = strings.Replace(_path, "{remote_ip}", remoteIP, -1) - } else { - return nil, errors.New("remoteIp is required on DeleteConfigBfdRemoteIPURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/netlox/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var instanceQ string - if o.Instance != nil { - instanceQ = *o.Instance - } - if instanceQ != "" { - qs.Set("instance", instanceQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *DeleteConfigBfdRemoteIPURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *DeleteConfigBfdRemoteIPURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *DeleteConfigBfdRemoteIPURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on DeleteConfigBfdRemoteIPURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on DeleteConfigBfdRemoteIPURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *DeleteConfigBfdRemoteIPURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/loxinet/cluster.go b/loxinet/cluster.go index f6de07b3..a0359844 100644 --- a/loxinet/cluster.go +++ b/loxinet/cluster.go @@ -288,6 +288,16 @@ func (h *CIStateH) CIBFDSessionAdd(bm cmn.BFDMod) (int, error) { return -1, errors.New("cluster instance not found") } + ip := net.ParseIP(string(bm.RemoteIP)) + if ip == nil { + return -1, errors.New("remoteIP address malformed") + } + + myIP := net.ParseIP(string(bm.SourceIP)) + if myIP == nil { + return -1, errors.New("source address malformed") + } + if !h.SpawnKa { tk.LogIt(tk.LogInfo, "[CLUSTER] Cluster Instance %s starting BFD..\n", bm.Instance) h.SpawnKa = true From 7ed7c766eeee9d3ea63e790b4ea170c78b5e0378 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 12 Mar 2024 13:09:21 +0900 Subject: [PATCH 2/6] single node k3s calico with loxilb docker --- cicd/docker-k3s-calico/Vagrantfile | 29 ++ cicd/docker-k3s-calico/common.sh | 555 +++++++++++++++++++++++++ cicd/docker-k3s-calico/config.sh | 3 + cicd/docker-k3s-calico/k3s_common.sh | 36 ++ cicd/docker-k3s-calico/kube-loxilb.yml | 129 ++++++ cicd/docker-k3s-calico/loxilb-llb1 | 1 + cicd/docker-k3s-calico/loxilb.sh | 55 +++ cicd/docker-k3s-calico/rmconfig.sh | 2 + cicd/docker-k3s-calico/tcp-svc-lb.yml | 30 ++ cicd/docker-k3s-calico/validation.sh | 69 +++ 10 files changed, 909 insertions(+) create mode 100644 cicd/docker-k3s-calico/Vagrantfile create mode 100644 cicd/docker-k3s-calico/common.sh create mode 100755 cicd/docker-k3s-calico/config.sh create mode 100644 cicd/docker-k3s-calico/k3s_common.sh create mode 100644 cicd/docker-k3s-calico/kube-loxilb.yml create mode 100644 cicd/docker-k3s-calico/loxilb-llb1 create mode 100644 cicd/docker-k3s-calico/loxilb.sh create mode 100755 cicd/docker-k3s-calico/rmconfig.sh create mode 100644 cicd/docker-k3s-calico/tcp-svc-lb.yml create mode 100755 cicd/docker-k3s-calico/validation.sh diff --git a/cicd/docker-k3s-calico/Vagrantfile b/cicd/docker-k3s-calico/Vagrantfile new file mode 100644 index 00000000..c992d21d --- /dev/null +++ b/cicd/docker-k3s-calico/Vagrantfile @@ -0,0 +1,29 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +loxilbs = (ENV['LOXILBS'] || "1").to_i +eps = (ENV['LOXILBS'] || "1").to_i +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = "0.7.1" +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + (1..loxilbs).each do |node_number| + config.vm.define "llb#{node_number}" do |loxilb| + loxilb.vm.hostname = "llb#{node_number}" + ip = node_number + 245 + loxilb.vm.network :private_network, ip: "192.168.163.#{ip}", :netmask => "255.255.255.0" + loxilb.vm.provision :shell, :path => "loxilb.sh" + loxilb.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 16000] + vbox.customize ["modifyvm", :id, "--cpus", 8] + vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + end + end + end +end diff --git a/cicd/docker-k3s-calico/common.sh b/cicd/docker-k3s-calico/common.sh new file mode 100644 index 00000000..b82f6e0e --- /dev/null +++ b/cicd/docker-k3s-calico/common.sh @@ -0,0 +1,555 @@ +#!/bin/bash + +if [[ "$1" == "init" ]]; then + pull_dockers +fi + +hn="netns" +pid="" +vrn="/var/run/" +hexec="sudo ip netns exec " +dexec="sudo docker exec -i " +hns="sudo ip netns " +hexist="$vrn$hn" +lxdocker="ghcr.io/loxilb-io/loxilb:latest" +var=$(lsb_release -r | cut -f2) +if [[ $var == *"22.04"* ]];then + lxdocker="ghcr.io/loxilb-io/loxilb:latestU22" +fi + +loxilbs=() + +## Given a docker name(arg1), return its pid +get_docker_pid() { + id=`docker ps -f name=$1| grep -w $1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + pid=`docker inspect -f '{{.State.Pid}}' $id` +} + +## Pull all necessary dockers for testbed +pull_dockers() { + ## loxilb docker + docker pull $lxdocker + ## Host docker + docker pull eyes852/ubuntu-iperf-test:0.5 + ## BGP host docker + docker pull ewindisch/quagga + ## Keepalive docker + docker pull osixia/keepalived:2.0.20 +} + +## Creates a docker host +## arg1 - "loxilb"|"host" +## arg2 - instance-name +spawn_docker_host() { + POSITIONAL_ARGS=() + local bpath + local kpath + local ka + local bgp + while [[ $# -gt 0 ]]; do + case "$1" in + -t | --dock-type ) + dtype="$2" + shift 2 + ;; + -d | --dock-name ) + dname="$2" + shift 2 + ;; + -b | --with-bgp ) + if [[ "$2" == "yes" ]]; then + bgp=$2 + fi + shift 2 + ;; + -c | --bgp-config ) + bpath="$2" + bgp="yes" + shift 2 + ;; + -k | --with-ka ) + ka="in" + if [[ "$2" == "out" ]]; then + ka=$2 + fi + shift 2 + ;; + -d | --ka-config ) + kpath="$2" + if [[ -z ${ka+x} ]]; then + ka="in" + fi + shift 2 + ;; + -*|--*) + echo "Unknown option $1" + exit + ;; + esac + done + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + echo "Spawning $dname($dtype)" >&2 + if [[ "$dtype" == "loxilb" ]]; then + loxilbs+=("$dname") + if [[ "$pick_config" == "yes" ]]; then + echo "$dname will pick config from $(pwd)/${dname}_config" + loxilb_config="-v $(pwd)/${dname}_config:/etc/loxilb/" + fi + if [[ "$bgp" == "yes" ]]; then + bgp_opts="-b" + if [[ ! -z "$bpath" ]]; then + bgp_conf="-v $bpath:/etc/gobgp/" + fi + fi + if [[ "$dname" == "llb1" ]]; then + cluster_opts=" --cluster=172.17.0.3 --self=0" + elif [[ "$dname" == "llb2" ]]; then + cluster_opts=" --cluster=172.17.0.2 --self=1" + fi + + if [[ ! -z ${ka+x} ]]; then + sudo mkdir -p /etc/shared/$dname/ + if [[ "$ka" == "in" ]];then + ka_opts="-k in" + if [[ ! -z "$kpath" ]]; then + ka_conf="-v $kpath:/etc/keepalived/" + fi + fi + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config $ka_conf --name $dname $lxdocker + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts + + if [[ "$ka" == "out" ]];then + ka_opts="-k out" + if [[ ! -z "$kpath" ]]; then + ka_conf="-v $kpath:/container/service/keepalived/assets/" + fi + + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit --network=container:$dname $ka_conf -v /etc/shared/$dname:/etc/shared --name ka_$dname osixia/keepalived:2.0.20 + fi + else + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts + fi + elif [[ "$dtype" == "host" ]]; then + if [[ ! -z "$bpath" ]]; then + bgp_conf="--volume $bpath:/etc/quagga" + fi + if [[ "$bgp" == "yes" || ! -z "$bpath" ]]; then + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit $bgp_conf --name $dname ewindisch/quagga + else + docker run -u root --cap-add SYS_ADMIN -dit --name $dname eyes852/ubuntu-iperf-test:0.5 + fi + fi + + pid="" + + sleep 2 + get_docker_pid $dname + echo $pid + if [ ! -f "$hexist/$dname" -a "$pid" != "" ]; then + sudo mkdir -p /var/run/netns + sudo touch /var/run/netns/$dname + #echo "sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$2" + sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$dname + fi + + $hexec $dname ifconfig lo up + $hexec $dname sysctl net.ipv6.conf.all.disable_ipv6=1 2>&1 >> /dev/null + #$hexec $dname sysctl net.ipv4.conf.all.arp_accept=1 2>&1 >> /dev/null + $hexec $dname sysctl net.ipv4.conf.eth0.arp_ignore=2 2>&1 >> /dev/null +} + +## Deletes a docker host +## arg1 - hostname +delete_docker_host() { + id=`docker ps -f name=$1| grep -w $1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + if [ "$id" != "" ]; then + docker stop $1 2>&1 >> /dev/null + hd="true" + ka=`docker ps -f name=ka_$1| grep -w ka_$1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + loxilbs=( "${loxilbs[@]/$1}" ) + if [ "$ka" != "" ]; then + docker stop ka_$1 2>&1 >> /dev/null + docker rm ka_$1 2>&1 >> /dev/null + fi + fi + if [ -f "$hexist/$1" ]; then + $hns del $1 + sudo rm -fr "$hexist/$1" 2>&1 >> /dev/null + fi + if [ "$id" != "" ]; then + docker rm $1 2>&1 >> /dev/null + fi +} + +## Connects two docker hosts +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - mtu +connect_docker_hosts() { + link1=e$1$2 + link2=e$2$1 + + mtu="9000" + if [[ $# -gt 2 ]]; then + mtu=$3 + fi + + #echo $link1 $link2 + sudo ip -n $1 link add $link1 type veth peer name $link2 netns $2 + sudo ip -n $1 link set $link1 mtu $mtu up + sudo ip -n $2 link set $link2 mtu $mtu up +} + +## arg1 - hostname1 +## arg2 - hostname2 +disconnect_docker_hosts() { + link1=e$1$2 + link2=e$2$1 + # echo $link1 $link2 + if [ -f "$hexist/$1" ]; then + ifexist1=`sudo ip -n $1 link show $link1 | grep -w $link1` + if [ "chk$ifexist1" != "chk" ]; then + sudo ip -n $1 link set $link1 down 2>&1 >> /dev/null + sudo ip -n $1 link del $link1 2>&1 >> /dev/null + fi + fi + + if [ -f "$hexist/$2" ]; then + ifexist2=`sudo ip -n $2 link show | grep -w $link2` + if [ "chk$ifexist2" != "chk" ]; then + sudo ip -n $2 link set $link2 down 2>&1 >> /dev/null + sudo ip -n $2 link del $link2 2>&1 >> /dev/null + fi + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - ip_addr +## arg4 - gw +config_docker_host() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --ptype) + local ptype="$2" + shift + shift + ;; + --id) + local xid="$2" + shift + shift + ;; + --addr) + local addr="$2" + shift + shift + ;; + --gw) + local gw="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + + link1=e$h1$h2 + link2=e$h2$h1 + #echo "$h1:$link1->$h2:$link2" + + #if [[ -n "${loxilbs[$h1]}" && "$pick_config" == "yes" ]]; then + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + if [[ "$ptype" == "phy" ]]; then + sudo ip -n $h1 addr add $addr dev $link1 + elif [[ "$ptype" == "vlan" ]]; then + sudo ip -n $h1 addr add $addr dev vlan$xid + elif [[ "$ptype" == "vxlan" ]]; then + sudo ip -n $h1 addr add $addr dev vxlan$xid + elif [[ "$ptype" == "trunk" ]]; then + trunk="bond$xid" + sudo ip -n $h1 link set $link1 down + sudo ip -n $h1 link add $trunk type bond + sudo ip -n $h1 link set $link1 master $trunk + sudo ip -n $h1 link set $link1 up + sudo ip -n $h1 link set $trunk up + + sudo ip -n $h2 link set $link2 down + sudo ip -n $h2 link add $trunk type bond + sudo ip -n $h2 link set $link2 master $trunk + sudo ip -n $h2 link set $link2 up + sudo ip -n $h2 link set $trunk up + + sudo ip -n $h1 addr add $addr dev bond$xid + if [[ "$gw" != "" ]]; then + sudo ip -n $h2 addr add $gw/24 dev bond$xid + sudo ip -n $h1 route add default via $gw proto static + fi + else + echo "Check port-type" + fi + + if [[ "$gw" != "" ]]; then + sudo ip -n $h1 route del default 2>&1 >> /dev/null + sudo ip -n $h1 route add default via $gw + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - vlan +## arg4 - tagged/untagged +create_docker_host_vlan() { + local addr="" + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --ptype) + local ptype="$2" + shift + shift + ;; + --id) + local vid="$2" + shift + shift + ;; + --addr) + addr="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + if [[ "$ptype" == "tagged" ]]; then + brport="$link1.$vid" + sudo ip -n $h1 link add link $link1 name $brport type vlan id $vid + sudo ip -n $h1 link set $brport up + else + brport=$link1 + fi + + sudo ip -n $h1 link add vlan$vid type bridge 2>&1 | true + sudo ip -n $h1 link set $brport master vlan$vid + sudo ip -n $h1 link set vlan$vid up + if [[ "$addr" != "" ]]; then + sudo ip -n $h1 addr add $addr dev vlan$vid + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - vxlan-id +## arg4 - phy/vlan +## arg5 - local ip if arg4 is phy/vlan-id if arg4 is vlan +## arg6 - local ip if arg4 is vlan +create_docker_host_vxlan() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --uif) + local uifType="$2" + shift + shift + ;; + --vid) + local vid="$2" + shift + shift + ;; + --pvid) + local pvid="$2" + shift + shift + ;; + --id) + local vxid="$2" + shift + shift + ;; + --ep) + local ep="$2" + shift + shift + ;; + --lip) + local lip="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + if [[ "$uifType" == "phy" ]]; then + sudo ip -n $h1 link add vxlan$vxid type vxlan id $vxid local $lip dev $link1 dstport 4789 + sudo ip -n $h1 link set vxlan$vxid up + elif [[ "$uifType" == "vlan" ]]; then + sudo ip -n $h1 link add vxlan$vxid type vxlan id $vxid local $lip dev vlan$vid dstport 4789 + sudo ip -n $h1 link set vxlan$vxid up + fi + + if [[ "$pvid" != "" ]]; then + sudo ip -n $h1 link add vlan$pvid type bridge 2>&1 | true + sudo ip -n $h1 link set vxlan$vxid master vlan$pvid + sudo ip -n $h1 link set vlan$pvid up + fi + + if [[ "$ep" != "" ]]; then + sudo bridge -n $h1 fdb append 00:00:00:00:00:00 dst $ep dev vxlan$vxid + fi + +} + +## arg1 - hostname1 +## arg2 - hostname2 +create_docker_host_cnbridge() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + brport=$link1 + + sudo ip -n $h1 link add br$h1 type bridge 2>&1 | true + sudo ip -n $h1 link set $brport master br$h1 + sudo ip -n $h1 link set br$h1 up +} + +#Arg1: host name +#Arg2: --:: +#Arg3: --endpoints::,.. +function create_lb_rule() { + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + args=( "$@" ) + args=( "${args[@]/$1}" ) + echo "$1: loxicmd create lb ${args[*]}" + $dexec $1 loxicmd create lb ${args[*]} + + hook=$($dexec llb1 ntc filter show dev eth0 ingress | grep tc_packet_hook) + if [[ $hook != *"tc_packet_hook"* ]]; then + echo "ERROR : No hook point found"; + exit 1 + fi +} + +#Arg1: host name +#Arg2: +#Arg3: +function add_route() { + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + echo "$1: ip route add $2 via $3 proto static" + $hexec $1 ip route add $2 via $3 proto static +} diff --git a/cicd/docker-k3s-calico/config.sh b/cicd/docker-k3s-calico/config.sh new file mode 100755 index 00000000..6b8ee48e --- /dev/null +++ b/cicd/docker-k3s-calico/config.sh @@ -0,0 +1,3 @@ +#!/bin/bash +vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f +vagrant up diff --git a/cicd/docker-k3s-calico/k3s_common.sh b/cicd/docker-k3s-calico/k3s_common.sh new file mode 100644 index 00000000..5a0dfe99 --- /dev/null +++ b/cicd/docker-k3s-calico/k3s_common.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +function wait_cluster_ready { + Res=$(sudo kubectl $KUBECONFIG get pods -A | + while IFS= read -r line; do + if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then + echo "not ready" + return + fi + done) + if [[ $Res == *"not ready"* ]]; then + return 1 + fi + return 0 +} + +function wait_cluster_ready_full { + i=1 + nr=0 + for ((;;)) do + wait_cluster_ready + nr=$? + if [[ $nr == 0 ]]; then + echo "Cluster is ready" + break + fi + i=$(( $i + 1 )) + if [[ $i -ge 40 ]]; then + echo "Cluster is not ready.Giving up" + exit 1 + fi + echo "Cluster is not ready...." + sleep 10 + done +} + diff --git a/cicd/docker-k3s-calico/kube-loxilb.yml b/cicd/docker-k3s-calico/kube-loxilb.yml new file mode 100644 index 00000000..05ea21ab --- /dev/null +++ b/cicd/docker-k3s-calico/kube-loxilb.yml @@ -0,0 +1,129 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + - --loxiURL=http://172.17.0.2:11111 + - --externalCIDR=192.168.163.247/32 + #- --monitor + #- --setBGP + #- --setLBMode=1 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/docker-k3s-calico/loxilb-llb1 b/cicd/docker-k3s-calico/loxilb-llb1 new file mode 100644 index 00000000..c212a9b1 --- /dev/null +++ b/cicd/docker-k3s-calico/loxilb-llb1 @@ -0,0 +1 @@ +192.168.163.246 diff --git a/cicd/docker-k3s-calico/loxilb.sh b/cicd/docker-k3s-calico/loxilb.sh new file mode 100644 index 00000000..9a5f42a2 --- /dev/null +++ b/cicd/docker-k3s-calico/loxilb.sh @@ -0,0 +1,55 @@ +source /vagrant/common.sh +source /vagrant/k3s_common.sh + +export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +## Set promisc mode for mac-vlan to work +sudo ifconfig eth1 promisc + +apt-get update +apt-get install -y software-properties-common ethtool +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get install -y docker-ce + +echo "Start loxilb installation" +sudo docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged --entrypoint /root/loxilb-io/loxilb/loxilb -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest + +# Create mac-vlan on top of underlying eth1 interface +docker network create -d macvlan -o parent=eth1 --subnet 192.168.163.0/24 --gateway 192.168.163.1 --aux-address 'host=192.168.163.252' llbnet + +# Assign mac-vlan to loxilb docker with specified IP (which will be used as LB VIP) +docker network connect llbnet loxilb --ip=192.168.163.247 + +# Add iptables rule to allow traffic from source IP(192.168.163.1) to loxilb +sudo iptables -A DOCKER -s 192.168.163.1 -j ACCEPT + +echo "Start K3s installation" + +# Install K3s with Calico and IPVS +sudo apt-get -y install ipset ipvsadm +curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik,metrics-server,servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --disable-network-policy" K3S_KUBECONFIG_MODE="644" sh -s - server --kube-proxy-arg proxy-mode=ipvs +sleep 10 + +# Install Calico +kubectl $KUBECONFIG create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml + +kubectl $KUBECONFIG create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml + +# Check kubectl works +kubectl $KUBECONFIG get pods -A + +# Remove taints in k3s if any (usually happens if started without cloud-manager) +kubectl $KUBECONFIG taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- + +echo "End K3s installation" +sleep 60 + +kubectl apply -f /vagrant/kube-loxilb.yml +sleep 60 +kubectl apply -f /vagrant/tcp-svc-lb.yml + +# Wait for cluster to be ready +wait_cluster_ready_full + +echo $LOXILB_IP > /vagrant/loxilb-$(hostname) diff --git a/cicd/docker-k3s-calico/rmconfig.sh b/cicd/docker-k3s-calico/rmconfig.sh new file mode 100755 index 00000000..e3e17122 --- /dev/null +++ b/cicd/docker-k3s-calico/rmconfig.sh @@ -0,0 +1,2 @@ +#!/bin/bash +vagrant destroy -f llb1 diff --git a/cicd/docker-k3s-calico/tcp-svc-lb.yml b/cicd/docker-k3s-calico/tcp-svc-lb.yml new file mode 100644 index 00000000..ce561486 --- /dev/null +++ b/cicd/docker-k3s-calico/tcp-svc-lb.yml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + name: tcp-lb-onearm + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "onearm" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: tcp-onearm-test + ports: + - port: 56002 + targetPort: 80 + nodePort: 30001 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: tcp-onearm-test + labels: + what: tcp-onearm-test +spec: + containers: + - name: tcp-onearm-test + image: ghcr.io/loxilb-io/nginx:stable + ports: + - containerPort: 80 diff --git a/cicd/docker-k3s-calico/validation.sh b/cicd/docker-k3s-calico/validation.sh new file mode 100755 index 00000000..ca6c4b7d --- /dev/null +++ b/cicd/docker-k3s-calico/validation.sh @@ -0,0 +1,69 @@ +#!/bin/bash +source ../common.sh +echo docker-k3s-calico + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +for((i=0; i<120; i++)) +do + extLB=$(vagrant ssh llb1 -c 'sudo kubectl get svc' 2> /dev/null | grep tcp-lb"") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find tcp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +#sleep 30 +echo Service IP : $extIP +echo $extIP + +echo -e "\nEnd Points List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo kubectl get endpoints -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nSVC List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo kubectl get svc' 2> /dev/null +echo "******************************************************************************" +echo -e "\nPod List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo kubectl get pods -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nLB List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nEP List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" + +echo -e "\nTEST RESULTS" +echo "******************************************************************************" + +echo -e "Command: curl --connect-time 10 http://192.168.163.247:56002'\n" +res=`curl -s --connect-time 10 http://192.168.163.247:56002` +echo "Result" +echo $res +if [[ "$res" == *"Welcome to nginx"* ]]; then + echo -e "\n\ndocker-k3s-calico TCP service (loxilb) [OK]" +else + echo -e "\n\ndocker-k3s-calico TCP service (loxilb) [NOK]" + exit 1 +fi + From e4714e40a087f6419143552d173ea46a0fcfa520 Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 12 Mar 2024 13:09:58 +0900 Subject: [PATCH 3/6] single node k3s calico with loxilb docker --- cicd/docker-k3s-calico/loxilb-llb1 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 cicd/docker-k3s-calico/loxilb-llb1 diff --git a/cicd/docker-k3s-calico/loxilb-llb1 b/cicd/docker-k3s-calico/loxilb-llb1 deleted file mode 100644 index c212a9b1..00000000 --- a/cicd/docker-k3s-calico/loxilb-llb1 +++ /dev/null @@ -1 +0,0 @@ -192.168.163.246 From 5e4a7f260367b7bb5973726de8e42688d188488e Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 12 Mar 2024 13:40:39 +0900 Subject: [PATCH 4/6] docker-k3s-cilium validation script updated --- cicd/docker-k3s-cilium/validation.sh | 84 +++++++++++++++++----------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/cicd/docker-k3s-cilium/validation.sh b/cicd/docker-k3s-cilium/validation.sh index b358f971..20ac3c0d 100755 --- a/cicd/docker-k3s-cilium/validation.sh +++ b/cicd/docker-k3s-cilium/validation.sh @@ -1,6 +1,6 @@ #!/bin/bash source ../common.sh -echo docker-k3s-lb +echo docker-k3s-cilium if [ "$1" ]; then KUBECONFIG="$1" @@ -9,41 +9,61 @@ fi # Set space as the delimiter IFS=' ' -sleep 30 -extIP="192.168.163.247" +for((i=0; i<120; i++)) +do + extLB=$(vagrant ssh llb1 -c 'sudo kubectl get svc' 2> /dev/null | grep tcp-lb"") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find tcp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +#sleep 30 +echo Service IP : $extIP echo $extIP -echo "Service Info" -vagrant ssh llb1 -c 'sudo kubectl get svc' -echo "LB Info" -vagrant ssh llb1 -c 'sudo docker exec -i loxilb loxicmd get lb -o wide' -echo "EP Info" -vagrant ssh llb1 -c 'sudo docker exec -i loxilb loxicmd get ep -o wide' - -print_debug_info() { - echo "llb1 route-info" - vagrant ssh llb1 -c 'ip route' - vagrant ssh llb1 -c 'sudo kubectl get pods -A' - vagrant ssh llb1 -c 'sudo kubectl get svc' - vagrant ssh llb1 -c 'sudo kubectl get nodes' -} - -sctp_darn -H 192.168.163.1 -h 192.168.163.247 -p 55003 -s < input > output -sleep 5 -exp="New connection, peer addresses -192.168.163.247:55003" - -res=`cat output | grep -A 1 "New connection, peer addresses"` +echo -e "\nEnd Points List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo kubectl get endpoints -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nSVC List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo kubectl get svc' 2> /dev/null +echo "******************************************************************************" +echo -e "\nPod List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo kubectl get pods -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nLB List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null +echo "******************************************************************************" +echo -e "\nEP List" +echo "******************************************************************************" +vagrant ssh llb1 -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null +echo "******************************************************************************" + +echo -e "\nTEST RESULTS" +echo "******************************************************************************" + +echo -e "Command: curl --connect-time 10 http://192.168.163.247:56002'\n" +res=`curl -s --connect-time 10 http://192.168.163.247:56002` echo "Result" echo $res -echo "Expected" -echo $exp -sudo rm -rf output -if [[ "$res" == "$exp" ]]; then - echo $res - echo "docker-k3s-lb SCTP service sctp-lb (loxilb) [OK]" +if [[ "$res" == *"Welcome to nginx"* ]]; then + echo -e "\n\ndocker-k3s-cilium TCP service (loxilb) [OK]" else - echo "docker-k3s-lb SCTP service sctp-lb (loxilb) [NOK]" - print_debug_info + echo -e "\n\ndocker-k3s-cilium TCP service (loxilb) [NOK]" exit 1 fi + From 7a0855299d756289f89712d69a414aab9fa0acdd Mon Sep 17 00:00:00 2001 From: UltraInstinct14 Date: Tue, 12 Mar 2024 15:51:06 +0900 Subject: [PATCH 5/6] chore: Reorganized README --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 480001c5..0df01912 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S - Hitless failover support might be essential for mission-critical applications - E2 might require SCTP-LB with OpenVPN bundled together -## Getting Started +## How-To Guides - [How-To : Deploy loxilb in K8s with kube-loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/kube-loxilb.md) - [How-To : Run in K8s with in-cluster mode](https://www.loxilb.io/post/k8s-nuances-of-in-cluster-external-service-lb-with-loxilb) - [How-To : High-availability with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ha-deploy.md) @@ -74,8 +74,7 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S - [How-To : Standalone configuration](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/cmd.md) - [How-To : debug](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md) -## Getting started with different K8s distributions & tools - +## Getting started with different K8s distributions/tools - [K3s : loxilb with default flannel](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s_quick_start_flannel.md) - [K3s : loxilb with cilium](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/quick_start_with_cilium.md) - [K0s : loxilb with default kube-router networking](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k0s_quick_start.md) From 0b7a1afbad64024fcf6d8e91680d618d038a57a1 Mon Sep 17 00:00:00 2001 From: UltraInstinct14 Date: Tue, 12 Mar 2024 16:06:30 +0900 Subject: [PATCH 6/6] chore: updated README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0df01912..6546d4a4 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S ## Getting started with different K8s distributions/tools - [K3s : loxilb with default flannel](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s_quick_start_flannel.md) +- [K3s : loxilb with calico](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s_quick_start_calico.md) - [K3s : loxilb with cilium](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/quick_start_with_cilium.md) - [K0s : loxilb with default kube-router networking](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k0s_quick_start.md)