PR - gh-483 : Initial support for ipvs compatibilty

.golangci.yml

@@ -0,0 +1,8 @@
+linters:
+  disable-all: true
+  enable:
+    - gofmt
+    - govet
+    - ineffassign
+    - misspell
+    - revive

api/loxinlp/ipvs.go

@@ -0,0 +1,170 @@
+/*
+ * Copyright (c) 2023 NetLOX Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package loxinlp
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"time"
+
+	"github.com/loxilb-io/ipvs"
+	cmn "github.com/loxilb-io/loxilb/common"
+	tk "github.com/loxilb-io/loxilib"
+)
+
+type ipVSKey struct {
+	Address  string
+	Protocol string
+	Port     uint16
+}
+
+type ipvsEndPoint struct {
+	EpIP   string
+	EpPort uint16
+	Weight uint8
+}
+
+type ipVSEntry struct {
+	Key       ipVSKey
+	Type      string
+	InValid   bool
+	EndPoints []ipvsEndPoint
+}
+
+type IpVSH struct {
+	RMap   map[ipVSKey]*ipVSEntry
+	ticker *time.Ticker
+	tDone  chan bool
+	handle *ipvs.Handle
+}
+
+var ipVSCtx *IpVSH
+
+func (ctx *IpVSH) BuildIpVSDB() []*ipVSEntry {
+
+	var ipVSList []*ipVSEntry
+	svcs, err := ctx.handle.GetServices()
+	if err != nil {
+		tk.LogIt(tk.LogError, "[ipvs] failed to get services\n")
+		return nil
+	}
+
+	for _, svc := range svcs {
+		var newEntry ipVSEntry
+
+		endPoints, err := ctx.handle.GetDestinations(svc)
+		if err != nil {
+			continue
+		}
+
+		newEntry.Type = svc.SchedName
+		if svc.SchedName != "rr" {
+			continue
+		}
+
+		proto := ""
+		if svc.Protocol == 1 {
+			proto = "icmp"
+		} else if svc.Protocol == 6 {
+			proto = "tcp"
+		} else if svc.Protocol == 17 {
+			proto = "udp"
+		} else if svc.Protocol == 132 {
+			proto = "sctp"
+		} else {
+			continue
+		}
+
+		key := ipVSKey{Address: svc.Address.String(), Protocol: proto, Port: svc.Port}
+		for _, endPoint := range endPoints {
+			newEntry.EndPoints = append(newEntry.EndPoints, ipvsEndPoint{EpIP: endPoint.Address.String(), EpPort: endPoint.Port, Weight: uint8(endPoint.Weight)})
+		}
+
+		if len(newEntry.EndPoints) != 0 {
+			if eEnt := ctx.RMap[key]; eEnt != nil {
+				if reflect.DeepEqual(eEnt.EndPoints, newEntry.EndPoints) {
+					eEnt.InValid = false
+					continue
+				}
+			}
+
+			newEntry.Key = key
+			ipVSList = append(ipVSList, &newEntry)
+		}
+	}
+	return ipVSList
+}
+
+func IpVSSync() {
+	for {
+		select {
+		case <-ipVSCtx.tDone:
+			return
+		case <-ipVSCtx.ticker.C:
+
+			for _, ent := range ipVSCtx.RMap {
+				ent.InValid = true
+			}
+
+			ipVSList := ipVSCtx.BuildIpVSDB()
+
+			for _, ent := range ipVSCtx.RMap {
+				if ent.InValid {
+					name := fmt.Sprintf("ipvs_%s:%d-%s", ent.Key.Address, ent.Key.Port, ent.Key.Protocol)
+					lbrule := cmn.LbRuleMod{Serv: cmn.LbServiceArg{ServIP: ent.Key.Address, ServPort: ent.Key.Port, Proto: ent.Key.Protocol, Sel: cmn.LbSelRr, Name: name}}
+					_, err := hooks.NetLbRuleDel(&lbrule)
+					if err != nil {
+						tk.LogIt(tk.LogError, "IPVS LB %v delete failed\n", ent.Key)
+					}
+					tk.LogIt(tk.LogInfo, "IPVS ent %v deleted\n", ent.Key)
+					delete(ipVSCtx.RMap, ent.Key)
+				}
+			}
+
+			for _, newEnt := range ipVSList {
+				name := fmt.Sprintf("ipvs_%s:%d-%s", newEnt.Key.Address, newEnt.Key.Port, newEnt.Key.Protocol)
+				lbrule := cmn.LbRuleMod{Serv: cmn.LbServiceArg{ServIP: newEnt.Key.Address, ServPort: newEnt.Key.Port, Proto: newEnt.Key.Protocol, Sel: cmn.LbSelRr, Name: name}}
+				for _, ep := range newEnt.EndPoints {
+					lbrule.Eps = append(lbrule.Eps, cmn.LbEndPointArg{EpIP: ep.EpIP, EpPort: ep.EpPort, Weight: 1})
+				}
+
+				_, err := hooks.NetLbRuleAdd(&lbrule)
+				if err != nil {
+					tk.LogIt(tk.LogError, "IPVS LB %v add failed\n", newEnt.Key)
+					continue
+				}
+				ipVSCtx.RMap[newEnt.Key] = newEnt
+				tk.LogIt(tk.LogError, "IPVS ent %v added\n", newEnt.Key)
+			}
+		}
+	}
+}
+
+func IpVSInit() {
+	ipVSCtx = new(IpVSH)
+	ipVSCtx.ticker = time.NewTicker(10 * time.Second)
+	ipVSCtx.RMap = make(map[ipVSKey]*ipVSEntry)
+	ipVSCtx.tDone = make(chan bool)
+	handle, err := ipvs.New("")
+	if err != nil {
+		tk.LogIt(tk.LogError, "ipvs.New: %s\n", err)
+		os.Exit(1)
+	}
+	ipVSCtx.handle = handle
+	go IpVSSync()
+}

api/loxinlp/nlp.go

@@ -19,7 +19,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"os/exec"
@@ -118,7 +117,7 @@ func applyLoadBalancerConfig() bool {
 	var resp struct {
 		Attr []cmn.LbRuleMod `json:"lbAttr"`
 	}
-	byteBuf, err := ioutil.ReadFile("/etc/loxilb/lbconfig.txt")
+	byteBuf, err := os.ReadFile("/etc/loxilb/lbconfig.txt")
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -139,7 +138,7 @@ func applySessionConfig() bool {
 	var resp struct {
 		Attr []cmn.SessionMod `json:"sessionAttr"`
 	}
-	byteBuf, err := ioutil.ReadFile("/etc/loxilb/sessionconfig.txt")
+	byteBuf, err := os.ReadFile("/etc/loxilb/sessionconfig.txt")
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -160,7 +159,7 @@ func applyUlClConfig() bool {
 	var resp struct {
 		Attr []cmn.SessionUlClMod `json:"ulclAttr"`
 	}
-	byteBuf, err := ioutil.ReadFile("/etc/loxilb/sessionulclconfig.txt")
+	byteBuf, err := os.ReadFile("/etc/loxilb/sessionulclconfig.txt")
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -181,7 +180,7 @@ func applyFWConfig() bool {
 	var resp struct {
 		Attr []cmn.FwRuleMod `json:"fwAttr"`
 	}
-	byteBuf, err := ioutil.ReadFile("/etc/loxilb/FWconfig.txt")
+	byteBuf, err := os.ReadFile("/etc/loxilb/FWconfig.txt")
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -202,7 +201,7 @@ func applyEPConfig() bool {
 	var resp struct {
 		Attr []cmn.EndPointMod `json:"Attr"`
 	}
-	byteBuf, err := ioutil.ReadFile("/etc/loxilb/EPconfig.txt")
+	byteBuf, err := os.ReadFile("/etc/loxilb/EPconfig.txt")
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -719,8 +718,12 @@ func ModLink(link nlp.Link, add bool) int {
 
 		vid, _ = strconv.Atoi(strings.Join(re.FindAllString(name, -1), " "))
 		// Dirty hack to support docker0 bridge
-		if vid == 0 && name == "docker0" {
-			vid = 4090
+		if vid == 0 {
+			if name == "docker0" {
+				vid = 4090
+			} else if name == "cni0" {
+				vid = 4091
+			}
 		}
 		if add {
 			ret, err = hooks.NetVlanAdd(&cmn.VlanMod{Vid: vid, Dev: name, LinkIndex: idx,
@@ -750,8 +753,12 @@ func ModLink(link nlp.Link, add bool) int {
 		}
 		vid, _ = strconv.Atoi(strings.Join(re.FindAllString(brLink.Attrs().Name, -1), " "))
 		// Dirty hack to support docker bridge
-		if vid == 0 && brLink.Attrs().Name == "docker0" {
-			vid = 4090
+		if vid == 0 {
+			if brLink.Attrs().Name == "docker0" {
+				vid = 4090
+			} else if brLink.Attrs().Name == "cni0" {
+				vid = 4091
+			}
 		}
 	}
 
@@ -1536,7 +1543,7 @@ func LbSessionGet(done bool) int {
 	return 0
 }
 
-func NlpInit(bgpPeerMode bool, blackList string) *NlH {
+func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH {
 
 	nNl = new(NlH)
 
@@ -1601,5 +1608,9 @@ func NlpInit(bgpPeerMode bool, blackList string) *NlH {
 
 	go LbSessionGet(done)
 
+	if ipvsCompat {
+		IpVSInit()
+	}
+
 	return nNl
 }

go.mod

@@ -25,10 +25,12 @@ require (
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/loxilb-io/ipvs v0.1.0 // indirect
 	github.com/loxilb-io/sctp v0.0.0-20230519081703-6d1baec82fd4 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 )
 
 require (
@@ -44,7 +46,7 @@ require (
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/prometheus/client_golang v1.14.0
-	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
+	github.com/vishvananda/netns v0.0.2 // indirect
 	go.mongodb.org/mongo-driver v1.11.6 // indirect
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
 	golang.org/x/text v0.13.0 // indirect

go.sum

@@ -233,6 +233,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/loxilb-io/ipvs v0.1.0 h1:TpTkwh5CLgJ7YW86rvWyqJPEpQFqs2TNbRG/IECeq+w=
+github.com/loxilb-io/ipvs v0.1.0/go.mod h1:EKjimnzyVL9AXMMNfPWeokxF1uNeuDrEGF5gPFMdmIo=
 github.com/loxilb-io/loxilib v0.8.8 h1:nW6RvLXDQxr5Pe2Ygg7qyYm4QG5y5cG+/jQ4m/DckP4=
 github.com/loxilb-io/loxilib v0.8.8/go.mod h1:LoQCxBz+N0fO9rGwRmPHrQPHol/jUf4MNpph63Cydkg=
 github.com/loxilb-io/loxilib v0.8.9-0.20231211082246-dc641488569f h1:q745LqB2G97bOpf6MCa34VyNJEuvVx4rTFVut3kz9oo=
@@ -309,6 +311,8 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -328,6 +332,8 @@ github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA85
 github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netns v0.0.2 h1:Cn05BRLm+iRP/DZxyVSsfVyrzgjDbwHwkVt38qvXnNI=
+github.com/vishvananda/netns v0.0.2/go.mod h1:yitZXdAVI+yPFSb4QUe+VW3vOVl4PZPNcBgbPxAtJxw=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
@@ -489,6 +495,8 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=

loxinet/loxinet.go

@@ -277,7 +277,7 @@ func loxiNetInit() {
 	// Initialize the nlp subsystem
 	if !opts.Opts.NoNlp {
 		nlp.NlpRegister(NetAPIInit(opts.Opts.BgpPeerMode))
-		nlp.NlpInit(opts.Opts.BgpPeerMode, opts.Opts.BlackList)
+		nlp.NlpInit(opts.Opts.BgpPeerMode, opts.Opts.BlackList, opts.Opts.IPVSCompat)
 	}
 
 	// Initialize the Prometheus subsystem

options/options.go

@@ -28,4 +28,5 @@ var Opts struct {
 	BgpPeerMode       bool           `short:"r" long:"peer" description:"Run loxilb with goBGP only, no Datapath"`
 	BlackList         string         `long:"blacklist" description:"Regex string of blacklisted ports" default:"none"`
 	Rpc               string         `long:"rpc" description:"RPC mode for syncing - netrpc or grpc" default:"netrpc"`
+	IPVSCompat        bool           `long:"ipvs-compat" description:"Enable ipvs-compat(experimental)"`
 }