calico blockaffinity static route support (#3715)

docs/RELEASE-NOTES.rst

@@ -13,6 +13,12 @@ Added Functionality
 Bug Fixes
 ````````````
 * `Issue 3679 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3679>`_: Certificate, CA chain, and private key shown in debug logs
+* `Issue 3655 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3655>`_: Calico static routes not updated when nodes added/removed from cluster
+* `Issue 3717 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3717>`_: StaticRoute CNI calico doesn't detect multiple blockaffinities
+
+Upgrade notes
+``````````````
+* For using calico cni with staticRoutingMode, update RBAC permissions to monitor calico blockaffinities resource. See `RBAC <./config_examples/rbac/k8s_rbac.yaml>`_
 
 2.19.0
 -------------

docs/config_examples/rbac/clusterrole.yaml

@@ -26,6 +26,9 @@ rules:
   - apiGroups: ["config.openshift.io/v1"]
     resources: ["network"]
     verbs: ["list"]
+  - apiGroups: [ "crd.projectcalico.org" ]
+    resources: [ "blockaffinities" ]
+    verbs: [ "get", "watch", "list" ]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

docs/config_examples/rbac/k8s_rbac.yml

@@ -26,6 +26,9 @@ rules:
   - apiGroups: ["config.openshift.io/v1"]
     resources: ["network"]
     verbs: ["list"]
+  - apiGroups: [ "crd.projectcalico.org" ]
+    resources: [ "blockaffinities" ]
+    verbs: [ "get", "watch", "list" ]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

pkg/controller/clusterHandler.go

@@ -48,6 +48,8 @@ func (ch *ClusterHandler) addClusterConfig(clusterName string, config *ClusterCo
 	config.namespaceLabel = ch.namespaceLabel
 	config.nodeLabelSelector = ch.nodeLabelSelector
 	config.routeLabel = ch.routeLabel
+	config.orchestrationCNI = ch.orchestrationCNI
+	config.staticRoutingMode = ch.staticRoutingMode
 	config.nativeResourceSelector, _ = createLabelSelector(DefaultNativeResourceLabel)
 	config.customResourceSelector, _ = createLabelSelector(DefaultCustomResourceLabel)
 	ch.ClusterConfigs[clusterName] = config

pkg/controller/constants.go

@@ -42,7 +42,7 @@ const (
 	CALICO_K8S                  = "calico-k8s"
 	CALICO_API_BLOCK_AFFINITIES = "/apis/crd.projectcalico.org/v1/blockaffinities"
 	CALICONodeIPAnnotation      = "projectcalico.org/IPv4Address"
+	BLOCKAFFINITIES             = "blockaffinities"
 
 	CommonPartition = "Common"
-	LocalCluster    = ""
 )

pkg/controller/controller.go

@@ -19,6 +19,8 @@ package controller
 import (
 	"context"
 	"fmt"
+	authv1 "k8s.io/api/authorization/v1"
+	"k8s.io/client-go/dynamic"
 	"net/http"
 	"os"
 	"strings"
@@ -173,6 +175,8 @@ func NewController(params Params, startController bool) *Controller {
 	clusterConfig.nodeLabelSelector = params.NodeLabelSelector
 	clusterConfig.nativeResourceSelector, _ = createLabelSelector(DefaultNativeResourceLabel)
 	clusterConfig.customResourceSelector, _ = createLabelSelector(DefaultCustomResourceLabel)
+	clusterConfig.orchestrationCNI = params.OrchestrationCNI
+	clusterConfig.staticRoutingMode = params.StaticRoutingMode
 	switch ctlr.mode {
 	case OpenShiftMode, KubernetesMode:
 		clusterConfig.routeLabel = params.RouteLabel
@@ -189,7 +193,6 @@ func NewController(params Params, startController bool) *Controller {
 	if ctlr.PoolMemberType == NodePort || ctlr.PoolMemberType == NodePortLocal {
 		ctlr.shareNodes = true
 	}
-
 	if err := ctlr.setupClientsforCluster(params.Config, params.IPAM, ctlr.mode == OpenShiftMode, ctlr.multiClusterHandler.LocalClusterName, clusterConfig); err != nil {
 		log.Errorf("Failed to Setup Clients: %v", err)
 	}
@@ -198,6 +201,8 @@ func NewController(params Params, startController bool) *Controller {
 	ctlr.multiClusterHandler.namespaces = params.Namespaces
 	ctlr.multiClusterHandler.nodeLabelSelector = params.NodeLabelSelector
 	ctlr.multiClusterHandler.routeLabel = params.RouteLabel
+	ctlr.multiClusterHandler.staticRoutingMode = params.StaticRoutingMode
+	ctlr.multiClusterHandler.orchestrationCNI = params.OrchestrationCNI
 	// add the cluster config for local cluster
 	ctlr.multiClusterHandler.addClusterConfig(ctlr.multiClusterHandler.LocalClusterName, clusterConfig)
 
@@ -420,12 +425,20 @@ func (ctlr *Controller) setupClientsforCluster(config *rest.Config, ipamClient,
 		}
 	}
 
+	var dynamicClient dynamic.Interface
+	if clusterConfig.orchestrationCNI == CALICO_K8S && clusterConfig.staticRoutingMode {
+		dynamicClient, err = dynamic.NewForConfig(config)
+		if nil != err {
+			return fmt.Errorf("Failed to create dynamic Client for cluster %s: %v", clusterName, err)
+		}
+	}
 	log.Debugf("Clients Created for cluster: %s", clusterName)
 	//Update the clusterConfig store
 	clusterConfig.kubeClient = kubeClient
 	clusterConfig.kubeCRClient = kubeCRClient
 	clusterConfig.kubeIPAMClient = kubeIPAMClient
 	clusterConfig.routeClientV1 = rclient
+	clusterConfig.dynamicClient = dynamicClient
 	return nil
 }
 
@@ -438,6 +451,12 @@ func (ctlr *Controller) setupInformers(clusterName string) error {
 		}
 	}
 	_ = ctlr.setNodeInformer(clusterName)
+	// create block affinities informer for calico cni enabled clusters.
+	if clusterConfig.orchestrationCNI == CALICO_K8S && clusterConfig.staticRoutingMode {
+		if clusterConfig.dynamicClient != nil {
+			_ = ctlr.newDynamicInformersForCluster(clusterConfig.dynamicClient, clusterName)
+		}
+	}
 	return nil
 }
 
@@ -492,6 +511,38 @@ func (ctlr *Controller) StartInformers(clusterName string) {
 	for _, inf := range informerStore.comInformers {
 		inf.start(ctlr.multiClusterHandler.LocalClusterName, false)
 	}
+
+	//start CNI informers if required
+	if ctlr.StaticRoutingMode && ctlr.OrchestrationCNI == CALICO_K8S {
+		// check if role permissions exists for calico crd
+		roleCheck := authv1.SelfSubjectAccessReview{
+			Spec: authv1.SelfSubjectAccessReviewSpec{
+				ResourceAttributes: &authv1.ResourceAttributes{
+					Resource:  BLOCKAFFINITIES,
+					Verb:      "watch",
+					Group:     "crd.projectcalico.org",
+					Namespace: "",
+				},
+			},
+		}
+		clusterConfig := ctlr.multiClusterHandler.getClusterConfig(ctlr.multiClusterHandler.LocalClusterName)
+		if clusterConfig != nil {
+			resp, err := clusterConfig.kubeClient.AuthorizationV1().
+				SelfSubjectAccessReviews().
+				Create(context.TODO(), &roleCheck, metaV1.CreateOptions{})
+			if err == nil {
+				if resp.Status.Allowed {
+					log.Debugf("RBAC present for blockaffinities watch: %v", resp)
+					informerStore.dynamicInformers.start()
+				} else {
+					log.Warning("Role Permissions to watch blockaffinities resource for calico CNI is not provided.Informers are not created for blockaffinities resource.Create proper RBAC for blockaffinities watch for static routing to work properly")
+				}
+			} else {
+				log.Errorf("Failed to create Self Subject Access Review for blockaffinities resource: %v.Skipping informer creation for blockaffinitoes resource", err)
+			}
+		}
+
+	}
 	switch ctlr.mode {
 	case OpenShiftMode, KubernetesMode:
 		// nrInformers only with openShiftMode
@@ -528,6 +579,10 @@ func (ctlr *Controller) StopInformers(clusterName string) {
 	for _, nsInf := range informerStore.nsInformers {
 		nsInf.stop()
 	}
+	// stop cni Informer
+	if informerStore.dynamicInformers != nil {
+		informerStore.dynamicInformers.stop()
+	}
 	// stop node Informer
 	informerStore.nodeInformer.stop()
 }

pkg/controller/informers.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"k8s.io/client-go/dynamic/dynamicinformer"
 	"reflect"
 	"time"
 
@@ -37,6 +38,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/tools/cache"
 )
 
@@ -1480,6 +1482,20 @@ func (nsInfr *NSInformer) stop() {
 	close(nsInfr.stopCh)
 }
 
+func (dynamicInf *DynamicInformers) start() {
+	if dynamicInf.CalicoBlockAffinityInformer != nil {
+		log.Infof("Starting calico block affinity Informer for cluster %v", dynamicInf.clusterName)
+		go dynamicInf.CalicoBlockAffinityInformer.Informer().Run(dynamicInf.stopCh)
+		if dynamicInf.CalicoBlockAffinityInformer.Informer().HasSynced() {
+			log.Debugf("Successfully synced block affinity informer caches for Cluster: %s", dynamicInf.clusterName)
+		}
+	}
+}
+
+func (dynamicInf *DynamicInformers) stop() {
+	close(dynamicInf.stopCh)
+}
+
 func (nodeInfr *NodeInformer) start(apiServerUnreachable bool) {
 	var cacheSyncs []cache.InformerSynced
 	if nodeInfr.nodeInformer != nil {
@@ -1628,3 +1644,33 @@ func (ctlr *Controller) getErrorHandlerFunc(rsType, clusterName string) func(r *
 		}
 	}
 }
+
+func (ctlr *Controller) newDynamicInformersForCluster(client dynamic.Interface, clusterName string) *DynamicInformers {
+	log.Debugf("Creating dynamic Informers for cluster: %s", clusterName)
+	clusterConfig := ctlr.multiClusterHandler.getClusterConfig(clusterName)
+	informers := &DynamicInformers{
+		clusterName: clusterName,
+		stopCh:      make(chan struct{}),
+	}
+	dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(client, 0)
+	// if orchestration cni is calico and static routing mode is true,
+	// create informers for blockaffinities CR.
+	if ctlr.StaticRoutingMode && ctlr.OrchestrationCNI == CALICO_K8S {
+		informers.CalicoBlockAffinityInformer = dynamicInformerFactory.ForResource(CalicoBlockaffinity)
+	}
+	clusterConfig.InformerStore.dynamicInformers = informers
+	ctlr.addDynamicResourceEventHandlers(informers)
+	return informers
+}
+
+func (ctlr *Controller) addDynamicResourceEventHandlers(dynamicInf *DynamicInformers) {
+	if dynamicInf.CalicoBlockAffinityInformer != nil {
+		dynamicInf.CalicoBlockAffinityInformer.Informer().AddEventHandler(
+			&cache.ResourceEventHandlerFuncs{
+				AddFunc:    func(obj interface{}) { ctlr.processBlockAffinities(dynamicInf.clusterName) },
+				DeleteFunc: func(obj interface{}) { ctlr.processBlockAffinities(dynamicInf.clusterName) },
+			},
+		)
+		dynamicInf.CalicoBlockAffinityInformer.Informer().SetWatchErrorHandler(ctlr.getErrorHandlerFunc(BLOCKAFFINITIES, dynamicInf.clusterName))
+	}
+}

pkg/controller/multiClusterInformers.go

@@ -260,6 +260,7 @@ func (ctlr *Controller) setupAndStartExternalClusterInformers(clusterName string
 		log.Errorf("[MultiCluster] unable to setup node informer for cluster: %v, Error: %v", clusterName, err)
 		return err
 	}
+
 	return nil
 }