Generate privateKey secret in one namespace

Azure · Jan 16, 2024 · ca44710 · ca44710
1 parent d9c5d2c
commit ca44710
Show file tree

Hide file tree

Showing 17 changed files with 324 additions and 63 deletions.
diff --git a/cmd/kube-egress-gateway-controller/cmd/root.go b/cmd/kube-egress-gateway-controller/cmd/root.go
@@ -58,6 +58,7 @@ var (
 	gatewayLBProbePort      int
 	enableLeaderElection    bool
 	leaderElectionNamespace string
+	secretNamespace         string
 	probePort               int
 	zapOpts                 = zap.Options{
 		Development: true,
@@ -85,6 +86,7 @@ func init() {
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 	rootCmd.Flags().StringVar(&leaderElectionNamespace, "leader-election-namespace", os.Getenv(consts.PodNamespaceEnvKey), "the namespace to create leader election objects")
+	rootCmd.Flags().StringVar(&secretNamespace, "secret-namespace", os.Getenv(consts.PodNamespaceEnvKey), "The namespace to store server privateKey secrets")
 
 	zapOpts.BindFlags(goflag.CommandLine)
 	rootCmd.Flags().AddGoFlagSet(goflag.CommandLine)
@@ -172,8 +174,9 @@ func startControllers(cmd *cobra.Command, args []string) {
 	}
 
 	if err = (&controllers.StaticGatewayConfigurationReconciler{
-		Client:   mgr.GetClient(),
-		Recorder: mgr.GetEventRecorderFor("staticGatewayConfiguration-controller"),
+		Client:          mgr.GetClient(),
+		SecretNamespace: secretNamespace,
+		Recorder:        mgr.GetEventRecorderFor("staticGatewayConfiguration-controller"),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "StaticGatewayConfiguration")
 		os.Exit(1)

diff --git a/cmd/kube-egress-gateway-daemon/cmd/root.go b/cmd/kube-egress-gateway-daemon/cmd/root.go
@@ -11,13 +11,16 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/cloud-provider-azure/pkg/azclient"
 	"sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -32,6 +35,7 @@ import (
 	controllers "github.com/Azure/kube-egress-gateway/controllers/daemon"
 	"github.com/Azure/kube-egress-gateway/pkg/azmanager"
 	"github.com/Azure/kube-egress-gateway/pkg/config"
+	"github.com/Azure/kube-egress-gateway/pkg/consts"
 	"github.com/Azure/kube-egress-gateway/pkg/healthprobe"
 )
 
@@ -60,6 +64,7 @@ var (
 	metricsPort        int
 	probePort          int
 	gatewayLBProbePort int
+	secretNamespace    string
 	zapOpts            = zap.Options{
 		Development: true,
 	}
@@ -83,6 +88,7 @@ func init() {
 	rootCmd.Flags().IntVar(&metricsPort, "metrics-bind-port", 8080, "The port the metric endpoint binds to.")
 	rootCmd.Flags().IntVar(&probePort, "health-probe-bind-port", 8081, "The port the probe endpoint binds to.")
 	rootCmd.Flags().IntVar(&gatewayLBProbePort, "gateway-lb-probe-port", 8082, "The port the gateway lb probe endpoint binds to.")
+	rootCmd.Flags().StringVar(&secretNamespace, "secret-namespace", os.Getenv(consts.PodNamespaceEnvKey), "The namespace to retrieve server privateKey secrets")
 
 	zapOpts.BindFlags(goflag.CommandLine)
 	rootCmd.Flags().AddGoFlagSet(goflag.CommandLine)
@@ -101,6 +107,14 @@ func startControllers(cmd *cobra.Command, args []string) {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts)))
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Cache: cache.Options{
+			// we only watch secrets in the namespace where the kube-egress-gateway pods are running
+			ByObject: map[client.Object]cache.ByObject{
+				&corev1.Secret{}: {
+					Field: client.InNamespace(secretNamespace).AsSelector(),
+				},
+			},
+		},
 		Scheme: scheme,
 		Metrics: metricsserver.Options{
 			BindAddress: ":" + strconv.Itoa(metricsPort),

diff --git a/config/daemon/manager/manager.yaml b/config/daemon/manager/manager.yaml
@@ -38,6 +38,7 @@ spec:
       - command:
         - /kube-egress-gateway-daemon
         args:
+        - --secret-namespace=$(MY_POD_NAMESPACE)
         - --gateway-lb-probe-port=8082
         - --cloud-config=/etc/config/azure.json
         image: daemon:latest

diff --git a/config/daemon/rbac/role.yaml b/config/daemon/rbac/role.yaml
@@ -12,14 +12,6 @@ rules:
   - get
   - list
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - get
-  - list
-  - watch
 - apiGroups:
   - egressgateway.kubernetes.azure.com
   resources:
@@ -63,3 +55,18 @@ rules:
   - get
   - patch
   - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: daemon-manager-role
+  namespace: kube-egress-gateway-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
diff --git a/config/daemon/rbac/role_binding.yaml b/config/daemon/rbac/role_binding.yaml
@@ -10,3 +10,16 @@ subjects:
 - kind: ServiceAccount
   name: daemon-manager
   namespace: system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: daemon-manager-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: daemon-manager-role
+subjects:
+- kind: ServiceAccount
+  name: daemon-manager
+  namespace: system
diff --git a/config/manager/manager/manager.yaml b/config/manager/manager/manager.yaml
@@ -32,6 +32,7 @@ spec:
         args:
         - --leader-elect
         - --leader-election-namespace=$(MY_POD_NAMESPACE)
+        - --secret-namespace=$(MY_POD_NAMESPACE)
         - --cloud-config=/etc/config/azure.json
         - --gateway-lb-probe-port=8082
         image: controller:latest

diff --git a/controllers/daemon/staticgatewayconfiguration_controller.go b/controllers/daemon/staticgatewayconfiguration_controller.go
@@ -57,7 +57,7 @@ type StaticGatewayConfigurationReconciler struct {
 
 //+kubebuilder:rbac:groups=egressgateway.kubernetes.azure.com,resources=staticgatewayconfigurations,verbs=get;list;watch
 //+kubebuilder:rbac:groups=egressgateway.kubernetes.azure.com,resources=staticgatewayconfigurations/status,verbs=get;update;patch
-//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch
+//+kubebuilder:rbac:groups=core,namespace=kube-egress-gateway-system,resources=secrets,verbs=get;list;watch
 //+kubebuilder:rbac:groups=egressgateway.kubernetes.azure.com,resources=gatewaystatuses,verbs=get;list;watch;create;update;patch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
@@ -291,7 +291,7 @@ func (r *StaticGatewayConfigurationReconciler) getWireguardPrivateKey(
 	gwConfig *egressgatewayv1alpha1.StaticGatewayConfiguration,
 ) (*wgtypes.Key, error) {
 	secretKey := &types.NamespacedName{
-		Namespace: gwConfig.Namespace,
+		Namespace: gwConfig.Status.PrivateKeySecretRef.Namespace,
 		Name:      gwConfig.Status.PrivateKeySecretRef.Name,
 	}
 	secret := &corev1.Secret{}

diff --git a/controllers/daemon/staticgatewayconfiguration_controller_test.go b/controllers/daemon/staticgatewayconfiguration_controller_test.go
@@ -52,19 +52,20 @@ import (
 )
 
 const (
-	testName         = "test"
-	testNamespace    = "testns"
-	testNodepoolName = "testgw"
-	testPodNamespace = "testns2"
-	testNodeName     = "testNode"
-	testUID          = "1234567890"
-	vmssRG           = "vmssRG"
-	vmssName         = "vmssName"
-	privK            = "GHuMwljFfqd2a7cs6BaUOmHflK23zME8VNvC5B37S3k="
-	pubK             = "aPxGwq8zERHQ3Q1cOZFdJ+cvJX5Ka4mLN38AyYKYF10="
-	ilbIP            = "10.0.0.4"
-	ilbIPCidr        = "10.0.0.4/31"
-	nsName           = "gw-1234567890-10_0_0_4"
+	testName            = "test"
+	testNamespace       = "testns"
+	testSecretNamespace = "testns2"
+	testNodepoolName    = "testgw"
+	testPodNamespace    = "testns2"
+	testNodeName        = "testNode"
+	testUID             = "1234567890"
+	vmssRG              = "vmssRG"
+	vmssName            = "vmssName"
+	privK               = "GHuMwljFfqd2a7cs6BaUOmHflK23zME8VNvC5B37S3k="
+	pubK                = "aPxGwq8zERHQ3Q1cOZFdJ+cvJX5Ka4mLN38AyYKYF10="
+	ilbIP               = "10.0.0.4"
+	ilbIPCidr           = "10.0.0.4/31"
+	nsName              = "gw-1234567890-10_0_0_4"
 )
 
 var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func() {
@@ -195,7 +196,7 @@ var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func
 			secret := &corev1.Secret{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      testName,
-					Namespace: testNamespace,
+					Namespace: testSecretNamespace,
 				},
 				Data: map[string][]byte{
 					consts.WireguardPrivateKeyName: []byte(privK),
@@ -796,6 +797,7 @@ func getTestGwConfigStatus() egressgatewayv1alpha1.StaticGatewayConfigurationSta
 				APIVersion: "v1",
 				Kind:       "Secret",
 				Name:       testName,
+				Namespace:  testSecretNamespace,
 			},
 		},
 	}