Add finalizer to egressgateway

Signed-off-by: bzsuni <[email protected]>

pkg/controller/webhook/mutating.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+
 	"gomodules.xyz/jsonpatch/v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"

pkg/egressgateway/egress_gateway.go

@@ -28,6 +28,7 @@ import (
 	egress "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1"
 	"github.com/spidernet-io/egressgateway/pkg/utils"
 	"github.com/spidernet-io/egressgateway/pkg/utils/ip"
+	"github.com/spidernet-io/egressgateway/pkg/utils/slice"
 )
 
 type egnReconciler struct {
@@ -166,7 +167,23 @@ func (r egnReconciler) reconcileEGW(ctx context.Context, req reconcile.Request,
 
 	if deleted {
 		log.Info("request item is deleted")
-		return reconcile.Result{}, nil
+		p, err := getEgressGatewayPolicies(r.client, ctx, egw)
+		if err != nil {
+			log.Error(err, "getEgressGatewayPolicies when delete egressgateway")
+			return reconcile.Result{Requeue: true}, err
+		}
+		if containsEgressGatewayFinalizer(egw, egressGatewayFinalizers) && len(p) == 0 {
+			log.Info("remove the egressGatewayFinalizer")
+			removeEgressGatewayFinalizer(egw)
+			log.V(1).Info("remove the egressGatewayFinalizer", "ObjectMeta", egw.ObjectMeta)
+
+			err = r.client.Update(ctx, egw)
+			if err != nil {
+				log.Error(err, "remove the egressGatewayFinalizer", "ObjectMeta", egw.ObjectMeta)
+				return reconcile.Result{Requeue: true}, err
+			}
+		}
+		return reconcile.Result{Requeue: false}, nil
 	}
 
 	if egw.Spec.NodeSelector.Selector == nil {
@@ -1268,3 +1285,48 @@ func countGatewayIP(egw *egress.EgressGateway) (ipv4sFree, ipv6sFree, ipv4sTotal
 	ipv4sTotal, ipv6sTotal, err = len(ipv4s), len(ipv6s), nil
 	return
 }
+
+// removeEgressGatewayFinalizer if the egress gateway is being deleted
+func removeEgressGatewayFinalizer(egw *egress.EgressGateway) {
+	if !egw.DeletionTimestamp.IsZero() {
+		if containsEgressGatewayFinalizer(egw, egressGatewayFinalizers) {
+			egw.Finalizers = slice.RemoveElement(egw.Finalizers, egressGatewayFinalizers)
+		}
+	}
+}
+
+func getEgressGatewayPolicies(client client.Client, ctx context.Context, egw *egress.EgressGateway) ([]egress.Policy, error) {
+	policies := make([]egress.Policy, 0)
+	// list policy
+	policyList := &egress.EgressPolicyList{}
+	err := client.List(ctx, policyList)
+	if err != nil {
+		return nil, err
+	}
+	for _, p := range policyList.Items {
+		if p.Spec.EgressGatewayName == egw.Name {
+			policies = append(policies, egress.Policy{Name: p.Name, Namespace: p.Namespace})
+		}
+	}
+	// list cluster policy
+	clusterPolicyList := &egress.EgressClusterPolicyList{}
+	err = client.List(ctx, clusterPolicyList)
+	if err != nil {
+		return nil, err
+	}
+	for _, cp := range clusterPolicyList.Items {
+		if cp.Spec.EgressGatewayName == egw.Name {
+			policies = append(policies, egress.Policy{Name: cp.Name, Namespace: cp.Namespace})
+		}
+	}
+	return policies, nil
+}
+
+func containsEgressGatewayFinalizer(gateway *egress.EgressGateway, finalizer string) bool {
+	for _, f := range gateway.ObjectMeta.Finalizers {
+		if f == finalizer {
+			return true
+		}
+	}
+	return false
+}

pkg/egressgateway/egress_gateway_webhook.go

@@ -36,22 +36,10 @@ type patchOperation struct {
 	Value interface{} `json:"value,omitempty"`
 }
 
+var egressGatewayFinalizers = "egressgateway.spidernet.io/egressgateway"
+
 func (egw *EgressGatewayWebhook) EgressGatewayValidate(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse {
-	// Check whether the deleted EgressGateway is referenced
 	if req.Operation == v1.Delete {
-		delEG := new(egress.EgressGateway)
-		err := json.Unmarshal(req.OldObject.Raw, delEG)
-		if err != nil {
-			return webhook.Denied(fmt.Sprintf("json unmarshal EgressGateway with error: %v", err))
-		}
-
-		for _, item := range delEG.Status.NodeList {
-			for _, eip := range item.Eips {
-				if len(eip.Policies) != 0 {
-					return webhook.Denied(fmt.Sprintf("Do not delete %v:%v because it is already referenced by EgressPolicy", req.Namespace, req.Name))
-				}
-			}
-		}
 		return webhook.Allowed("checked")
 	}
 
@@ -173,16 +161,16 @@ func (egw *EgressGatewayWebhook) EgressGatewayValidate(ctx context.Context, req
 
 func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse {
 	rander := rand.New(rand.NewSource(time.Now().UnixNano()))
-	isPatch := false
 	eg := new(egress.EgressGateway)
 	err := json.Unmarshal(req.Object.Raw, eg)
 	if err != nil {
 		return webhook.Denied(fmt.Sprintf("json unmarshal EgressGateway with error: %v", err))
 	}
 
 	reviewResponse := webhook.AdmissionResponse{}
-	var patch []patchOperation
+	var patchList []patchOperation
 
+	// patch egress gateway default eip
 	if egw.Config.FileConfig.EnableIPv4 {
 		if len(eg.Spec.Ippools.Ipv4DefaultEIP) == 0 && len(eg.Spec.Ippools.IPv4) != 0 {
 			ipv4Ranges, err := ip.MergeIPRanges(constant.IPv4, eg.Spec.Ippools.IPv4)
@@ -192,12 +180,11 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
 
 			ipv4s, _ := ip.ParseIPRanges(constant.IPv4, ipv4Ranges)
 			if len(ipv4s) != 0 {
-				patch = append(patch, patchOperation{
+				patchList = append(patchList, patchOperation{
 					Op:    "add",
 					Path:  "/spec/ippools/ipv4DefaultEIP",
 					Value: ipv4s[rander.Intn(len(ipv4s))].String(),
 				})
-				isPatch = true
 			}
 
 		}
@@ -213,21 +200,25 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
 
 			ipv6s, _ := ip.ParseIPRanges(constant.IPv6, ipv6Ranges)
 			if len(ipv6s) != 0 {
-				patch = append(patch, patchOperation{
+				patchList = append(patchList, patchOperation{
 					Op:    "add",
 					Path:  "/spec/ippools/ipv6DefaultEIP",
 					Value: ipv6s[rander.Intn(len(ipv6s))].String(),
 				})
-				isPatch = true
 			}
-
 		}
 	}
 
-	if isPatch {
-		patchBytes, err := json.Marshal(patch)
+	// patch egress gateway finalizer
+	patch := getEgressGatewayFinalizerPatch(req, []string{egressGatewayFinalizers})
+	if patch != nil {
+		patchList = append(patchList, *patch)
+	}
+
+	if len(patchList) > 0 {
+		patchBytes, err := json.Marshal(patchList)
 		if err != nil {
-			return webhook.Denied(fmt.Sprintf("failed to allocate defaultEIP.: %v", err))
+			return webhook.Denied(fmt.Sprintf("failed to Marshal patchList.: %v", err))
 		}
 
 		reviewResponse.Allowed = true
@@ -240,3 +231,14 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
 
 	return webhook.Allowed("checked")
 }
+
+func getEgressGatewayFinalizerPatch(req webhook.AdmissionRequest, finalizer []string) *patchOperation {
+	if req.Operation == v1.Create {
+		return &patchOperation{
+			Op:    "add",
+			Path:  "/metadata/finalizers",
+			Value: finalizer,
+		}
+	}
+	return nil
+}

pkg/utils/slice/slice.go

@@ -0,0 +1,21 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package slice
+
+func RemoveElement[T int | string](slice []T, element T) []T {
+	i := indexOf(slice, element)
+	if i == -1 {
+		return slice
+	}
+	return append(slice[:i], slice[i+1:]...)
+}
+
+func indexOf[T int | string](slice []T, element T) int {
+	for i, v := range slice {
+		if v == element {
+			return i
+		}
+	}
+	return -1
+}

pkg/utils/slice/slice_test.go

@@ -0,0 +1,31 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package slice
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestRemoveElement(t *testing.T) {
+	cases := []struct {
+		name   string
+		in     []string
+		el     string
+		expect []string
+	}{
+		{"when included", []string{"a", "b", "c"}, "a", []string{"b", "c"}},
+		{"when not included", []string{"a", "b", "c"}, "d", []string{"a", "b", "c"}},
+		{"when empty slice", []string{}, "a", []string{}},
+	}
+
+	for _, v := range cases {
+		t.Run(v.name, func(t *testing.T) {
+			out := RemoveElement[string](v.in, v.el)
+			if !reflect.DeepEqual(out, v.expect) {
+				t.Errorf("RemoveElement() got = %v, want = %v", out, v.expect)
+			}
+		})
+	}
+}

test/doc/egressgateway.md

@@ -20,4 +20,5 @@
 | G00014  | Edit the `NodeSelector` to match another `node`, `Status.NodeList` is updated to the newly matched node, using the `policy` of this `EgressGateway`, `Status.Node` is updated to the newly matched `node`, the `pod`'s egress ip is eip<br>Edit the `NodeSelector` to not match any `node`, `Status.NodeList` is empty, using the `policy` of this `EgressGateway`, `Status.Node` is empty, accessing external IP from the pod will error<br>Edit the `NodeSelector` to match one `node`, `Status.NodeList` is the matched `node`, using the `policy` of this `EgressGateway`, `Status.Node` is updated to the newly matched `node`, the `pod`'s egress ip is eip | p2       | false |        |       |
 | G00017  | When creating an `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressGatewayName`, the tenant or cluster default gateway can be automatically configured and created successfully                                                                                                                                                                                                                                                                                                                                                                                                                                                             | p2       | false | done   |       |
 | G00018  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressGateway` succeeds                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | p2       | false | done   |       |
-| G00019  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully                                                                                                                                                                                                                                                                                                                                                                                           | p2       | false | done   |       |
+| G00019  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully                                                                                                                                                                                                                                                                                                                                                                                           | p2       | false | done   |       |
+| G00020  | When a `policy` references a `gateway`, deleting the `gateway` will be in a "deleting" state until all the `policies` that reference the `gateway` are deleted. The `gateway` will be successfully deleted once all the referencing `policies` are removed                                                           | p2        | false  |        |       |

test/doc/egressgateway_zh.md

@@ -25,6 +25,7 @@
 | G00017   | When creating an `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressGatewayName`, the tenant or cluster default gateway can be automatically configured and created successfully                                                             | p2        | false  |        |       |
 | G00018  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressGateway` succeeds                                                           | p2        | false  |        |       |
 | G00019  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully                                                           | p2        | false  |        |       |
+| G00020  | When a policy references a gateway, deleting the gateway will be in a "deleting" state until all the policies that reference the gateway are deleted. The gateway will be successfully deleted once all the referencing policies are removed                                                           | p2        | false  |        |       |
 -->
 
 # EgressGateway E2E 用例
@@ -48,4 +49,5 @@
 | G00014 | 编辑 `NodeSelector` 使其匹配另一个节点，`Status.NodeList` 更新为新匹配的节点，使用该 EgressGateway 的 policy `Status.Node` 更新为新匹配的节点, `pod` 的出口 `ip` 为 `eip`<br>编辑 `NodeSelector` 使其不匹配任何节点，`Status.NodeList` 为空，用该 EgressGateway 的 policy `Status.Node` 为空, `pod` 使用访问外部 IP 会报错<br>编辑 `NodeSelector` 使其匹配一个节点，`Status.NodeList` 为所匹配的节点，使用该 EgressGateway 的 policy `Status.Node` 更新为新匹配的节点，`pod` 的出口 `ip` 为 `eip`                           | p2  | false |      |     |
 | G00017 | 创建 `EgressCluster` 或者 `EgressClusterPolicy` 时使用未指定 `spec.egressGatewayName` 时，可以使用自动设置租户或者集群默认网关，并创建成功 | p2  | false |    |    |
 | G00018 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 EgressGateway 成功 | p2  | false |    |    |
-| G00019 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 `EgressCluster` 或者 `EgressClusterPolicy`，未指定 `spec.egressIP.useNodeIP` 时，policy 创建失败，`spec.egressIP.useNodeIP` 为 true 时，policy 创建成功 | p2  | false |    |    |
+| G00019 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 `EgressCluster` 或者 `EgressClusterPolicy`，未指定 `spec.egressIP.useNodeIP` 时，policy 创建失败，`spec.egressIP.useNodeIP` 为 true 时，policy 创建成功 | p2  | false |    |    |
+| G00020 | 存在 `policy` 引用 `gateway` 时，删除 `gateway`，会处于 `deleting` 状态，直到所有引用 `gateway` 的 `policy` 都被删除，`gateway` 会被删除成功     | p2  | false |    |    |

test/e2e/egressgateway/egressgateway_test.go

@@ -15,7 +15,9 @@ import (
 
 	"github.com/go-faker/faker/v4"
 	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/spidernet-io/egressgateway/pkg/constant"
 	egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1"
@@ -634,6 +636,92 @@ var _ = Describe("Operate EgressGateway", Label("EgressGateway"), Ordered, func(
 			Expect(common.CheckDaemonSetEgressIP(ctx, cli, config, egressConfig, ds, egp.Status.Eip.Ipv4, egp.Status.Eip.Ipv6, true)).NotTo(HaveOccurred())
 		})
 	})
+
+	/*
+		此用例测试 egressgateway finalizer
+		1. 创建 egressgateway，检查 finalizer 被添加
+		2. 引用以上 egressgateway 创建 policy
+		3. 删除 egressgateway，检查 egressgateway 会处于 deleting 状态，不会被删除
+		4. 删除 policy，egressgateway 会被删除
+
+	*/
+	Context("Delete egressGateway", func() {
+		var ctx context.Context
+		// gateway
+		var egw *egressv1.EgressGateway
+
+		// policy
+		var egp *egressv1.EgressPolicy
+
+		// lalbe
+		var label map[string]string
+
+		// error
+		var err error
+
+		var gatewayFinalizer = "egressgateway.spidernet.io/egressgateway"
+
+		BeforeEach(func() {
+			ctx = context.Background()
+
+			label = map[string]string{"test-finalizer": faker.Word()}
+
+			// create gateway
+			egw = createEgressGateway(ctx)
+
+			// check finalizer
+			Expect(egw.GetFinalizers()).Should(ContainElement(gatewayFinalizer), "failed to check egressgateway finzalizer")
+
+			// create egressPolicy
+			egp, err = common.CreateEgressPolicyNew(ctx, cli, egressConfig, egw.Name, label)
+			Expect(err).NotTo(HaveOccurred())
+			GinkgoWriter.Printf("Succeeded create EgressPolicy: %s\n", egp.Name)
+
+			DeferCleanup(func() {
+				// delete the egp if it exists
+				if egp != nil {
+					GinkgoWriter.Printf("Delete egp: %s\n", egp.Name)
+					Expect(common.DeleteObj(ctx, cli, egp)).NotTo(HaveOccurred())
+				}
+
+				// delete the egw if it exists
+				if egw != nil {
+					GinkgoWriter.Printf("Delete egw: %s\n", egw.Name)
+					Expect(common.DeleteObj(ctx, cli, egw)).NotTo(HaveOccurred())
+				}
+			})
+		})
+
+		It("Test egressgateway finalizer", Label("G00020"), func() {
+			// delete gateway
+			GinkgoWriter.Printf("delete the egw: %s, we expect it to be in deleting status", egw.Name)
+			Expect(common.DeleteObj(ctx, cli, egw)).NotTo(HaveOccurred())
+			Consistently(ctx, func() error {
+				err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw)
+				if err != nil {
+					return err
+				}
+				if egw.DeletionTimestamp.Time.IsZero() {
+					return fmt.Errorf("not found deletionTimeStamp")
+				}
+				return nil
+			}).WithTimeout(time.Second * 6).WithPolling(time.Second * 2).Should(Succeed())
+
+			// delete egp
+			GinkgoWriter.Printf("delete the egp: %s\n", egp.Name)
+			Expect(common.DeleteObj(ctx, cli, egp)).NotTo(HaveOccurred())
+
+			// we expect the egw will be delete after a while
+			Eventually(ctx, func() error {
+				err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw)
+				if errors.IsNotFound(err) {
+					return nil
+				} else {
+					return fmt.Errorf("get the egw: %s that is not our expected", egw.Name)
+				}
+			}).WithTimeout(time.Second * 6).WithPolling(time.Second * 2).Should(Succeed())
+		})
+	})
 })
 
 func createEgressGateway(ctx context.Context) (egw *egressv1.EgressGateway) {