Add finalizer to egressgateway

cmd/controller/cmd/clean.go

@@ -116,6 +116,13 @@ func clean(validate, mutating string) error {
 	err = cli.List(ctx, gatewayList)
 	if err == nil {
 		for _, item := range gatewayList.Items {
+			if len(item.Finalizers) != 0 {
+				(&item).Finalizers = nil
+				err := cli.Update(ctx, &item)
+				if err != nil {
+					return err
+				}
+			}
 			err = cli.Delete(ctx, &item)
 			if err != nil {
 				return err

docs/usage/Install.en.md

@@ -273,3 +273,8 @@ helm repo update
     $ curl 10.6.1.92:8080
     Remote IP: 172.22.0.110
     ```
+## Uninstalling EgressGateway
+  In order to ensure uninterrupted business flow, the `egressgateway` incorporates a finalizer mechanism. When deleting the `egressgateway`, if there are `policies` referencing it, the `egressgateway` will remain in the "deleting" state until all `policies` are deleted, and the finalizer will be automatically removed.  
+  Therefore, if you want to delete the `egressgateway`, it is recommended to follow these steps:  
+  1. Delete all `policies` that reference the `egressgateway`.  
+  2. Delete the `egressgateway`.

docs/usage/Install.zh.md

@@ -274,3 +274,9 @@ helm repo update
     $ curl 10.6.1.92:8080
     Remote IP: 172.22.0.110
     ```
+
+## 卸载 EgressGateway
+  为了保证业务不断流，`egressgateway` 中加入了 finalizer 机制，删除 `egressgateway` 时，如果存在 policy 引用此 `egressgateway`，`egressgateway` 会一直处于 deleting 状态, 直到 所有的 policy 被删除，finalizer 就会被自动删除。  
+  所以，如果要删除 `egressgateway`，建议使用如下步骤：
+  1. 删除所有引用 egressgateway 的 policy
+  2. 删除 egressgateway

pkg/controller/webhook/mutating.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+
 	"gomodules.xyz/jsonpatch/v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"

pkg/egressgateway/egress_gateway.go

@@ -28,6 +28,7 @@ import (
 	egress "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1"
 	"github.com/spidernet-io/egressgateway/pkg/utils"
 	"github.com/spidernet-io/egressgateway/pkg/utils/ip"
+	"github.com/spidernet-io/egressgateway/pkg/utils/slice"
 )
 
 type egnReconciler struct {
@@ -166,7 +167,23 @@ func (r egnReconciler) reconcileEGW(ctx context.Context, req reconcile.Request,
 
 	if deleted {
 		log.Info("request item is deleted")
-		return reconcile.Result{}, nil
+		p, err := getEgressGatewayPolicies(r.client, ctx, egw)
+		if err != nil {
+			log.Error(err, "getEgressGatewayPolicies when delete egressgateway")
+			return reconcile.Result{Requeue: true}, err
+		}
+		if containsEgressGatewayFinalizer(egw, egressGatewayFinalizers) && len(p) == 0 {
+			log.Info("remove the egressGatewayFinalizer")
+			removeEgressGatewayFinalizer(egw)
+			log.V(1).Info("remove the egressGatewayFinalizer", "ObjectMeta", egw.ObjectMeta)
+
+			err = r.client.Update(ctx, egw)
+			if err != nil {
+				log.Error(err, "remove the egressGatewayFinalizer", "ObjectMeta", egw.ObjectMeta)
+				return reconcile.Result{Requeue: true}, err
+			}
+		}
+		return reconcile.Result{Requeue: false}, nil
 	}
 
 	if egw.Spec.NodeSelector.Selector == nil {
@@ -1268,3 +1285,48 @@ func countGatewayIP(egw *egress.EgressGateway) (ipv4sFree, ipv6sFree, ipv4sTotal
 	ipv4sTotal, ipv6sTotal, err = len(ipv4s), len(ipv6s), nil
 	return
 }
+
+// removeEgressGatewayFinalizer if the egress gateway is being deleted
+func removeEgressGatewayFinalizer(egw *egress.EgressGateway) {
+	if !egw.DeletionTimestamp.IsZero() {
+		if containsEgressGatewayFinalizer(egw, egressGatewayFinalizers) {
+			egw.Finalizers = slice.RemoveElement(egw.Finalizers, egressGatewayFinalizers)
+		}
+	}
+}
+
+func getEgressGatewayPolicies(client client.Client, ctx context.Context, egw *egress.EgressGateway) ([]egress.Policy, error) {
+	policies := make([]egress.Policy, 0)
+	// list policy
+	policyList := &egress.EgressPolicyList{}
+	err := client.List(ctx, policyList)
+	if err != nil {
+		return nil, err
+	}
+	for _, p := range policyList.Items {
+		if p.Spec.EgressGatewayName == egw.Name {
+			policies = append(policies, egress.Policy{Name: p.Name, Namespace: p.Namespace})
+		}
+	}
+	// list cluster policy
+	clusterPolicyList := &egress.EgressClusterPolicyList{}
+	err = client.List(ctx, clusterPolicyList)
+	if err != nil {
+		return nil, err
+	}
+	for _, cp := range clusterPolicyList.Items {
+		if cp.Spec.EgressGatewayName == egw.Name {
+			policies = append(policies, egress.Policy{Name: cp.Name, Namespace: cp.Namespace})
+		}
+	}
+	return policies, nil
+}
+
+func containsEgressGatewayFinalizer(gateway *egress.EgressGateway, finalizer string) bool {
+	for _, f := range gateway.ObjectMeta.Finalizers {
+		if f == finalizer {
+			return true
+		}
+	}
+	return false
+}

pkg/egressgateway/egress_gateway_webhook.go

@@ -36,22 +36,10 @@ type patchOperation struct {
 	Value interface{} `json:"value,omitempty"`
 }
 
+var egressGatewayFinalizers = "egressgateway.spidernet.io/egressgateway"
+
 func (egw *EgressGatewayWebhook) EgressGatewayValidate(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse {
-	// Check whether the deleted EgressGateway is referenced
 	if req.Operation == v1.Delete {
-		delEG := new(egress.EgressGateway)
-		err := json.Unmarshal(req.OldObject.Raw, delEG)
-		if err != nil {
-			return webhook.Denied(fmt.Sprintf("json unmarshal EgressGateway with error: %v", err))
-		}
-
-		for _, item := range delEG.Status.NodeList {
-			for _, eip := range item.Eips {
-				if len(eip.Policies) != 0 {
-					return webhook.Denied(fmt.Sprintf("Do not delete %v:%v because it is already referenced by EgressPolicy", req.Namespace, req.Name))
-				}
-			}
-		}
 		return webhook.Allowed("checked")
 	}
 
@@ -173,16 +161,16 @@ func (egw *EgressGatewayWebhook) EgressGatewayValidate(ctx context.Context, req
 
 func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse {
 	rander := rand.New(rand.NewSource(time.Now().UnixNano()))
-	isPatch := false
 	eg := new(egress.EgressGateway)
 	err := json.Unmarshal(req.Object.Raw, eg)
 	if err != nil {
 		return webhook.Denied(fmt.Sprintf("json unmarshal EgressGateway with error: %v", err))
 	}
 
 	reviewResponse := webhook.AdmissionResponse{}
-	var patch []patchOperation
+	var patchList []patchOperation
 
+	// patch egress gateway default eip
 	if egw.Config.FileConfig.EnableIPv4 {
 		if len(eg.Spec.Ippools.Ipv4DefaultEIP) == 0 && len(eg.Spec.Ippools.IPv4) != 0 {
 			ipv4Ranges, err := ip.MergeIPRanges(constant.IPv4, eg.Spec.Ippools.IPv4)
@@ -192,12 +180,11 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
 
 			ipv4s, _ := ip.ParseIPRanges(constant.IPv4, ipv4Ranges)
 			if len(ipv4s) != 0 {
-				patch = append(patch, patchOperation{
+				patchList = append(patchList, patchOperation{
 					Op:    "add",
 					Path:  "/spec/ippools/ipv4DefaultEIP",
 					Value: ipv4s[rander.Intn(len(ipv4s))].String(),
 				})
-				isPatch = true
 			}
 
 		}
@@ -213,21 +200,25 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
 
 			ipv6s, _ := ip.ParseIPRanges(constant.IPv6, ipv6Ranges)
 			if len(ipv6s) != 0 {
-				patch = append(patch, patchOperation{
+				patchList = append(patchList, patchOperation{
 					Op:    "add",
 					Path:  "/spec/ippools/ipv6DefaultEIP",
 					Value: ipv6s[rander.Intn(len(ipv6s))].String(),
 				})
-				isPatch = true
 			}
-
 		}
 	}
 
-	if isPatch {
-		patchBytes, err := json.Marshal(patch)
+	// patch egress gateway finalizer
+	patch := getEgressGatewayFinalizerPatch(req, []string{egressGatewayFinalizers})
+	if patch != nil {
+		patchList = append(patchList, *patch)
+	}
+
+	if len(patchList) > 0 {
+		patchBytes, err := json.Marshal(patchList)
 		if err != nil {
-			return webhook.Denied(fmt.Sprintf("failed to allocate defaultEIP.: %v", err))
+			return webhook.Denied(fmt.Sprintf("failed to Marshal patchList.: %v", err))
 		}
 
 		reviewResponse.Allowed = true
@@ -240,3 +231,14 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
 
 	return webhook.Allowed("checked")
 }
+
+func getEgressGatewayFinalizerPatch(req webhook.AdmissionRequest, finalizer []string) *patchOperation {
+	if req.Operation == v1.Create {
+		return &patchOperation{
+			Op:    "add",
+			Path:  "/metadata/finalizers",
+			Value: finalizer,
+		}
+	}
+	return nil
+}

pkg/utils/slice/slice.go

@@ -0,0 +1,21 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package slice
+
+func RemoveElement[T int | string](slice []T, element T) []T {
+	i := indexOf(slice, element)
+	if i == -1 {
+		return slice
+	}
+	return append(slice[:i], slice[i+1:]...)
+}
+
+func indexOf[T int | string](slice []T, element T) int {
+	for i, v := range slice {
+		if v == element {
+			return i
+		}
+	}
+	return -1
+}

pkg/utils/slice/slice_test.go

@@ -0,0 +1,31 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package slice
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestRemoveElement(t *testing.T) {
+	cases := []struct {
+		name   string
+		in     []string
+		el     string
+		expect []string
+	}{
+		{"when included", []string{"a", "b", "c"}, "a", []string{"b", "c"}},
+		{"when not included", []string{"a", "b", "c"}, "d", []string{"a", "b", "c"}},
+		{"when empty slice", []string{}, "a", []string{}},
+	}
+
+	for _, v := range cases {
+		t.Run(v.name, func(t *testing.T) {
+			out := RemoveElement[string](v.in, v.el)
+			if !reflect.DeepEqual(out, v.expect) {
+				t.Errorf("RemoveElement() got = %v, want = %v", out, v.expect)
+			}
+		})
+	}
+}

test/doc/egressgateway.md

@@ -20,4 +20,5 @@
 | G00014  | Edit the `NodeSelector` to match another `node`, `Status.NodeList` is updated to the newly matched node, using the `policy` of this `EgressGateway`, `Status.Node` is updated to the newly matched `node`, the `pod`'s egress ip is eip<br>Edit the `NodeSelector` to not match any `node`, `Status.NodeList` is empty, using the `policy` of this `EgressGateway`, `Status.Node` is empty, accessing external IP from the pod will error<br>Edit the `NodeSelector` to match one `node`, `Status.NodeList` is the matched `node`, using the `policy` of this `EgressGateway`, `Status.Node` is updated to the newly matched `node`, the `pod`'s egress ip is eip | p2       | false |        |       |
 | G00017  | When creating an `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressGatewayName`, the tenant or cluster default gateway can be automatically configured and created successfully                                                                                                                                                                                                                                                                                                                                                                                                                                                             | p2       | false | done   |       |
 | G00018  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressGateway` succeeds                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | p2       | false | done   |       |
-| G00019  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully                                                                                                                                                                                                                                                                                                                                                                                           | p2       | false | done   |       |
+| G00019  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully                                                                                                                                                                                                                                                                                                                                                                                           | p2       | false | done   |       |
+| G00020  | When a `policy` references a `gateway`, deleting the `gateway` will be in a "deleting" state until all the `policies` that reference the `gateway` are deleted. The `gateway` will be successfully deleted once all the referencing `policies` are removed                                                           | p2        | false  |        |       |

test/doc/egressgateway_zh.md

@@ -25,6 +25,7 @@
 | G00017   | When creating an `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressGatewayName`, the tenant or cluster default gateway can be automatically configured and created successfully                                                             | p2        | false  |        |       |
 | G00018  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressGateway` succeeds                                                           | p2        | false  |        |       |
 | G00019  | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully                                                           | p2        | false  |        |       |
+| G00020  | When a policy references a gateway, deleting the gateway will be in a "deleting" state until all the policies that reference the gateway are deleted. The gateway will be successfully deleted once all the referencing policies are removed                                                           | p2        | false  |        |       |
 -->
 
 # EgressGateway E2E 用例
@@ -48,4 +49,5 @@
 | G00014 | 编辑 `NodeSelector` 使其匹配另一个节点，`Status.NodeList` 更新为新匹配的节点，使用该 EgressGateway 的 policy `Status.Node` 更新为新匹配的节点, `pod` 的出口 `ip` 为 `eip`<br>编辑 `NodeSelector` 使其不匹配任何节点，`Status.NodeList` 为空，用该 EgressGateway 的 policy `Status.Node` 为空, `pod` 使用访问外部 IP 会报错<br>编辑 `NodeSelector` 使其匹配一个节点，`Status.NodeList` 为所匹配的节点，使用该 EgressGateway 的 policy `Status.Node` 更新为新匹配的节点，`pod` 的出口 `ip` 为 `eip`                           | p2  | false |      |     |
 | G00017 | 创建 `EgressCluster` 或者 `EgressClusterPolicy` 时使用未指定 `spec.egressGatewayName` 时，可以使用自动设置租户或者集群默认网关，并创建成功 | p2  | false |    |    |
 | G00018 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 EgressGateway 成功 | p2  | false |    |    |
-| G00019 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 `EgressCluster` 或者 `EgressClusterPolicy`，未指定 `spec.egressIP.useNodeIP` 时，policy 创建失败，`spec.egressIP.useNodeIP` 为 true 时，policy 创建成功 | p2  | false |    |    |
+| G00019 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 `EgressCluster` 或者 `EgressClusterPolicy`，未指定 `spec.egressIP.useNodeIP` 时，policy 创建失败，`spec.egressIP.useNodeIP` 为 true 时，policy 创建成功 | p2  | false |    |    |
+| G00020 | 存在 `policy` 引用 `gateway` 时，删除 `gateway`，会处于 `deleting` 状态，直到所有引用 `gateway` 的 `policy` 都被删除，`gateway` 会被删除成功     | p2  | false |    |    |