diff --git a/cmd/controller/cmd/clean.go b/cmd/controller/cmd/clean.go
index 1818dcf0d..9ee786fcd 100644
--- a/cmd/controller/cmd/clean.go
+++ b/cmd/controller/cmd/clean.go
@@ -116,6 +116,13 @@ func clean(validate, mutating string) error {
err = cli.List(ctx, gatewayList)
if err == nil {
for _, item := range gatewayList.Items {
+ if len(item.Finalizers) != 0 {
+ (&item).Finalizers = nil
+ err := cli.Update(ctx, &item)
+ if err != nil {
+ return err
+ }
+ }
err = cli.Delete(ctx, &item)
if err != nil {
return err
diff --git a/docs/usage/Install.en.md b/docs/usage/Install.en.md
index d0688a9c2..f7dd0a2be 100644
--- a/docs/usage/Install.en.md
+++ b/docs/usage/Install.en.md
@@ -273,3 +273,8 @@ helm repo update
$ curl 10.6.1.92:8080
Remote IP: 172.22.0.110
```
+## Uninstalling EgressGateway
+ In order to ensure uninterrupted business flow, the `egressgateway` incorporates a finalizer mechanism. When deleting the `egressgateway`, if there are `policies` referencing it, the `egressgateway` will remain in the "deleting" state until all `policies` are deleted, and the finalizer will be automatically removed.
+ Therefore, if you want to delete the `egressgateway`, it is recommended to follow these steps:
+ 1. Delete all `policies` that reference the `egressgateway`.
+ 2. Delete the `egressgateway`.
\ No newline at end of file
diff --git a/docs/usage/Install.zh.md b/docs/usage/Install.zh.md
index c407079d1..f63a9ffb5 100644
--- a/docs/usage/Install.zh.md
+++ b/docs/usage/Install.zh.md
@@ -274,3 +274,9 @@ helm repo update
$ curl 10.6.1.92:8080
Remote IP: 172.22.0.110
```
+
+## 卸载 EgressGateway
+ 为了保证业务不断流，`egressgateway` 中加入了 finalizer 机制，删除 `egressgateway` 时，如果存在 policy 引用此 `egressgateway`，`egressgateway` 会一直处于 deleting 状态, 直到 所有的 policy 被删除，finalizer 就会被自动删除。
+ 所以，如果要删除 `egressgateway`，建议使用如下步骤：
+ 1. 删除所有引用 egressgateway 的 policy
+ 2. 删除 egressgateway
\ No newline at end of file
diff --git a/pkg/controller/webhook/mutating.go b/pkg/controller/webhook/mutating.go
index 661813404..6dd7cbd4f 100644
--- a/pkg/controller/webhook/mutating.go
+++ b/pkg/controller/webhook/mutating.go
@@ -7,6 +7,7 @@ import (
"context"
"encoding/json"
"fmt"
+
"gomodules.xyz/jsonpatch/v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
diff --git a/pkg/egressgateway/egress_gateway.go b/pkg/egressgateway/egress_gateway.go
index d26b02ecc..78a5373b2 100644
--- a/pkg/egressgateway/egress_gateway.go
+++ b/pkg/egressgateway/egress_gateway.go
@@ -28,6 +28,7 @@ import (
egress "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1"
"github.com/spidernet-io/egressgateway/pkg/utils"
"github.com/spidernet-io/egressgateway/pkg/utils/ip"
+ "github.com/spidernet-io/egressgateway/pkg/utils/slice"
)
type egnReconciler struct {
@@ -166,7 +167,23 @@ func (r egnReconciler) reconcileEGW(ctx context.Context, req reconcile.Request,
if deleted {
log.Info("request item is deleted")
- return reconcile.Result{}, nil
+ p, err := getEgressGatewayPolicies(r.client, ctx, egw)
+ if err != nil {
+ log.Error(err, "getEgressGatewayPolicies when delete egressgateway")
+ return reconcile.Result{Requeue: true}, err
+ }
+ if containsEgressGatewayFinalizer(egw, egressGatewayFinalizers) && len(p) == 0 {
+ log.Info("remove the egressGatewayFinalizer")
+ removeEgressGatewayFinalizer(egw)
+ log.V(1).Info("remove the egressGatewayFinalizer", "ObjectMeta", egw.ObjectMeta)
+
+ err = r.client.Update(ctx, egw)
+ if err != nil {
+ log.Error(err, "remove the egressGatewayFinalizer", "ObjectMeta", egw.ObjectMeta)
+ return reconcile.Result{Requeue: true}, err
+ }
+ }
+ return reconcile.Result{Requeue: false}, nil
}
if egw.Spec.NodeSelector.Selector == nil {
@@ -1268,3 +1285,48 @@ func countGatewayIP(egw *egress.EgressGateway) (ipv4sFree, ipv6sFree, ipv4sTotal
ipv4sTotal, ipv6sTotal, err = len(ipv4s), len(ipv6s), nil
return
}
+
+// removeEgressGatewayFinalizer if the egress gateway is being deleted
+func removeEgressGatewayFinalizer(egw *egress.EgressGateway) {
+ if !egw.DeletionTimestamp.IsZero() {
+ if containsEgressGatewayFinalizer(egw, egressGatewayFinalizers) {
+ egw.Finalizers = slice.RemoveElement(egw.Finalizers, egressGatewayFinalizers)
+ }
+ }
+}
+
+func getEgressGatewayPolicies(client client.Client, ctx context.Context, egw *egress.EgressGateway) ([]egress.Policy, error) {
+ policies := make([]egress.Policy, 0)
+ // list policy
+ policyList := &egress.EgressPolicyList{}
+ err := client.List(ctx, policyList)
+ if err != nil {
+ return nil, err
+ }
+ for _, p := range policyList.Items {
+ if p.Spec.EgressGatewayName == egw.Name {
+ policies = append(policies, egress.Policy{Name: p.Name, Namespace: p.Namespace})
+ }
+ }
+ // list cluster policy
+ clusterPolicyList := &egress.EgressClusterPolicyList{}
+ err = client.List(ctx, clusterPolicyList)
+ if err != nil {
+ return nil, err
+ }
+ for _, cp := range clusterPolicyList.Items {
+ if cp.Spec.EgressGatewayName == egw.Name {
+ policies = append(policies, egress.Policy{Name: cp.Name, Namespace: cp.Namespace})
+ }
+ }
+ return policies, nil
+}
+
+func containsEgressGatewayFinalizer(gateway *egress.EgressGateway, finalizer string) bool {
+ for _, f := range gateway.ObjectMeta.Finalizers {
+ if f == finalizer {
+ return true
+ }
+ }
+ return false
+}
diff --git a/pkg/egressgateway/egress_gateway_webhook.go b/pkg/egressgateway/egress_gateway_webhook.go
index 3672c8cc6..d731bc5d7 100644
--- a/pkg/egressgateway/egress_gateway_webhook.go
+++ b/pkg/egressgateway/egress_gateway_webhook.go
@@ -36,22 +36,10 @@ type patchOperation struct {
Value interface{} `json:"value,omitempty"`
}
+var egressGatewayFinalizers = "egressgateway.spidernet.io/egressgateway"
+
func (egw *EgressGatewayWebhook) EgressGatewayValidate(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse {
- // Check whether the deleted EgressGateway is referenced
if req.Operation == v1.Delete {
- delEG := new(egress.EgressGateway)
- err := json.Unmarshal(req.OldObject.Raw, delEG)
- if err != nil {
- return webhook.Denied(fmt.Sprintf("json unmarshal EgressGateway with error: %v", err))
- }
-
- for _, item := range delEG.Status.NodeList {
- for _, eip := range item.Eips {
- if len(eip.Policies) != 0 {
- return webhook.Denied(fmt.Sprintf("Do not delete %v:%v because it is already referenced by EgressPolicy", req.Namespace, req.Name))
- }
- }
- }
return webhook.Allowed("checked")
}
@@ -173,7 +161,6 @@ func (egw *EgressGatewayWebhook) EgressGatewayValidate(ctx context.Context, req
func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse {
rander := rand.New(rand.NewSource(time.Now().UnixNano()))
- isPatch := false
eg := new(egress.EgressGateway)
err := json.Unmarshal(req.Object.Raw, eg)
if err != nil {
@@ -181,8 +168,9 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
}
reviewResponse := webhook.AdmissionResponse{}
- var patch []patchOperation
+ var patchList []patchOperation
+ // patch egress gateway default eip
if egw.Config.FileConfig.EnableIPv4 {
if len(eg.Spec.Ippools.Ipv4DefaultEIP) == 0 && len(eg.Spec.Ippools.IPv4) != 0 {
ipv4Ranges, err := ip.MergeIPRanges(constant.IPv4, eg.Spec.Ippools.IPv4)
@@ -192,12 +180,11 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
ipv4s, _ := ip.ParseIPRanges(constant.IPv4, ipv4Ranges)
if len(ipv4s) != 0 {
- patch = append(patch, patchOperation{
+ patchList = append(patchList, patchOperation{
Op: "add",
Path: "/spec/ippools/ipv4DefaultEIP",
Value: ipv4s[rander.Intn(len(ipv4s))].String(),
})
- isPatch = true
}
}
@@ -213,21 +200,25 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
ipv6s, _ := ip.ParseIPRanges(constant.IPv6, ipv6Ranges)
if len(ipv6s) != 0 {
- patch = append(patch, patchOperation{
+ patchList = append(patchList, patchOperation{
Op: "add",
Path: "/spec/ippools/ipv6DefaultEIP",
Value: ipv6s[rander.Intn(len(ipv6s))].String(),
})
- isPatch = true
}
-
}
}
- if isPatch {
- patchBytes, err := json.Marshal(patch)
+ // patch egress gateway finalizer
+ patch := getEgressGatewayFinalizerPatch(req, []string{egressGatewayFinalizers})
+ if patch != nil {
+ patchList = append(patchList, *patch)
+ }
+
+ if len(patchList) > 0 {
+ patchBytes, err := json.Marshal(patchList)
if err != nil {
- return webhook.Denied(fmt.Sprintf("failed to allocate defaultEIP.: %v", err))
+ return webhook.Denied(fmt.Sprintf("failed to Marshal patchList.: %v", err))
}
reviewResponse.Allowed = true
@@ -240,3 +231,14 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we
return webhook.Allowed("checked")
}
+
+func getEgressGatewayFinalizerPatch(req webhook.AdmissionRequest, finalizer []string) *patchOperation {
+ if req.Operation == v1.Create {
+ return &patchOperation{
+ Op: "add",
+ Path: "/metadata/finalizers",
+ Value: finalizer,
+ }
+ }
+ return nil
+}
diff --git a/pkg/utils/slice/slice.go b/pkg/utils/slice/slice.go
new file mode 100644
index 000000000..5d09c5e91
--- /dev/null
+++ b/pkg/utils/slice/slice.go
@@ -0,0 +1,21 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package slice
+
+func RemoveElement[T int | string](slice []T, element T) []T {
+ i := indexOf(slice, element)
+ if i == -1 {
+ return slice
+ }
+ return append(slice[:i], slice[i+1:]...)
+}
+
+func indexOf[T int | string](slice []T, element T) int {
+ for i, v := range slice {
+ if v == element {
+ return i
+ }
+ }
+ return -1
+}
diff --git a/pkg/utils/slice/slice_test.go b/pkg/utils/slice/slice_test.go
new file mode 100644
index 000000000..160cc7e2a
--- /dev/null
+++ b/pkg/utils/slice/slice_test.go
@@ -0,0 +1,31 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package slice
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestRemoveElement(t *testing.T) {
+ cases := []struct {
+ name string
+ in []string
+ el string
+ expect []string
+ }{
+ {"when included", []string{"a", "b", "c"}, "a", []string{"b", "c"}},
+ {"when not included", []string{"a", "b", "c"}, "d", []string{"a", "b", "c"}},
+ {"when empty slice", []string{}, "a", []string{}},
+ }
+
+ for _, v := range cases {
+ t.Run(v.name, func(t *testing.T) {
+ out := RemoveElement[string](v.in, v.el)
+ if !reflect.DeepEqual(out, v.expect) {
+ t.Errorf("RemoveElement() got = %v, want = %v", out, v.expect)
+ }
+ })
+ }
+}
diff --git a/test/doc/egressgateway.md b/test/doc/egressgateway.md
index a86a46bc3..315515036 100644
--- a/test/doc/egressgateway.md
+++ b/test/doc/egressgateway.md
@@ -20,4 +20,5 @@
| G00014 | Edit the `NodeSelector` to match another `node`, `Status.NodeList` is updated to the newly matched node, using the `policy` of this `EgressGateway`, `Status.Node` is updated to the newly matched `node`, the `pod`'s egress ip is eip
Edit the `NodeSelector` to not match any `node`, `Status.NodeList` is empty, using the `policy` of this `EgressGateway`, `Status.Node` is empty, accessing external IP from the pod will error
Edit the `NodeSelector` to match one `node`, `Status.NodeList` is the matched `node`, using the `policy` of this `EgressGateway`, `Status.Node` is updated to the newly matched `node`, the `pod`'s egress ip is eip | p2 | false | | |
| G00017 | When creating an `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressGatewayName`, the tenant or cluster default gateway can be automatically configured and created successfully | p2 | false | done | |
| G00018 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressGateway` succeeds | p2 | false | done | |
-| G00019 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully | p2 | false | done | |
\ No newline at end of file
+| G00019 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully | p2 | false | done | |
+| G00020 | When a `policy` references a `gateway`, deleting the `gateway` will be in a "deleting" state until all the `policies` that reference the `gateway` are deleted. The `gateway` will be successfully deleted once all the referencing `policies` are removed | p2 | false | | |
\ No newline at end of file
diff --git a/test/doc/egressgateway_zh.md b/test/doc/egressgateway_zh.md
index 61f844581..903929c72 100644
--- a/test/doc/egressgateway_zh.md
+++ b/test/doc/egressgateway_zh.md
@@ -25,6 +25,7 @@
| G00017 | When creating an `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressGatewayName`, the tenant or cluster default gateway can be automatically configured and created successfully | p2 | false | | |
| G00018 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressGateway` succeeds | p2 | false | | |
| G00019 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully | p2 | false | | |
+| G00020 | When a policy references a gateway, deleting the gateway will be in a "deleting" state until all the policies that reference the gateway are deleted. The gateway will be successfully deleted once all the referencing policies are removed | p2 | false | | |
-->
# EgressGateway E2E 用例
@@ -48,4 +49,5 @@
| G00014 | 编辑 `NodeSelector` 使其匹配另一个节点，`Status.NodeList` 更新为新匹配的节点，使用该 EgressGateway 的 policy `Status.Node` 更新为新匹配的节点, `pod` 的出口 `ip` 为 `eip`
编辑 `NodeSelector` 使其不匹配任何节点，`Status.NodeList` 为空，用该 EgressGateway 的 policy `Status.Node` 为空, `pod` 使用访问外部 IP 会报错
编辑 `NodeSelector` 使其匹配一个节点，`Status.NodeList` 为所匹配的节点，使用该 EgressGateway 的 policy `Status.Node` 更新为新匹配的节点，`pod` 的出口 `ip` 为 `eip` | p2 | false | | |
| G00017 | 创建 `EgressCluster` 或者 `EgressClusterPolicy` 时使用未指定 `spec.egressGatewayName` 时，可以使用自动设置租户或者集群默认网关，并创建成功 | p2 | false | | |
| G00018 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 EgressGateway 成功 | p2 | false | | |
-| G00019 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 `EgressCluster` 或者 `EgressClusterPolicy`，未指定 `spec.egressIP.useNodeIP` 时，policy 创建失败，`spec.egressIP.useNodeIP` 为 true 时，policy 创建成功 | p2 | false | | |
\ No newline at end of file
+| G00019 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 `EgressCluster` 或者 `EgressClusterPolicy`，未指定 `spec.egressIP.useNodeIP` 时，policy 创建失败，`spec.egressIP.useNodeIP` 为 true 时，policy 创建成功 | p2 | false | | |
+| G00020 | 存在 `policy` 引用 `gateway` 时，删除 `gateway`，会处于 `deleting` 状态，直到所有引用 `gateway` 的 `policy` 都被删除，`gateway` 会被删除成功 | p2 | false | | |
\ No newline at end of file
diff --git a/test/e2e/egressgateway/egressgateway_test.go b/test/e2e/egressgateway/egressgateway_test.go
index fc445383f..0ab477109 100644
--- a/test/e2e/egressgateway/egressgateway_test.go
+++ b/test/e2e/egressgateway/egressgateway_test.go
@@ -15,7 +15,9 @@ import (
"github.com/go-faker/faker/v4"
appsv1 "k8s.io/api/apps/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
"github.com/spidernet-io/egressgateway/pkg/constant"
egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1"
@@ -634,6 +636,92 @@ var _ = Describe("Operate EgressGateway", Label("EgressGateway"), Ordered, func(
Expect(common.CheckDaemonSetEgressIP(ctx, cli, config, egressConfig, ds, egp.Status.Eip.Ipv4, egp.Status.Eip.Ipv6, true)).NotTo(HaveOccurred())
})
})
+
+ /*
+ Test Case: EgressGateway Finalizer Testing
+
+ 1. Create an egress gateway and verify that the finalizer is added.
+ 2. Create a policy referencing the egress gateway created in the previous step.
+ 3. Delete the egress gateway and verify that it enters the "deleting" state but is not immediately deleted.
+ 4. Delete the policy, and verify that the egress gateway is subsequently deleted.
+ */
+ Context("Delete egressGateway", func() {
+ var ctx context.Context
+ // gateway
+ var egw *egressv1.EgressGateway
+
+ // policy
+ var egp *egressv1.EgressPolicy
+
+ // lalbe
+ var label map[string]string
+
+ // error
+ var err error
+
+ var gatewayFinalizer = "egressgateway.spidernet.io/egressgateway"
+
+ BeforeEach(func() {
+ ctx = context.Background()
+
+ label = map[string]string{"test-finalizer": faker.Word()}
+
+ // create gateway
+ egw = createEgressGateway(ctx)
+
+ // check finalizer
+ Expect(egw.GetFinalizers()).Should(ContainElement(gatewayFinalizer), "failed to check egressgateway finzalizer")
+
+ // create egressPolicy
+ egp, err = common.CreateEgressPolicyNew(ctx, cli, egressConfig, egw.Name, label)
+ Expect(err).NotTo(HaveOccurred())
+ GinkgoWriter.Printf("Succeeded create EgressPolicy: %s\n", egp.Name)
+
+ DeferCleanup(func() {
+ // delete the egp if it exists
+ if egp != nil {
+ GinkgoWriter.Printf("Delete egp: %s\n", egp.Name)
+ Expect(common.DeleteObj(ctx, cli, egp)).NotTo(HaveOccurred())
+ }
+
+ // delete the egw if it exists
+ if egw != nil {
+ GinkgoWriter.Printf("Delete egw: %s\n", egw.Name)
+ Expect(common.DeleteObj(ctx, cli, egw)).NotTo(HaveOccurred())
+ }
+ })
+ })
+
+ It("Test egressgateway finalizer", Label("G00020"), func() {
+ // delete gateway
+ GinkgoWriter.Printf("delete the egw: %s, we expect it to be in deleting status", egw.Name)
+ Expect(common.DeleteObj(ctx, cli, egw)).NotTo(HaveOccurred())
+ Consistently(ctx, func() error {
+ err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw)
+ if err != nil {
+ return err
+ }
+ if egw.DeletionTimestamp.Time.IsZero() {
+ return fmt.Errorf("not found deletionTimeStamp")
+ }
+ return nil
+ }).WithTimeout(time.Second * 6).WithPolling(time.Second * 2).Should(Succeed())
+
+ // delete egp
+ GinkgoWriter.Printf("delete the egp: %s\n", egp.Name)
+ Expect(common.DeleteObj(ctx, cli, egp)).NotTo(HaveOccurred())
+
+ // we expect the egw will be delete after a while
+ Eventually(ctx, func() error {
+ err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw)
+ if errors.IsNotFound(err) {
+ return nil
+ } else {
+ return fmt.Errorf("get the egw: %s that is not our expected", egw.Name)
+ }
+ }).WithTimeout(time.Second * 6).WithPolling(time.Second * 2).Should(Succeed())
+ })
+ })
})
func createEgressGateway(ctx context.Context) (egw *egressv1.EgressGateway) {