From 9e6aa5e3ea1c38ce10195bfad9cf895075fcbcf9 Mon Sep 17 00:00:00 2001 From: bzsuni Date: Mon, 27 Nov 2023 18:21:41 +0800 Subject: [PATCH] Add finalizer to egressgateway Signed-off-by: bzsuni --- cmd/controller/cmd/clean.go | 7 ++ docs/usage/Install.en.md | 5 ++ docs/usage/Install.zh.md | 6 ++ pkg/controller/webhook/mutating.go | 1 + pkg/egressgateway/egress_gateway.go | 64 +++++++++++++- pkg/egressgateway/egress_gateway_webhook.go | 50 +++++------ pkg/utils/slice/slice.go | 21 +++++ pkg/utils/slice/slice_test.go | 31 +++++++ test/doc/egressgateway.md | 3 +- test/doc/egressgateway_zh.md | 4 +- test/e2e/egressgateway/egressgateway_test.go | 88 ++++++++++++++++++++ 11 files changed, 253 insertions(+), 27 deletions(-) create mode 100644 pkg/utils/slice/slice.go create mode 100644 pkg/utils/slice/slice_test.go diff --git a/cmd/controller/cmd/clean.go b/cmd/controller/cmd/clean.go index 1818dcf0d..9ee786fcd 100644 --- a/cmd/controller/cmd/clean.go +++ b/cmd/controller/cmd/clean.go @@ -116,6 +116,13 @@ func clean(validate, mutating string) error { err = cli.List(ctx, gatewayList) if err == nil { for _, item := range gatewayList.Items { + if len(item.Finalizers) != 0 { + (&item).Finalizers = nil + err := cli.Update(ctx, &item) + if err != nil { + return err + } + } err = cli.Delete(ctx, &item) if err != nil { return err diff --git a/docs/usage/Install.en.md b/docs/usage/Install.en.md index d0688a9c2..f7dd0a2be 100644 --- a/docs/usage/Install.en.md +++ b/docs/usage/Install.en.md @@ -273,3 +273,8 @@ helm repo update $ curl 10.6.1.92:8080 Remote IP: 172.22.0.110 ``` +## Uninstalling EgressGateway + In order to ensure uninterrupted business flow, the `egressgateway` incorporates a finalizer mechanism. When deleting the `egressgateway`, if there are `policies` referencing it, the `egressgateway` will remain in the "deleting" state until all `policies` are deleted, and the finalizer will be automatically removed. + Therefore, if you want to delete the `egressgateway`, it is recommended to follow these steps: + 1. Delete all `policies` that reference the `egressgateway`. + 2. Delete the `egressgateway`. \ No newline at end of file diff --git a/docs/usage/Install.zh.md b/docs/usage/Install.zh.md index c407079d1..f63a9ffb5 100644 --- a/docs/usage/Install.zh.md +++ b/docs/usage/Install.zh.md @@ -274,3 +274,9 @@ helm repo update $ curl 10.6.1.92:8080 Remote IP: 172.22.0.110 ``` + +## 卸载 EgressGateway + 为了保证业务不断流，`egressgateway` 中加入了 finalizer 机制，删除 `egressgateway` 时，如果存在 policy 引用此 `egressgateway`，`egressgateway` 会一直处于 deleting 状态, 直到 所有的 policy 被删除，finalizer 就会被自动删除。 + 所以，如果要删除 `egressgateway`，建议使用如下步骤： + 1. 删除所有引用 egressgateway 的 policy + 2. 删除 egressgateway \ No newline at end of file diff --git a/pkg/controller/webhook/mutating.go b/pkg/controller/webhook/mutating.go index 661813404..6dd7cbd4f 100644 --- a/pkg/controller/webhook/mutating.go +++ b/pkg/controller/webhook/mutating.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "gomodules.xyz/jsonpatch/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/egressgateway/egress_gateway.go b/pkg/egressgateway/egress_gateway.go index d26b02ecc..78a5373b2 100644 --- a/pkg/egressgateway/egress_gateway.go +++ b/pkg/egressgateway/egress_gateway.go @@ -28,6 +28,7 @@ import ( egress "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" "github.com/spidernet-io/egressgateway/pkg/utils" "github.com/spidernet-io/egressgateway/pkg/utils/ip" + "github.com/spidernet-io/egressgateway/pkg/utils/slice" ) type egnReconciler struct { @@ -166,7 +167,23 @@ func (r egnReconciler) reconcileEGW(ctx context.Context, req reconcile.Request, if deleted { log.Info("request item is deleted") - return reconcile.Result{}, nil + p, err := getEgressGatewayPolicies(r.client, ctx, egw) + if err != nil { + log.Error(err, "getEgressGatewayPolicies when delete egressgateway") + return reconcile.Result{Requeue: true}, err + } + if containsEgressGatewayFinalizer(egw, egressGatewayFinalizers) && len(p) == 0 { + log.Info("remove the egressGatewayFinalizer") + removeEgressGatewayFinalizer(egw) + log.V(1).Info("remove the egressGatewayFinalizer", "ObjectMeta", egw.ObjectMeta) + + err = r.client.Update(ctx, egw) + if err != nil { + log.Error(err, "remove the egressGatewayFinalizer", "ObjectMeta", egw.ObjectMeta) + return reconcile.Result{Requeue: true}, err + } + } + return reconcile.Result{Requeue: false}, nil } if egw.Spec.NodeSelector.Selector == nil { @@ -1268,3 +1285,48 @@ func countGatewayIP(egw *egress.EgressGateway) (ipv4sFree, ipv6sFree, ipv4sTotal ipv4sTotal, ipv6sTotal, err = len(ipv4s), len(ipv6s), nil return } + +// removeEgressGatewayFinalizer if the egress gateway is being deleted +func removeEgressGatewayFinalizer(egw *egress.EgressGateway) { + if !egw.DeletionTimestamp.IsZero() { + if containsEgressGatewayFinalizer(egw, egressGatewayFinalizers) { + egw.Finalizers = slice.RemoveElement(egw.Finalizers, egressGatewayFinalizers) + } + } +} + +func getEgressGatewayPolicies(client client.Client, ctx context.Context, egw *egress.EgressGateway) ([]egress.Policy, error) { + policies := make([]egress.Policy, 0) + // list policy + policyList := &egress.EgressPolicyList{} + err := client.List(ctx, policyList) + if err != nil { + return nil, err + } + for _, p := range policyList.Items { + if p.Spec.EgressGatewayName == egw.Name { + policies = append(policies, egress.Policy{Name: p.Name, Namespace: p.Namespace}) + } + } + // list cluster policy + clusterPolicyList := &egress.EgressClusterPolicyList{} + err = client.List(ctx, clusterPolicyList) + if err != nil { + return nil, err + } + for _, cp := range clusterPolicyList.Items { + if cp.Spec.EgressGatewayName == egw.Name { + policies = append(policies, egress.Policy{Name: cp.Name, Namespace: cp.Namespace}) + } + } + return policies, nil +} + +func containsEgressGatewayFinalizer(gateway *egress.EgressGateway, finalizer string) bool { + for _, f := range gateway.ObjectMeta.Finalizers { + if f == finalizer { + return true + } + } + return false +} diff --git a/pkg/egressgateway/egress_gateway_webhook.go b/pkg/egressgateway/egress_gateway_webhook.go index 3672c8cc6..d731bc5d7 100644 --- a/pkg/egressgateway/egress_gateway_webhook.go +++ b/pkg/egressgateway/egress_gateway_webhook.go @@ -36,22 +36,10 @@ type patchOperation struct { Value interface{} `json:"value,omitempty"` } +var egressGatewayFinalizers = "egressgateway.spidernet.io/egressgateway" + func (egw *EgressGatewayWebhook) EgressGatewayValidate(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse { - // Check whether the deleted EgressGateway is referenced if req.Operation == v1.Delete { - delEG := new(egress.EgressGateway) - err := json.Unmarshal(req.OldObject.Raw, delEG) - if err != nil { - return webhook.Denied(fmt.Sprintf("json unmarshal EgressGateway with error: %v", err)) - } - - for _, item := range delEG.Status.NodeList { - for _, eip := range item.Eips { - if len(eip.Policies) != 0 { - return webhook.Denied(fmt.Sprintf("Do not delete %v:%v because it is already referenced by EgressPolicy", req.Namespace, req.Name)) - } - } - } return webhook.Allowed("checked") } @@ -173,7 +161,6 @@ func (egw *EgressGatewayWebhook) EgressGatewayValidate(ctx context.Context, req func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse { rander := rand.New(rand.NewSource(time.Now().UnixNano())) - isPatch := false eg := new(egress.EgressGateway) err := json.Unmarshal(req.Object.Raw, eg) if err != nil { @@ -181,8 +168,9 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we } reviewResponse := webhook.AdmissionResponse{} - var patch []patchOperation + var patchList []patchOperation + // patch egress gateway default eip if egw.Config.FileConfig.EnableIPv4 { if len(eg.Spec.Ippools.Ipv4DefaultEIP) == 0 && len(eg.Spec.Ippools.IPv4) != 0 { ipv4Ranges, err := ip.MergeIPRanges(constant.IPv4, eg.Spec.Ippools.IPv4) @@ -192,12 +180,11 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we ipv4s, _ := ip.ParseIPRanges(constant.IPv4, ipv4Ranges) if len(ipv4s) != 0 { - patch = append(patch, patchOperation{ + patchList = append(patchList, patchOperation{ Op: "add", Path: "/spec/ippools/ipv4DefaultEIP", Value: ipv4s[rander.Intn(len(ipv4s))].String(), }) - isPatch = true } } @@ -213,21 +200,25 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we ipv6s, _ := ip.ParseIPRanges(constant.IPv6, ipv6Ranges) if len(ipv6s) != 0 { - patch = append(patch, patchOperation{ + patchList = append(patchList, patchOperation{ Op: "add", Path: "/spec/ippools/ipv6DefaultEIP", Value: ipv6s[rander.Intn(len(ipv6s))].String(), }) - isPatch = true } - } } - if isPatch { - patchBytes, err := json.Marshal(patch) + // patch egress gateway finalizer + patch := getEgressGatewayFinalizerPatch(req, []string{egressGatewayFinalizers}) + if patch != nil { + patchList = append(patchList, *patch) + } + + if len(patchList) > 0 { + patchBytes, err := json.Marshal(patchList) if err != nil { - return webhook.Denied(fmt.Sprintf("failed to allocate defaultEIP.: %v", err)) + return webhook.Denied(fmt.Sprintf("failed to Marshal patchList.: %v", err)) } reviewResponse.Allowed = true @@ -240,3 +231,14 @@ func (egw *EgressGatewayWebhook) EgressGatewayMutate(ctx context.Context, req we return webhook.Allowed("checked") } + +func getEgressGatewayFinalizerPatch(req webhook.AdmissionRequest, finalizer []string) *patchOperation { + if req.Operation == v1.Create { + return &patchOperation{ + Op: "add", + Path: "/metadata/finalizers", + Value: finalizer, + } + } + return nil +} diff --git a/pkg/utils/slice/slice.go b/pkg/utils/slice/slice.go new file mode 100644 index 000000000..5d09c5e91 --- /dev/null +++ b/pkg/utils/slice/slice.go @@ -0,0 +1,21 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package slice + +func RemoveElement[T int | string](slice []T, element T) []T { + i := indexOf(slice, element) + if i == -1 { + return slice + } + return append(slice[:i], slice[i+1:]...) +} + +func indexOf[T int | string](slice []T, element T) int { + for i, v := range slice { + if v == element { + return i + } + } + return -1 +} diff --git a/pkg/utils/slice/slice_test.go b/pkg/utils/slice/slice_test.go new file mode 100644 index 000000000..160cc7e2a --- /dev/null +++ b/pkg/utils/slice/slice_test.go @@ -0,0 +1,31 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package slice + +import ( + "reflect" + "testing" +) + +func TestRemoveElement(t *testing.T) { + cases := []struct { + name string + in []string + el string + expect []string + }{ + {"when included", []string{"a", "b", "c"}, "a", []string{"b", "c"}}, + {"when not included", []string{"a", "b", "c"}, "d", []string{"a", "b", "c"}}, + {"when empty slice", []string{}, "a", []string{}}, + } + + for _, v := range cases { + t.Run(v.name, func(t *testing.T) { + out := RemoveElement[string](v.in, v.el) + if !reflect.DeepEqual(out, v.expect) { + t.Errorf("RemoveElement() got = %v, want = %v", out, v.expect) + } + }) + } +} diff --git a/test/doc/egressgateway.md b/test/doc/egressgateway.md index a86a46bc3..315515036 100644 --- a/test/doc/egressgateway.md +++ b/test/doc/egressgateway.md @@ -20,4 +20,5 @@ | G00014 | Edit the `NodeSelector` to match another `node`, `Status.NodeList` is updated to the newly matched node, using the `policy` of this `EgressGateway`, `Status.Node` is updated to the newly matched `node`, the `pod`'s egress ip is eip
Edit the `NodeSelector` to not match any `node`, `Status.NodeList` is empty, using the `policy` of this `EgressGateway`, `Status.Node` is empty, accessing external IP from the pod will error
Edit the `NodeSelector` to match one `node`, `Status.NodeList` is the matched `node`, using the `policy` of this `EgressGateway`, `Status.Node` is updated to the newly matched `node`, the `pod`'s egress ip is eip | p2 | false | | | | G00017 | When creating an `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressGatewayName`, the tenant or cluster default gateway can be automatically configured and created successfully | p2 | false | done | | | G00018 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressGateway` succeeds | p2 | false | done | | -| G00019 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully | p2 | false | done | | \ No newline at end of file +| G00019 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully | p2 | false | done | | +| G00020 | When a `policy` references a `gateway`, deleting the `gateway` will be in a "deleting" state until all the `policies` that reference the `gateway` are deleted. The `gateway` will be successfully deleted once all the referencing `policies` are removed | p2 | false | | | \ No newline at end of file diff --git a/test/doc/egressgateway_zh.md b/test/doc/egressgateway_zh.md index 61f844581..903929c72 100644 --- a/test/doc/egressgateway_zh.md +++ b/test/doc/egressgateway_zh.md @@ -25,6 +25,7 @@ | G00017 | When creating an `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressGatewayName`, the tenant or cluster default gateway can be automatically configured and created successfully | p2 | false | | | | G00018 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressGateway` succeeds | p2 | false | | | | G00019 | When `Ippools.IPv4` and `Ippools.IPv6` are empty, creating `EgressCluster` or `EgressClusterPolicy` without specifying `spec.egressIP.useNodeIP` will fail to create the policy. When `spec.egressIP.useNodeIP` is set to true, the policy will be created successfully | p2 | false | | | +| G00020 | When a policy references a gateway, deleting the gateway will be in a "deleting" state until all the policies that reference the gateway are deleted. The gateway will be successfully deleted once all the referencing policies are removed | p2 | false | | | --> # EgressGateway E2E 用例 @@ -48,4 +49,5 @@ | G00014 | 编辑 `NodeSelector` 使其匹配另一个节点，`Status.NodeList` 更新为新匹配的节点，使用该 EgressGateway 的 policy `Status.Node` 更新为新匹配的节点, `pod` 的出口 `ip` 为 `eip`
编辑 `NodeSelector` 使其不匹配任何节点，`Status.NodeList` 为空，用该 EgressGateway 的 policy `Status.Node` 为空, `pod` 使用访问外部 IP 会报错
编辑 `NodeSelector` 使其匹配一个节点，`Status.NodeList` 为所匹配的节点，使用该 EgressGateway 的 policy `Status.Node` 更新为新匹配的节点，`pod` 的出口 `ip` 为 `eip` | p2 | false | | | | G00017 | 创建 `EgressCluster` 或者 `EgressClusterPolicy` 时使用未指定 `spec.egressGatewayName` 时，可以使用自动设置租户或者集群默认网关，并创建成功 | p2 | false | | | | G00018 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 EgressGateway 成功 | p2 | false | | | -| G00019 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 `EgressCluster` 或者 `EgressClusterPolicy`，未指定 `spec.egressIP.useNodeIP` 时，policy 创建失败，`spec.egressIP.useNodeIP` 为 true 时，policy 创建成功 | p2 | false | | | \ No newline at end of file +| G00019 | 当 `Ippools.IPv4` 和 `Ippools.IPv6` 为空时，创建 `EgressCluster` 或者 `EgressClusterPolicy`，未指定 `spec.egressIP.useNodeIP` 时，policy 创建失败，`spec.egressIP.useNodeIP` 为 true 时，policy 创建成功 | p2 | false | | | +| G00020 | 存在 `policy` 引用 `gateway` 时，删除 `gateway`，会处于 `deleting` 状态，直到所有引用 `gateway` 的 `policy` 都被删除，`gateway` 会被删除成功 | p2 | false | | | \ No newline at end of file diff --git a/test/e2e/egressgateway/egressgateway_test.go b/test/e2e/egressgateway/egressgateway_test.go index fc445383f..0ab477109 100644 --- a/test/e2e/egressgateway/egressgateway_test.go +++ b/test/e2e/egressgateway/egressgateway_test.go @@ -15,7 +15,9 @@ import ( "github.com/go-faker/faker/v4" appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "github.com/spidernet-io/egressgateway/pkg/constant" egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" @@ -634,6 +636,92 @@ var _ = Describe("Operate EgressGateway", Label("EgressGateway"), Ordered, func( Expect(common.CheckDaemonSetEgressIP(ctx, cli, config, egressConfig, ds, egp.Status.Eip.Ipv4, egp.Status.Eip.Ipv6, true)).NotTo(HaveOccurred()) }) }) + + /* + Test Case: EgressGateway Finalizer Testing + + 1. Create an egress gateway and verify that the finalizer is added. + 2. Create a policy referencing the egress gateway created in the previous step. + 3. Delete the egress gateway and verify that it enters the "deleting" state but is not immediately deleted. + 4. Delete the policy, and verify that the egress gateway is subsequently deleted. + */ + Context("Delete egressGateway", func() { + var ctx context.Context + // gateway + var egw *egressv1.EgressGateway + + // policy + var egp *egressv1.EgressPolicy + + // lalbe + var label map[string]string + + // error + var err error + + var gatewayFinalizer = "egressgateway.spidernet.io/egressgateway" + + BeforeEach(func() { + ctx = context.Background() + + label = map[string]string{"test-finalizer": faker.Word()} + + // create gateway + egw = createEgressGateway(ctx) + + // check finalizer + Expect(egw.GetFinalizers()).Should(ContainElement(gatewayFinalizer), "failed to check egressgateway finzalizer") + + // create egressPolicy + egp, err = common.CreateEgressPolicyNew(ctx, cli, egressConfig, egw.Name, label) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Printf("Succeeded create EgressPolicy: %s\n", egp.Name) + + DeferCleanup(func() { + // delete the egp if it exists + if egp != nil { + GinkgoWriter.Printf("Delete egp: %s\n", egp.Name) + Expect(common.DeleteObj(ctx, cli, egp)).NotTo(HaveOccurred()) + } + + // delete the egw if it exists + if egw != nil { + GinkgoWriter.Printf("Delete egw: %s\n", egw.Name) + Expect(common.DeleteObj(ctx, cli, egw)).NotTo(HaveOccurred()) + } + }) + }) + + It("Test egressgateway finalizer", Label("G00020"), func() { + // delete gateway + GinkgoWriter.Printf("delete the egw: %s, we expect it to be in deleting status", egw.Name) + Expect(common.DeleteObj(ctx, cli, egw)).NotTo(HaveOccurred()) + Consistently(ctx, func() error { + err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw) + if err != nil { + return err + } + if egw.DeletionTimestamp.Time.IsZero() { + return fmt.Errorf("not found deletionTimeStamp") + } + return nil + }).WithTimeout(time.Second * 6).WithPolling(time.Second * 2).Should(Succeed()) + + // delete egp + GinkgoWriter.Printf("delete the egp: %s\n", egp.Name) + Expect(common.DeleteObj(ctx, cli, egp)).NotTo(HaveOccurred()) + + // we expect the egw will be delete after a while + Eventually(ctx, func() error { + err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw) + if errors.IsNotFound(err) { + return nil + } else { + return fmt.Errorf("get the egw: %s that is not our expected", egw.Name) + } + }).WithTimeout(time.Second * 6).WithPolling(time.Second * 2).Should(Succeed()) + }) + }) }) func createEgressGateway(ctx context.Context) (egw *egressv1.EgressGateway) {