diff --git a/test/e2e/common/pod.go b/test/e2e/common/pod.go index 95cc31ec0..dc1e63f62 100644 --- a/test/e2e/common/pod.go +++ b/test/e2e/common/pod.go @@ -247,3 +247,63 @@ func IfPodListRestarted(pods *corev1.PodList) bool { } return true } + +// DeletePodsUntilReady delete pods until rerunning +func DeletePodsUntilReady(ctx context.Context, cli client.Client, labels map[string]string, timeout time.Duration) error { + pl, err := GetNodesPodList(ctx, cli, labels, []string{}) + if err != nil { + return err + } + oldUIDs := GetPodListUIDs(pl) + + err = DeletePodList(ctx, cli, pl) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + for { + select { + case <-ctx.Done(): + return e2eerr.ErrTimeout + default: + pl, err := GetNodesPodList(ctx, cli, labels, []string{}) + if err != nil { + continue + } + newUIDs := GetPodListUIDs(pl) + if len(e2etools.SubtractionSlice(oldUIDs, newUIDs)) != len(oldUIDs) { + time.Sleep(time.Second) + continue + } + + for _, p := range pl.Items { + if p.Status.Phase != corev1.PodRunning { + time.Sleep(time.Second) + continue + } + } + return nil + } + } +} + +func GetPodListUIDs(podList *corev1.PodList) []string { + res := make([]string, 0) + for _, p := range podList.Items { + res = append(res, string(p.UID)) + } + return res +} + +func DeletePodList(ctx context.Context, cli client.Client, podList *corev1.PodList) error { + for _, p := range podList.Items { + err := cli.Delete(ctx, &p) + if err != nil { + return err + } + } + return nil +} diff --git a/test/e2e/constant/constant.go b/test/e2e/constant/constant.go index 6e35b5c1f..432f21032 100644 --- a/test/e2e/constant/constant.go +++ b/test/e2e/constant/constant.go @@ -4,3 +4,13 @@ package constant var WebhookMsgClusterDefaultGateway = "A cluster can only have one default gateway" + +var ( + KubeControllerManagerLabel = map[string]string{"component": "kube-controller-manager"} + KubeApiServerLabel = map[string]string{"component": "kube-apiserver"} + KubeEtcdLabel = map[string]string{"component": "etcd"} + KubeSchedulerLabel = map[string]string{"component": "kube-scheduler"} + KubeProxyLabel = map[string]string{"k8s-app": "kube-proxy"} + CalicoNodeLabel = map[string]string{"k8s-app": "calico-node"} + CalicoControllerLabel = map[string]string{"k8s-app": "calico-kube-controllers"} +) diff --git a/test/e2e/reliability/reliability_test.go b/test/e2e/reliability/reliability_test.go index a226c5ddf..7d9bd8f1d 100644 --- a/test/e2e/reliability/reliability_test.go +++ b/test/e2e/reliability/reliability_test.go @@ -6,6 +6,8 @@ package reliability_test import ( "context" "fmt" + "sync" + "sync/atomic" "time" "github.com/go-faker/faker/v4" @@ -21,6 +23,7 @@ import ( egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" "github.com/spidernet-io/egressgateway/test/e2e/common" + "github.com/spidernet-io/egressgateway/test/e2e/constant" "github.com/spidernet-io/egressgateway/test/e2e/tools" ) @@ -210,6 +213,88 @@ var _ = Describe("Reliability", Serial, Label("Reliability"), func() { GinkgoWriter.Println("check the egress gateway status; the EIP should not drift") checkGatewayStatus(ctx, cli, pool, ipNum, gatewayNode, workerNodes[1:], []string{}, policy, egw, time.Second*5) }) + + /* + restart the components such as calico, etcd and kube-proxy + check the eip of the pods + check the status of the egress gateway crs + */ + DescribeTable("restart components", Serial, Label("R00007"), func(labels map[string]string, timeout time.Duration) { + // get gateway + beforeEgw := new(egressv1.EgressGateway) + err := cli.Get(ctx, types.NamespacedName{Name: egw.Name}, beforeEgw) + Expect(err).NotTo(HaveOccurred()) + + // get policy + beforPolicy := new(egressv1.EgressPolicy) + err = cli.Get(ctx, types.NamespacedName{Namespace: policy.Namespace, Name: policy.Name}, beforPolicy) + Expect(err).NotTo(HaveOccurred()) + + // get egressClusterInfo + beforeEgci := new(egressv1.EgressClusterInfo) + err = cli.Get(ctx, types.NamespacedName{Name: "default"}, beforeEgci) + Expect(err).NotTo(HaveOccurred()) + + // get egressEndPoints + beforeEgep, err := common.GetEgressEndPointSliceByEgressPolicy(ctx, cli, policy) + Expect(err).NotTo(HaveOccurred()) + + var wg sync.WaitGroup + var isRerun atomic.Bool + wg.Add(1) + + go func() { + defer GinkgoRecover() + defer wg.Done() + + err := common.DeletePodsUntilReady(ctx, cli, labels, timeout) + Expect(err).NotTo(HaveOccurred()) + isRerun.Store(true) + }() + + for !isRerun.Load() { + err := common.CheckDaemonSetEgressIP(ctx, cli, config, egressConfig, daemonSet, + policy.Status.Eip.Ipv4, policy.Status.Eip.Ipv6, true) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Printf("succeeded to check the export IP of the daemonSet: %s\n", daemonSet.Name) + time.Sleep(time.Second / 2) + } + wg.Wait() + + // check the gateway status + GinkgoWriter.Println("check egress gateway status") + nowEgw := new(egressv1.EgressGateway) + err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, nowEgw) + Expect(err).NotTo(HaveOccurred()) + Expect(nowEgw.Status).Should(Equal(beforeEgw.Status), fmt.Sprintf("expect %v\ngot: %v\n", beforeEgw.Status, nowEgw.Status)) + + // check the policy status + GinkgoWriter.Println("check egress policy status") + nowPolicy := new(egressv1.EgressPolicy) + err = cli.Get(ctx, types.NamespacedName{Namespace: policy.Namespace, Name: policy.Name}, nowPolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(nowPolicy.Status).Should(Equal(beforPolicy.Status), fmt.Sprintf("expect:\n%v\ngot:\n %v\n", beforPolicy.Status, nowPolicy.Status)) + + // check the egressClusterInfo status + nowEgci := new(egressv1.EgressClusterInfo) + err = cli.Get(ctx, types.NamespacedName{Name: "default"}, nowEgci) + Expect(err).NotTo(HaveOccurred()) + Expect(nowEgci.Status).Should(Equal(beforeEgci.Status), fmt.Sprintf("expect:\n%v\ngot:\n %v\n", beforeEgci.Status, nowEgci.Status)) + + // check the egressEndPointSlice + // get egressEndPoints + nowEgep, err := common.GetEgressEndPointSliceByEgressPolicy(ctx, cli, policy) + Expect(err).NotTo(HaveOccurred()) + Expect(nowEgep.Endpoints).Should(Equal(beforeEgep.Endpoints), fmt.Sprintf("expect:\n%v\ngot:\n %v\n", beforeEgep.Endpoints, nowEgep.Endpoints)) + }, + Entry("restart kube-controller-manager", constant.KubeControllerManagerLabel, time.Minute), + Entry("restart kube-apiserver", constant.KubeApiServerLabel, time.Minute), + Entry("restart etcd", constant.KubeEtcdLabel, time.Minute), + Entry("restart kube-scheduler", constant.KubeSchedulerLabel, time.Minute), + Entry("restart kube-proxy", constant.KubeProxyLabel, time.Minute), + Entry("restart calico-node", constant.CalicoNodeLabel, time.Minute), + Entry("restart calico-kube-controllers", constant.CalicoControllerLabel, time.Minute), + ) }) })