Add test case R00001

Signed-off-by: bzsuni <[email protected]>

go.mod

@@ -25,7 +25,6 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.18.1
-	github.com/spidernet-io/e2eframework v0.0.0-20230403061847-445757b963b3
 	github.com/stretchr/testify v1.8.4
 	github.com/tigera/operator v1.32.3
 	github.com/vishvananda/netlink v1.2.1-beta.2.0.20230130171208-05506ada9f99
@@ -89,7 +88,6 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -146,7 +144,6 @@ require (
 	k8s.io/component-base v0.28.1 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
-	k8s.io/kubectl v0.27.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

go.sum

@@ -332,8 +332,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:VzM3TYHDgqPkettiP6I6q2jOeQFL4nrJM+UcAc4f6Fs=
-github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -483,8 +481,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.18.1 h1:rmuU42rScKWlhhJDyXZRKJQHXFX02chSVW1IvkPGiVM=
 github.com/spf13/viper v1.18.1/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
-github.com/spidernet-io/e2eframework v0.0.0-20230403061847-445757b963b3 h1:DwiGehYumYZUF3wUVQqSaY/WvpZ71zLeIJ1WaSNQn20=
-github.com/spidernet-io/e2eframework v0.0.0-20230403061847-445757b963b3/go.mod h1:fCnYp0IxYHYmGMMm7WDy8pTb3BICSMV2Z7sro9vJhrs=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -988,8 +984,6 @@ k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
-k8s.io/kubectl v0.27.2 h1:sSBM2j94MHBFRWfHIWtEXWCicViQzZsb177rNsKBhZg=
-k8s.io/kubectl v0.27.2/go.mod h1:GCOODtxPcrjh+EC611MqREkU8RjYBh10ldQCQ6zpFKw=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=

test/e2e/common/deploy.go

@@ -5,7 +5,6 @@ package common
 
 import (
 	"context"
-	"fmt"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -19,8 +18,8 @@ import (
 	e2eerr "github.com/spidernet-io/egressgateway/test/e2e/err"
 )
 
-func CreateDeploy(ctx context.Context, cli client.Client, name string, image string, repolicas int) (*appsv1.Deployment, error) {
-	ctx, cancel := context.WithTimeout(ctx, time.Second*20)
+func CreateDeploy(ctx context.Context, cli client.Client, name string, image string, repolicas int, timeout time.Duration) (*appsv1.Deployment, error) {
+	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
 	var terminationGracePeriodSeconds int64 = 0
@@ -58,7 +57,7 @@ func CreateDeploy(ctx context.Context, cli client.Client, name string, image str
 		select {
 		case <-ctx.Done():
 			_ = DeleteObj(context.Background(), cli, res)
-			return nil, fmt.Errorf("create DaemonSet time out")
+			return nil, e2eerr.ErrTimeout
 		default:
 			err := cli.Get(ctx, types.NamespacedName{Namespace: res.Namespace, Name: res.Name}, res)
 			if err != nil {
@@ -101,6 +100,11 @@ func WaitDeployDeleted(ctx context.Context, cli client.Client, deploy *appsv1.De
 				time.Sleep(time.Second / 2)
 				continue
 			}
+			pl, err := GetNodesPodList(ctx, cli, deploy.Spec.Template.Labels, []string{})
+			if err != nil || len(pl.Items) != 0 {
+				time.Sleep(time.Second / 2)
+				continue
+			}
 			return nil
 		}
 	}

test/e2e/common/egp.go

@@ -322,6 +322,52 @@ func WaitEgressPolicyStatusReady(ctx context.Context, cli client.Client, egp *eg
 	}
 }
 
+// WaitEgressClusterPolicyStatusReady waits for the EgressPolicy status.Eip to be allocated after the EgressPolicy is created
+func WaitEgressClusterPolicyStatusReady(ctx context.Context, cli client.Client, egcp *egressv1.EgressClusterPolicy, v4Enabled, v6Enabled bool, timeout time.Duration) error {
+	if !v4Enabled && !v6Enabled {
+		return fmt.Errorf("both v4 and v6 are not enabled")
+	}
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	var v4Ok, v6Ok bool
+
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("timeout to wait egressPolicy status ready")
+		default:
+			err := cli.Get(ctx, types.NamespacedName{Name: egcp.Name}, egcp)
+			if err != nil {
+				time.Sleep(time.Second / 2)
+				continue
+			}
+			if !egcp.Spec.EgressIP.UseNodeIP {
+				if v4Enabled && len(egcp.Status.Eip.Ipv4) != 0 {
+					v4Ok = true
+				}
+				if v6Enabled && len(egcp.Status.Eip.Ipv6) != 0 {
+					v6Ok = true
+				}
+			} else {
+				if len(egcp.Status.Eip.Ipv4) == 0 && len(egcp.Status.Eip.Ipv6) == 0 {
+					return nil
+				}
+			}
+			if v4Enabled && v6Enabled {
+				if v4Ok && v6Ok {
+					return nil
+				}
+			} else if v4Enabled && v4Ok {
+				return nil
+			} else if v6Enabled && v6Ok {
+				return nil
+			}
+			time.Sleep(time.Second / 2)
+		}
+	}
+}
+
 // CreateEgressPolicyWithEipAllocatorRR  creates an egressPolicy  and sets Spec.EgressIP.AllocatorPolicy to "rr"
 func CreateEgressPolicyWithEipAllocatorRR(ctx context.Context, cli client.Client, egw *egressv1.EgressGateway, labels map[string]string) (*egressv1.EgressPolicy, error) {
 	return CreateEgressPolicyCustom(ctx, cli,

test/e2e/common/kwok.go

@@ -4,13 +4,12 @@
 package common
 
 import (
+	"context"
 	"strconv"
 
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/spidernet-io/e2eframework/framework"
 )
 
 func GenerateKwokNodeYaml(n int) *corev1.Node {
@@ -38,7 +37,7 @@ func GenerateKwokNodeYaml(n int) *corev1.Node {
 		},
 		Spec: corev1.NodeSpec{
 			Taints: []corev1.Taint{
-				KwokNodeTaint,
+				// KwokNodeTaint,
 			},
 		},
 		Status: corev1.NodeStatus{
@@ -51,23 +50,28 @@ func GenerateKwokNodeYaml(n int) *corev1.Node {
 	return node
 }
 
-func CreateKwokNodes(f *framework.Framework, n int) error {
+func CreateKwokNodes(ctx context.Context, cli client.Client, n int) error {
 	for i := 0; i < n; i++ {
-		e := f.CreateResource(GenerateKwokNodeYaml(i))
-		if e != nil {
-			return e
+		err := cli.Create(ctx, GenerateKwokNodeYaml(i))
+		if err != nil {
+			return err
 		}
 	}
 	return nil
 }
 
-func GetKwokNodes(f *framework.Framework) (*corev1.NodeList, error) {
-	return f.GetNodeList(client.MatchingLabels(KwokNodeLabel))
+func GetKwokNodes(ctx context.Context, cli client.Client) (*corev1.NodeList, error) {
+	nodeList := new(corev1.NodeList)
+	err := cli.List(ctx, nodeList, client.MatchingLabels(KwokNodeLabel))
+	if err != nil {
+		return nil, err
+	}
+	return nodeList, nil
 }
 
-func DeleteKwokNodes(f *framework.Framework, nodes *corev1.NodeList) error {
+func DeleteKwokNodes(ctx context.Context, cli client.Client, nodes *corev1.NodeList) error {
 	for _, node := range nodes.Items {
-		err := f.DeleteResource(&node)
+		err := DeleteObj(ctx, cli, &node)
 		if err != nil {
 			return err
 		}

test/e2e/common/pod.go

@@ -134,7 +134,7 @@ func WaitPodRunning(ctx context.Context, cli client.Client, pod *corev1.Pod, tim
 				time.Sleep(time.Second)
 				continue
 			}
-			if pod.Status.Phase == corev1.PodRunning {
+			if pod.Status.Phase == corev1.PodRunning && IfContainerRunning(pod) {
 				return nil
 			}
 			time.Sleep(time.Second)
@@ -307,3 +307,13 @@ func DeletePodList(ctx context.Context, cli client.Client, podList *corev1.PodLi
 	}
 	return nil
 }
+
+// IfContainerRunning check if the containers of the pod running
+func IfContainerRunning(pod *corev1.Pod) bool {
+	for _, c := range pod.Status.ContainerStatuses {
+		if c.State.Running == nil || !c.Ready {
+			return false
+		}
+	}
+	return true
+}

test/e2e/egressendpointslice/egressendpointslice_test.go

@@ -92,7 +92,7 @@ var _ = Describe("Egressendpointslice", func() {
 		It("test the namespace-level policy", func() {
 			// create deploy
 			deployName := "deploy-" + uuid.NewString()
-			deploy, err = common.CreateDeploy(ctx, cli, deployName, config.Image, podNum)
+			deploy, err = common.CreateDeploy(ctx, cli, deployName, config.Image, podNum, time.Second*20)
 			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create daemonset %s\n", deployName))
 			GinkgoWriter.Printf("succeeded to create deploy %s\n", deploy.Name)
 
@@ -106,7 +106,7 @@ var _ = Describe("Egressendpointslice", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			// create deploy agen
-			deploy, err = common.CreateDeploy(ctx, cli, deployName, config.Image, podNum)
+			deploy, err = common.CreateDeploy(ctx, cli, deployName, config.Image, podNum, time.Second*20)
 			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create daemonset %s\n", deployName))
 			GinkgoWriter.Printf("succeeded to create deploy %s\n", deploy.Name)
 
@@ -133,7 +133,7 @@ var _ = Describe("Egressendpointslice", func() {
 		It("test the cluster-level policy", func() {
 			// create deploy
 			deployName := "deploy-" + uuid.NewString()
-			deploy, err = common.CreateDeploy(ctx, cli, deployName, config.Image, podNum)
+			deploy, err = common.CreateDeploy(ctx, cli, deployName, config.Image, podNum, time.Second*20)
 			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create daemonset %s\n", deployName))
 			GinkgoWriter.Printf("succeeded to create deploy %s\n", deploy.Name)
 
@@ -147,7 +147,7 @@ var _ = Describe("Egressendpointslice", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			// create deploy again
-			deploy, err = common.CreateDeploy(ctx, cli, deployName, config.Image, podNum)
+			deploy, err = common.CreateDeploy(ctx, cli, deployName, config.Image, podNum, time.Second*20)
 			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create daemonset %s\n", deployName))
 			GinkgoWriter.Printf("succeeded to create deploy %s\n", deploy.Name)
 

test/e2e/reliability/reliability_suite_test.go

@@ -32,6 +32,7 @@ var (
 	cli client.Client
 
 	nodeNameList, workerNodes []string
+	nodeLabel                 map[string]string
 )
 
 var _ = BeforeSuite(func() {
@@ -60,6 +61,8 @@ var _ = BeforeSuite(func() {
 	}
 	Expect(len(workerNodes) > 1).To(BeTrue(), "this test case needs at lest 2 worker nodes")
 
+	nodeLabel = nodes.Items[0].Labels
+
 	// get egressgateway config
 	configMap := &corev1.ConfigMap{}
 	err = cli.Get(ctx, types.NamespacedName{Name: "egressgateway", Namespace: config.Namespace}, configMap)