Add CI to handle auto test and promote to strict and moonray branches (…

…#476)

* Add CI to handle auto test and promote to strict and moonray branches

* also apply patch when running tests

* fix missing namespace issues with calico

* Disable more tests until features are implemented

* Add --timeout flags to status check commands

* do not fail fast on failing informing tests

* skip cleanup test (not yet implemented)

* relax default timeout

* refactor check network ready

* improve wording of x-wait-for failure commands

* retry on failures of checking DNS and network

* never return false without error

* disable test_network too

* avoid returning false, nil on CheckNetwork cilium

* fixup cilium messages

.github/workflows/update-branches.yaml

@@ -38,14 +38,14 @@ jobs:
           else
             exit 1
           fi
-      - name: Sync ${{ github.ref }} to ${{ steps.determine.outputs.branch }}
+      - name: Sync ${{ github.ref }} to ${{ needs.prepare.outputs.branch }}
         uses: actions/checkout@v4
         with:
           ssh-key: ${{ secrets.DEPLOY_KEY_TO_UPDATE_STRICT_BRANCH }}
       - name: Apply ${{ matrix.patch }} patch
         run: |
-          git checkout -b ${{ steps.determine.outputs.branch }}
+          git checkout -b ${{ needs.prepare.outputs.branch }}
           ./build-scripts/patches/${{ matrix.patch }}/apply
-      - name: Push to ${{ steps.determine.outputs.branch }}
+      - name: Push to ${{ needs.prepare.outputs.branch }}
         run: |
-          git push origin --force ${{ steps.determine.outputs.branch }}
+          git push origin --force ${{ needs.prepare.outputs.branch }}

build-scripts/patches/moonray/apply

@@ -16,6 +16,8 @@ rm "${DIR}/../../../tests/integration/tests/test_gateway.py"
 rm "${DIR}/../../../tests/integration/tests/test_ingress.py"
 ## TODO: restore when cleanup is implemented
 rm "${DIR}/../../../tests/integration/tests/test_cleanup.py"
+## TODO: restore when network test is fixed
+rm "${DIR}/../../../tests/integration/tests/test_network.py"
 
 git commit -a -m "Remove unrelated tests"
 

src/k8s/cmd/k8s/k8s_x_wait_for.go

@@ -21,11 +21,12 @@ func newXWaitForCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 			ctx, cancel := context.WithTimeout(cmd.Context(), opts.timeout)
 			defer cancel()
 			if err := control.WaitUntilReady(ctx, func() (bool, error) {
-				err := features.StatusChecks.CheckDNS(cmd.Context(), env.Snap)
-				if err != nil {
-					cmd.PrintErrf("DNS not ready yet: %v\n", err.Error())
+				ok, err := features.StatusChecks.CheckDNS(cmd.Context(), env.Snap)
+				if ok {
+					return true, nil
 				}
-				return err == nil, nil
+				cmd.PrintErrf("DNS not ready yet: %v\n", err.Error())
+				return false, nil
 			}); err != nil {
 				cmd.PrintErrf("Error: DNS did not become ready: %v\n", err)
 				env.Exit(1)
@@ -41,11 +42,12 @@ func newXWaitForCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 			ctx, cancel := context.WithTimeout(cmd.Context(), opts.timeout)
 			defer cancel()
 			if err := control.WaitUntilReady(ctx, func() (bool, error) {
-				err := features.StatusChecks.CheckNetwork(cmd.Context(), env.Snap)
-				if err != nil {
-					cmd.PrintErrf("network not ready yet: %v\n", err.Error())
+				ok, err := features.StatusChecks.CheckNetwork(cmd.Context(), env.Snap)
+				if ok {
+					return true, nil
 				}
-				return err == nil, nil
+				cmd.PrintErrf("network not ready yet: %v\n", err.Error())
+				return false, nil
 			}); err != nil {
 				cmd.PrintErrf("Error: network did not become ready: %v\n", err)
 				env.Exit(1)

src/k8s/pkg/k8sd/features/calico/status.go

@@ -6,12 +6,27 @@ import (
 
 	"github.com/canonical/k8s/pkg/snap"
 
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+func podIsReady(pod v1.Pod) bool {
+	if pod.Status.Phase != v1.PodRunning {
+		return false
+	}
+
+	for _, condition := range pod.Status.Conditions {
+		if condition.Type == v1.PodReady && condition.Status == v1.ConditionTrue {
+			return true
+		}
+	}
+
+	return false
+}
+
 // CheckNetwork checks the status of the Calico pods in the Kubernetes cluster.
 // We verify that the tigera-operator and calico-node pods are Ready and in Running state.
-func CheckNetwork(ctx context.Context, snap snap.Snap) error {
+func CheckNetwork(ctx context.Context, snap snap.Snap) (bool, error) {
 	client, err := snap.KubernetesClient("calico-system")
 	if err != nil {
 		return fmt.Errorf("failed to create kubernetes client: %w", err)
@@ -27,10 +42,20 @@ func CheckNetwork(ctx context.Context, snap snap.Snap) error {
 		// check that calico-node pods are ready
 		{name: "calico-node", namespace: "calico-system", labels: map[string]string{"app.kubernetes.io/name": "calico-node"}},
 	} {
-		if err := client.CheckForReadyPods(ctx, check.namespace, metav1.ListOptions{
+		pods, err := client.ListPods(ctx, check.namespace, metav1.ListOptions{
 			LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: check.labels}),
-		}); err != nil {
-			return fmt.Errorf("%v pods not yet ready: %w", check.name, err)
+		})
+		if err != nil {
+			return false, fmt.Errorf("failed to get %v pods: %w", check.name, err)
+		}
+		if len(pods) == 0 {
+			return false, fmt.Errorf("no %v pods exist on the cluster", check.name)
+		}
+
+		for _, pod := range pods {
+			if !podIsReady(pod) {
+				return false, fmt.Errorf("%v pod %q not ready", check.name, pod.Name)
+			}
 		}
 	}
 

src/k8s/pkg/k8sd/features/cilium/status.go

@@ -15,18 +15,18 @@ func CheckNetwork(ctx context.Context, snap snap.Snap) error {
 		return fmt.Errorf("failed to create kubernetes client: %w", err)
 	}
 
-	for _, check := range []struct {
-		name      string
-		namespace string
-		labels    map[string]string
-	}{
-		{name: "cilium-operator", namespace: "kube-system", labels: map[string]string{"io.cilium/app": "operator"}},
-		{name: "cilium", namespace: "kube-system", labels: map[string]string{"k8s-app": "cilium"}},
-	} {
-		if err := client.CheckForReadyPods(ctx, check.namespace, metav1.ListOptions{
-			LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: check.labels}),
-		}); err != nil {
-			return fmt.Errorf("%v pods not yet ready: %w", check.name, err)
+	ciliumPods := map[string]string{
+		"cilium-operator": "io.cilium/app=operator",
+		"cilium":          "k8s-app=cilium",
+	}
+
+	for ciliumPod, selector := range ciliumPods {
+		isReady, err := client.IsPodReady(ctx, ciliumPod, "kube-system", metav1.ListOptions{LabelSelector: selector})
+		if err != nil {
+			return false, fmt.Errorf("failed to check if pod %q is ready: %w", ciliumPod, err)
+		}
+		if !isReady {
+			return false, fmt.Errorf("cilium pod %q is not yet ready", ciliumPod)
 		}
 	}
 

src/k8s/pkg/k8sd/features/coredns/status.go

@@ -29,6 +29,9 @@ func CheckDNS(ctx context.Context, snap snap.Snap) error {
 			return fmt.Errorf("%v pods not yet ready: %w", check.name, err)
 		}
 	}
+	if !isReady {
+		return false, fmt.Errorf("coredns pod not ready yet")
+	}
 
 	return nil
 }