From 035de610b12086f492dd54ee90a9d779f243041e Mon Sep 17 00:00:00 2001 From: howard zhang Date: Thu, 2 Jan 2025 10:44:31 +0800 Subject: [PATCH 1/8] cluster-sync, kind: add setup_hostname_resolution_for_registry func For the Kind provider, we need to configure hostname resolution for the local image registry in the CoreDNS service. This ensures that local container images can be successfully pulled into Kubernetes pods during certain e2e tests. Signed-off-by: howard zhang --- cluster-sync/ephemeral_provider.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cluster-sync/ephemeral_provider.sh b/cluster-sync/ephemeral_provider.sh index 90428d25d3..f569900364 100644 --- a/cluster-sync/ephemeral_provider.sh +++ b/cluster-sync/ephemeral_provider.sh @@ -25,6 +25,17 @@ function seed_images(){ } +# For the Kind provider, we need to configure hostname resolution for the local image registry in the CoreDNS service. +# This ensures that local container images can be successfully pulled into Kubernetes pods during certain e2e tests. +function setup_hostname_resolution_for_registry { + host_name="registry" + host_ip=$(${CDI_CRI} inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(${CDI_CRI} ps|grep registry|awk '{print $1}')) + _kubectl patch configmap coredns \ + -n kube-system \ + --type merge \ + -p "{\"data\":{\"Corefile\":\".:53 {\n errors\n health {\n lameduck 5s\n }\n ready\n kubernetes cluster.local in-addr.arpa ip6.arpa {\n pods insecure\n fallthrough in-addr.arpa ip6.arpa\n ttl 30\n }\n prometheus :9153\n forward . /etc/resolv.conf {\n max_concurrent 1000\n }\n cache 30\n loop\n reload\n loadbalance\n hosts {\n $host_ip $host_name\n fallthrough\n }\n}\"}}" +} + function verify() { echo 'Wait until all nodes are ready' until [[ $(_kubectl get nodes --no-headers | wc -l) -eq $(_kubectl get nodes --no-headers | grep " Ready" | wc -l) ]]; do From 8f129eb86131f4222a0bf59268006f27112bedfe Mon Sep 17 00:00:00 2001 From: howard zhang Date: Thu, 2 Jan 2025 11:05:05 +0800 Subject: [PATCH 2/8] cluster-sync kind: support kind provider in cluster-sync Signed-off-by: howard zhang --- cluster-sync/ephemeral_provider.sh | 12 ++++++++---- cluster-sync/sync.sh | 28 ++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/cluster-sync/ephemeral_provider.sh b/cluster-sync/ephemeral_provider.sh index f569900364..a71a9be7ed 100644 --- a/cluster-sync/ephemeral_provider.sh +++ b/cluster-sync/ephemeral_provider.sh @@ -49,10 +49,14 @@ function configure_storage() { } function configure_hpp() { - for i in $(seq 1 ${KUBEVIRT_NUM_NODES}); do - ./cluster-up/ssh.sh "node$(printf "%02d" ${i})" "sudo mkdir -p /var/hpvolumes" - ./cluster-up/ssh.sh "node$(printf "%02d" ${i})" "sudo chcon -t container_file_t -R /var/hpvolumes" - done + if [[ $KUBEVIRT_PROVIDER =~ kind.* ]]; then + ./cluster-up/ssh.sh ${KUBEVIRT_PROVIDER}-control-plane mkdir -p /var/hpvolumes + else + for i in $(seq 1 ${KUBEVIRT_NUM_NODES}); do + ./cluster-up/ssh.sh "node$(printf "%02d" ${i})" "sudo mkdir -p /var/hpvolumes" + ./cluster-up/ssh.sh "node$(printf "%02d" ${i})" "sudo chcon -t container_file_t -R /var/hpvolumes" + done + fi HPP_RELEASE=$(get_latest_release "kubevirt/hostpath-provisioner-operator") _kubectl apply -f https://github.com/kubevirt/hostpath-provisioner-operator/releases/download/$HPP_RELEASE/namespace.yaml #install cert-manager diff --git a/cluster-sync/sync.sh b/cluster-sync/sync.sh index bb4a22ee6b..7fade3e527 100755 --- a/cluster-sync/sync.sh +++ b/cluster-sync/sync.sh @@ -37,11 +37,21 @@ PULL_POLICY=${PULL_POLICY:-IfNotPresent} # have to refactor/rewrite any of the code that works currently. MANIFEST_REGISTRY=$DOCKER_PREFIX if [ "${KUBEVIRT_PROVIDER}" != "external" ]; then - registry=${IMAGE_REGISTRY:-localhost:$(_port registry)} + if [[ $KUBEVIRT_PROVIDER =~ kind.* ]]; then + registry=${IMAGE_REGISTRY:-localhost:5000} + else + registry=${IMAGE_REGISTRY:-localhost:$(_port registry)} + fi DOCKER_PREFIX=${registry} MANIFEST_REGISTRY="registry:5000" fi +# When the Kubevirt provider is kind, We set up the cluster level hostname resolution for registry, thus, we can +# visit the registry:5000 in pods. +if [[ $KUBEVIRT_PROVIDER =~ kind.* ]]; then + setup_hostname_resolution_for_registry +fi + if [ "${KUBEVIRT_PROVIDER}" == "external" ]; then # No kubevirtci local registry, likely using something external if [[ $(${CDI_CRI} login --help | grep authfile) ]]; then @@ -96,8 +106,16 @@ function wait_cdi_available { } function configure_uploadproxy_override { - host_port=$(./cluster-up/cli.sh ports uploadproxy | xargs) - override="https://127.0.0.1:$host_port" + if [[ $KUBEVIRT_PROVIDER =~ kind.* ]]; then + # To enable port mapping, it must be configured both in the Kind configuration and the uploadProxyURLOverride. + # We use the environment variable KIND_PORT_MAPPING to ensure the setup is applied in both locations. + container_port=$(echo "$KIND_PORT_MAPPING" | awk -F: '{print $1}') + host_port=$(echo "$KIND_PORT_MAPPING" | awk -F: '{print $2}') + override="https://127.0.0.1:$host_port" + else + host_port=$(./cluster-up/cli.sh ports uploadproxy | xargs) + override="https://127.0.0.1:$host_port" + fi _kubectl patch cdi ${CR_NAME} --type=merge -p '{"spec": {"config": {"uploadProxyURLOverride": "'"$override"'"}}}' } @@ -198,7 +216,9 @@ fi mkdir -p ./_out/tests rm -f $OLD_CDI_VER_PODS $NEW_CDI_VER_PODS -seed_images +if [[ ! $KUBEVIRT_PROVIDER =~ kind.* ]]; then + seed_images +fi # Install CDI install_cdi From d35b9269745c6cfa2424b06cc2a82693f9c1ab0a Mon Sep 17 00:00:00 2001 From: howard zhang Date: Thu, 2 Jan 2025 12:28:14 +0800 Subject: [PATCH 3/8] bazel arm64: stop build and deploy vddk on Arm64 VDDK is not supported on Arm64 Signed-off-by: howard zhang --- BUILD.bazel | 1 - cluster-sync/sync.sh | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index 938dc3c0ec..569e06a36f 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -81,7 +81,6 @@ container_bundle( "$(container_prefix)/cdi-func-test-registry-populate:$(container_tag)": "//tools/cdi-func-test-registry-init:cdi-func-test-registry-populate-image", "$(container_prefix)/cdi-func-test-registry:$(container_tag)": "//tools/cdi-func-test-registry-init:cdi-func-test-registry-image", "$(container_prefix)/imageio-init:$(container_tag)": "//tools/imageio-init:imageio-init-image", - "$(container_prefix)/vcenter-simulator:$(container_tag)": "//tools/vddk-test:vcenter-simulator", "$(container_prefix)/cdi-func-test-tinycore:$(container_tag)": "//tests:cdi-func-test-tinycore", "$(container_prefix)/cdi-func-test-imageio:$(container_tag)": "//tools/image-io:cdi-func-test-imageio-image", "$(container_prefix)/cdi-func-test-cirros-qcow2:$(container_tag)": "//tests:cdi-func-test-cirros-qcow2", diff --git a/cluster-sync/sync.sh b/cluster-sync/sync.sh index 7fade3e527..b86329534d 100755 --- a/cluster-sync/sync.sh +++ b/cluster-sync/sync.sh @@ -203,6 +203,10 @@ if [ "${CDI_SYNC}" == "test-infra" ]; then if [ "${ARCHITECTURE}" != "s390x" ]; then # Imageio test service: _kubectl apply -f "./_out/manifests/imageio.yaml" + fi + + # Disable deploy VDDK on s390x and arm64 + if [ "${ARCHITECTURE}" != "s390x" ] && [ "${ARCHITECTURE}" != "aarch64" ]; then # vCenter (VDDK) test service: _kubectl apply -f "./_out/manifests/vcenter.yaml" fi From 700f3e27bc00fbc22fb92d96f95fc567416743b7 Mon Sep 17 00:00:00 2001 From: howard zhang Date: Thu, 2 Jan 2025 12:43:37 +0800 Subject: [PATCH 4/8] cluster-sync: add capabilities for registry-populate in kind Add SYS_ADMIN and SYS_CHROOT capability to make buildah work in kind cluster Signed-off-by: howard zhang --- cluster-sync/sync.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cluster-sync/sync.sh b/cluster-sync/sync.sh index b86329534d..9d00a4d7e1 100755 --- a/cluster-sync/sync.sh +++ b/cluster-sync/sync.sh @@ -180,6 +180,15 @@ function setup_for_upgrade_testing { _kubectl apply -f "./_out/manifests/cdi-testing-sa.yaml" _kubectl apply -f "./_out/manifests/file-host.yaml" _kubectl apply -f "./_out/manifests/registry-host.yaml" + + # In the kind cluster, registry-populate need more capability to use buildah + if [[ $KUBEVIRT_PROVIDER =~ kind.* ]]; then + _kubectl patch deployment \ + -n ${CDI_NAMESPACE} cdi-docker-registry-host \ + --type=json \ + -p='[{"op": "add", "path": "/spec/template/spec/containers/2/securityContext/capabilities/add", "value": ["SETFCAP", "SYS_ADMIN", "SYS_CHROOT"]}]' + fi + echo "Waiting for testing tools to be ready" _kubectl wait pod -n ${CDI_NAMESPACE} --for=condition=Ready --all --timeout=${CDI_AVAILABLE_TIMEOUT}s _kubectl apply -f "./_out/manifests/upgrade-testing-artifacts.yaml" @@ -199,6 +208,14 @@ if [ "${CDI_SYNC}" == "test-infra" ]; then _kubectl apply -f "./_out/manifests/sample-populator.yaml" _kubectl apply -f "./_out/manifests/uploadproxy-nodeport.yaml" + # In the kind cluster, registry-populate need more capability to use buildah + if [[ $KUBEVIRT_PROVIDER =~ kind.* ]]; then + _kubectl patch deployment \ + -n ${CDI_NAMESPACE} cdi-docker-registry-host \ + --type=json \ + -p='[{"op": "add", "path": "/spec/template/spec/containers/2/securityContext/capabilities/add", "value": ["SETFCAP", "SYS_ADMIN", "SYS_CHROOT"]}]' + fi + # Disable unsupported functest images for s390x if [ "${ARCHITECTURE}" != "s390x" ]; then # Imageio test service: From c107923c2e26fcf0e1bcf0da595c9d642b61ddd5 Mon Sep 17 00:00:00 2001 From: howard zhang Date: Thu, 2 Jan 2025 12:50:07 +0800 Subject: [PATCH 5/8] e2e: add vddk label Signed-off-by: howard zhang --- tests/datavolume_test.go | 6 +++--- tests/import_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/datavolume_test.go b/tests/datavolume_test.go index 0af8362fde..b3b88ce8de 100644 --- a/tests/datavolume_test.go +++ b/tests/datavolume_test.go @@ -1074,7 +1074,7 @@ var _ = Describe("[vendor:cnv-qe@redhat.com][level:component]DataVolume tests", Message: "Import Complete", Reason: "Completed", }}), - Entry("[test_id:5077]succeed creating import dv from VDDK source", dataVolumeTestArguments{ + Entry("[test_id:5077]succeed creating import dv from VDDK source", Label("VDDK"), dataVolumeTestArguments{ name: "dv-import-vddk", size: "1Gi", url: vcenterURL, @@ -1202,7 +1202,7 @@ var _ = Describe("[vendor:cnv-qe@redhat.com][level:component]DataVolume tests", testDataVolume(args) }, - Entry("[test_id:5079]should fail with \"AwaitingVDDK\" reason when VDDK image config map is not present", dataVolumeTestArguments{ + Entry("[test_id:5079]should fail with \"AwaitingVDDK\" reason when VDDK image config map is not present", Label("VDDK"), dataVolumeTestArguments{ name: "dv-awaiting-vddk", size: "1Gi", url: vcenterURL, @@ -1230,7 +1230,7 @@ var _ = Describe("[vendor:cnv-qe@redhat.com][level:component]DataVolume tests", Type: cdiv1.DataVolumeRunning, Status: v1.ConditionFalse, }}), - Entry("[test_id:5080]succeed importing VDDK data volume with init image URL set", dataVolumeTestArguments{ + Entry("[test_id:5080]succeed importing VDDK data volume with init image URL set", Label("VDDK"), dataVolumeTestArguments{ name: "dv-import-vddk", size: "1Gi", url: vcenterURL, diff --git a/tests/import_test.go b/tests/import_test.go index b13a74ba23..84965e37f6 100644 --- a/tests/import_test.go +++ b/tests/import_test.go @@ -1268,7 +1268,7 @@ var _ = Describe("Preallocation", func() { dataVolume.Annotations[controller.AnnPodRetainAfterCompletion] = "true" return dataVolume }), - Entry("VddkImport", true, utils.VcenterMD5, utils.DefaultImagePath, func() *cdiv1.DataVolume { + Entry("VddkImport", Label("VDDK"), true, utils.VcenterMD5, utils.DefaultImagePath, func() *cdiv1.DataVolume { // Find vcenter-simulator pod pod, err := utils.FindPodByPrefix(f.K8sClient, f.CdiInstallNs, "vcenter-deployment", "app=vcenter") Expect(err).ToNot(HaveOccurred()) @@ -1609,8 +1609,8 @@ var _ = Describe("Import populator", func() { Entry("[test_id:11005]with Registry image without preallocation", utils.TinyCoreMD5, createRegistryImportPopulatorCR, false, false), Entry("[test_id:11006]with ImageIO image with preallocation", Serial, utils.ImageioMD5, createImageIOImportPopulatorCR, true, false), Entry("[test_id:11007]with ImageIO image without preallocation", Serial, utils.ImageioMD5, createImageIOImportPopulatorCR, false, false), - Entry("[test_id:11008]with VDDK image with preallocation", utils.VcenterMD5, createVDDKImportPopulatorCR, true, false), - Entry("[test_id:11009]with VDDK image without preallocation", utils.VcenterMD5, createVDDKImportPopulatorCR, false, false), + Entry("[test_id:11008]with VDDK image with preallocation", Label("VDDK"), utils.VcenterMD5, createVDDKImportPopulatorCR, true, false), + Entry("[test_id:11009]with VDDK image without preallocation", Label("VDDK"), utils.VcenterMD5, createVDDKImportPopulatorCR, false, false), Entry("[test_id:11010]with Blank image with preallocation", utils.BlankMD5, createBlankImportPopulatorCR, true, false), Entry("[test_id:11011]with Blank image without preallocation", utils.BlankMD5, createBlankImportPopulatorCR, false, false), ) @@ -1658,7 +1658,7 @@ var _ = Describe("Import populator", func() { Entry("with HTTP image", utils.TinyCoreMD5, createHTTPImportPopulatorCR), Entry("with Registry image", utils.TinyCoreMD5, createRegistryImportPopulatorCR), Entry("with ImageIO image", Serial, utils.ImageioMD5, createImageIOImportPopulatorCR), - Entry("with VDDK image", utils.VcenterMD5, createVDDKImportPopulatorCR), + Entry("with VDDK image", Label("VDDK"), utils.VcenterMD5, createVDDKImportPopulatorCR), Entry("with Blank image", utils.BlankMD5, createBlankImportPopulatorCR), ) From 4a92a543121a1019abe06eb57aa1d7314f2570ac Mon Sep 17 00:00:00 2001 From: howard zhang Date: Thu, 2 Jan 2025 12:54:27 +0800 Subject: [PATCH 6/8] e2e: add no-kubernetes-in-docker label This label means the e2e tests are not suitable to test in containerized k8s env (kind provider). In this commit, we label two tests because the large size images would lead to OOM of cdi-uploader. For more detail, please refer to: https://github.com/kubevirt/containerized-data-importer/pull/3577#discussion_r1903547758 Signed-off-by: howard zhang --- tests/import_test.go | 2 +- tests/upload_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/import_test.go b/tests/import_test.go index 84965e37f6..8393210972 100644 --- a/tests/import_test.go +++ b/tests/import_test.go @@ -57,7 +57,7 @@ var _ = Describe("[rfe_id:1115][crit:high][vendor:cnv-qe@redhat.com][level:compo ns = f.Namespace.Name }) - DescribeTable("[test_id:2329] Should fail to import images that require too much space", func(uploadURL string) { + DescribeTable("[test_id:2329] Should fail to import images that require too much space", Label("no-kubernetes-in-docker"), func(uploadURL string) { imageURL := fmt.Sprintf(uploadURL, f.CdiInstallNs) By(imageURL) diff --git a/tests/upload_test.go b/tests/upload_test.go index e89b81657d..9c442d6969 100644 --- a/tests/upload_test.go +++ b/tests/upload_test.go @@ -307,7 +307,7 @@ var _ = Describe("[rfe_id:138][crit:high][vendor:cnv-qe@redhat.com][level:compon Entry("fail given a large physical size QCOW2 file", utils.UploadFileLargePhysicalDiskQcow), ) - DescribeTable("[posneg:negative][test_id:2330]Verify failure on sync upload if virtual size > pvc size", Serial, func(filename string) { + DescribeTable("[posneg:negative][test_id:2330]Verify failure on sync upload if virtual size > pvc size", Label("no-kubernetes-in-docker"), Serial, func(filename string) { By("Verify PVC annotation says ready") found, err := utils.WaitPVCPodStatusReady(f.K8sClient, pvc) Expect(err).ToNot(HaveOccurred()) From 4c57f848b2d4d9672571893b68369c1060e1c883 Mon Sep 17 00:00:00 2001 From: howard zhang Date: Thu, 2 Jan 2025 12:57:37 +0800 Subject: [PATCH 7/8] e2e: amend the VerifyPermissions strings The output of ls -ln for a file with 660 permissions can be either -rw-rw----. or -rw-rw---- Signed-off-by: howard zhang --- tests/framework/pvc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/framework/pvc.go b/tests/framework/pvc.go index 82fea27591..a441475c8e 100644 --- a/tests/framework/pvc.go +++ b/tests/framework/pvc.go @@ -340,7 +340,7 @@ func (f *Framework) VerifyPermissions(namespace *k8sv1.Namespace, pvc *k8sv1.Per return f.verifyInPod(namespace, pvc, cmd, func(output, stderr string) (bool, error) { fmt.Fprintf(ginkgo.GinkgoWriter, "INFO: permissions of disk.img: %s\n", output) - return strings.Compare(output, "-rw-rw----.") == 0, nil + return strings.Contains(output, "-rw-rw----"), nil }) } From 4881c706dacffb4a0d23ee1a49d0ff74f0c7c01e Mon Sep 17 00:00:00 2001 From: howard zhang Date: Thu, 2 Jan 2025 13:14:26 +0800 Subject: [PATCH 8/8] automation, test: add kind provider in automation test Signed-off-by: howard zhang --- automation/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automation/test.sh b/automation/test.sh index 1e5f034653..dff9c32d85 100755 --- a/automation/test.sh +++ b/automation/test.sh @@ -44,6 +44,9 @@ if [[ $TARGET =~ openshift-.* ]]; then elif [[ $TARGET =~ k8s-.* ]]; then export KUBEVIRT_NUM_NODES=2 export KUBEVIRT_MEMORY_SIZE=8192 +elif [[ $TARGET =~ kind-.* ]]; then + export KUBEVIRT_NUM_NODES=1 + export KIND_PORT_MAPPING=31001:31002 fi if [ ! -d "cluster-up/cluster/$KUBEVIRT_PROVIDER" ]; then @@ -68,7 +71,6 @@ if [[ -z "$UPGRADE_FROM" ]] && [[ -z "$RANDOM_CR" ]]; then fi echo "Upgrading from versions: $UPGRADE_FROM" fi -export KUBEVIRT_NUM_NODES=2 kubectl() { cluster-up/kubectl.sh "$@"; }