Fix nightly tests

.github/workflows/integration.yaml

@@ -107,7 +107,17 @@ jobs:
           TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31"
           TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}, {"name": "rocks.canonical.com", "port": 5002, "remote": "https://rocks.canonical.com/cdk"}]'
         run: |
-          cd tests/integration && sg lxd -c 'tox -e integration -- --tags pull_request'
+          tags="pull_request"
+          # Run all tests if there are test changes. In case of a PR, we'll
+          # get a merge commit that includes all changes.
+          if git diff HEAD HEAD~1 --name-only | grep "tests/"; then
+            tags="up_to_weekly"
+          fi
+          # Run all tests on backports.
+          if echo ${{ github.base_ref }} | grep "release-"; then
+            tags="up_to_weekly"
+          fi
+          cd tests/integration && sg lxd -c "tox -e integration -- --tags $tags"
       - name: Prepare inspection reports
         if: failure()
         run: |

.github/workflows/nightly-test.yaml

@@ -17,7 +17,7 @@ jobs:
         release: ["latest/edge"]
       fail-fast: false # TODO: remove once arm64 works
 
-    runs-on: ${{ matrix.arch == 'arm64' && ["self-hosted", "Linux", "ARM64", "jammy", "large"] || ["self-hosted", "Linux", "AMD64", "jammy", "large"] }}
+    runs-on: ${{ matrix.arch == 'arm64' && 'self-hosted-linux-arm64-jammy-large' || 'self-hosted-linux-amd64-jammy-large' }}
 
     steps:
       - name: Checking out repo
@@ -30,6 +30,8 @@ jobs:
           sudo lxd init --auto
           sudo usermod --append --groups lxd $USER
           sg lxd -c 'lxc version'
+          sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT
+          sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
       - name: Create build directory
         run: mkdir -p build
       - name: Install ${{ matrix.release }} k8s snap

src/k8s/pkg/k8sd/api/cluster_join.go

@@ -37,16 +37,18 @@ func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Res
 
 	joinConfig := struct {
 		// We only care about this field from the entire join config.
-		containerdBaseDir string `yaml:"containerd-base-dir,omitempty"`
+		// Note that the field must be public (uppercase) in order for the yaml
+		// parser to handle it.
+		ContainerdBaseDir string `yaml:"containerd-base-dir,omitempty"`
 	}{}
 
 	if err := yaml.Unmarshal([]byte(req.Config), &joinConfig); err != nil {
 		return response.BadRequest(fmt.Errorf("failed to parse request config: %w", err))
 	}
 
-	if joinConfig.containerdBaseDir != "" {
+	if joinConfig.ContainerdBaseDir != "" {
 		// append k8s-containerd to the given base dir, so we don't flood it with our own folders.
-		e.provider.Snap().SetContainerdBaseDir(filepath.Join(joinConfig.containerdBaseDir, "k8s-containerd"))
+		e.provider.Snap().SetContainerdBaseDir(filepath.Join(joinConfig.ContainerdBaseDir, "k8s-containerd"))
 	}
 
 	config := map[string]string{}

tests/integration/tests/conftest.py

@@ -139,6 +139,7 @@ def pytest_configure(config):
         "bootstrap_config: Provide a custom bootstrap config to the bootstrapping node.\n"
         "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that.\n"
         "no_setup: No setup steps (pushing snap, bootstrapping etc.) are performed on any node for this test.\n"
+        "containerd_cfgdir: The instance containerd config directory, defaults to /etc/containerd."
         "network_type: Specify network type to use for the infrastructure (IPv4, Dualstack or IPv6).\n"
         "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)\n"
         "node_count: Mark a test to specify how many instance nodes need to be created\n"
@@ -192,6 +193,15 @@ def network_type(request) -> Union[str, None]:
     return network_type
 
 
+@pytest.fixture(scope="function")
+def containerd_cfgdir(request) -> str:
+    marker = request.node.get_closest_marker("containerd_cfgdir")
+    if not marker:
+        return "/etc/containerd"
+    cfgdir, *_ = marker.args
+    return cfgdir
+
+
 @pytest.fixture(scope="function")
 def instances(
     h: harness.Harness,
@@ -200,6 +210,7 @@ def instances(
     tmp_path: Path,
     disable_k8s_bootstrapping: bool,
     no_setup: bool,
+    containerd_cfgdir: str,
     bootstrap_config: Union[str, None],
     request,
     network_type: str,
@@ -241,7 +252,7 @@ def instances(
             util.setup_k8s_snap(instance, tmp_path, snap)
 
             if config.USE_LOCAL_MIRROR:
-                registry.apply_configuration(instance)
+                registry.apply_configuration(instance, containerd_cfgdir)
 
     if not disable_k8s_bootstrapping and not no_setup:
         first_node, *_ = instances

tests/integration/tests/test_cleanup.py

@@ -42,6 +42,7 @@ def test_node_cleanup(instances: List[harness.Instance], tmp_path):
 
 @pytest.mark.node_count(2)
 @pytest.mark.disable_k8s_bootstrapping()
+@pytest.mark.containerd_cfgdir("/home/ubuntu/etc/containerd")
 @pytest.mark.tags(tags.NIGHTLY)
 def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]):
     main = instances[0]
@@ -64,15 +65,26 @@ def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]):
 
     boostrap_config = yaml.safe_load(containerd_path_bootstrap_config)
     new_containerd_paths = [
-        os.path.join(boostrap_config["containerd-base-dir"], "k8s-containerd", p)
+        os.path.join(
+            boostrap_config["containerd-base-dir"], "k8s-containerd", p.lstrip("/")
+        )
         for p in CONTAINERD_PATHS
     ]
+
+    # /run/containerd gets created but isn't actually used (requires further
+    # investigation).
+    exp_missing_paths = [
+        "/etc/containerd",
+        "/run/containerd/containerd.sock",
+        "/var/lib/containerd",
+    ]
+
     for instance in instances:
         # Check that the containerd-related folders are not in the default locations.
         process = instance.exec(
-            ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False
+            ["ls", *exp_missing_paths], capture_output=True, text=True, check=False
         )
-        for path in CONTAINERD_PATHS:
+        for path in exp_missing_paths:
             assert (
                 f"cannot access '{path}': No such file or directory" in process.stderr
             )

tests/integration/tests/test_util/registry.py

@@ -2,6 +2,7 @@
 # Copyright 2024 Canonical, Ltd.
 #
 import logging
+import os
 from pathlib import Path
 from string import Template
 from typing import List, Optional
@@ -165,21 +166,22 @@ def ip(self) -> str:
         return self._ip
 
     # Configure the specified instance to use this registry mirror.
-    def apply_configuration(self, instance):
+    def apply_configuration(self, instance, containerd_basedir="/etc/containerd"):
         for mirror in self.mirrors:
             substitutes = {
                 "IP": self.ip,
                 "PORT": mirror.port,
             }
 
-            instance.exec(["mkdir", "-p", f"/etc/containerd/hosts.d/{mirror.name}"])
+            mirror_dir = os.path.join(containerd_basedir, "hosts.d", mirror.name)
+            instance.exec(["mkdir", "-p", mirror_dir])
 
             with open(config.REGISTRY_DIR / "hosts.toml", "r") as registry_template:
                 src = Template(registry_template.read())
                 instance.exec(
                     [
                         "dd",
-                        f"of=/etc/containerd/hosts.d/{mirror.name}/hosts.toml",
+                        f"of={mirror_dir}/hosts.toml",
                     ],
                     input=str.encode(src.substitute(substitutes)),
                 )