From 0a3669c921be5e59a8adce87d6f927455f6342e8 Mon Sep 17 00:00:00 2001 From: niam0522 Date: Wed, 4 Sep 2024 11:50:31 +0530 Subject: [PATCH 01/17] new k8s v1.31.0 --- .../compatibility/internal/kubernetes_images.yaml | 14 ++++++++++++++ .../compatibility/internal/packages.yaml | 6 ++++++ .../compatibility/internal/plugins.yaml | 11 +++++++++++ .../compatibility/internal/thirdparties.yaml | 12 ++++++++++++ .../compatibility/kubernetes_versions.yaml | 9 ++++++++- 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml b/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml index c1584f733..f0ed2fc2b 100644 --- a/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml @@ -41,6 +41,8 @@ kube-apiserver: version: v1.30.1 v1.30.3: version: v1.30.3 + v1.31.0: + version: v1.31.0 kube-controller-manager: v1.26.3: version: v1.26.3 @@ -82,6 +84,8 @@ kube-controller-manager: version: v1.30.1 v1.30.3: version: v1.30.3 + v1.31.0: + version: v1.31.0 kube-scheduler: v1.26.3: version: v1.26.3 @@ -123,6 +127,8 @@ kube-scheduler: version: v1.30.1 v1.30.3: version: v1.30.3 + v1.31.0: + version: v1.31.0 kube-proxy: v1.26.3: version: v1.26.3 @@ -164,6 +170,8 @@ kube-proxy: version: v1.30.1 v1.30.3: version: v1.30.3 + v1.31.0: + version: v1.31.0 pause: v1.26.3: version: '3.9' @@ -205,6 +213,8 @@ pause: version: '3.9' v1.30.3: version: '3.9' + v1.31.0: + version: '3.10' etcd: v1.26.3: version: 3.5.6-0 @@ -246,6 +256,8 @@ etcd: version: 3.5.12-0 v1.30.3: version: 3.5.12-0 + v1.31.0: + version: 3.5.15-0 coredns/coredns: v1.26.3: version: v1.9.3 @@ -287,3 +299,5 @@ coredns/coredns: version: v1.11.1 v1.30.3: version: v1.11.1 + v1.31.0: + version: v1.11.1 diff --git a/kubemarine/resources/configurations/compatibility/internal/packages.yaml b/kubemarine/resources/configurations/compatibility/internal/packages.yaml index e879d4436..beaa8456d 100644 --- a/kubemarine/resources/configurations/compatibility/internal/packages.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/packages.yaml @@ -45,6 +45,8 @@ containerd: version_debian: 1.6.* v1.30.3: version_debian: 1.6.* + v1.31.0: + version_debian: 1.6.* containerdio: v1.26.3: version_rhel: 1.6* @@ -126,6 +128,10 @@ containerdio: version_rhel: 1.6* version_rhel8: 1.6* version_rhel9: 1.6* + v1.31.0: + version_rhel: 1.6* + version_rhel8: 1.6* + version_rhel9: 1.6* haproxy: version_rhel: 1.8* version_rhel8: 1.8* diff --git a/kubemarine/resources/configurations/compatibility/internal/plugins.yaml b/kubemarine/resources/configurations/compatibility/internal/plugins.yaml index 95ba434db..06941477f 100644 --- a/kubemarine/resources/configurations/compatibility/internal/plugins.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/plugins.yaml @@ -45,6 +45,8 @@ calico: version: v3.27.3 v1.30.3: version: v3.28.1 + v1.31.0: + version: v3.28.1 nginx-ingress-controller: v1.26.3: version: v1.4.0 @@ -106,6 +108,9 @@ nginx-ingress-controller: v1.30.3: version: v1.11.1 webhook-version: v1.4.1 + v1.31.0: + version: v1.11.1 + webhook-version: v1.4.1 kubernetes-dashboard: v1.26.3: version: v2.7.0 @@ -167,6 +172,9 @@ kubernetes-dashboard: v1.30.3: version: v2.7.0 metrics-scraper-version: v1.0.8 + v1.31.0: + version: v2.7.0 + metrics-scraper-version: v1.0.8 local-path-provisioner: v1.26.3: version: v0.0.23 @@ -228,3 +236,6 @@ local-path-provisioner: v1.30.3: version: v0.0.27 busybox-version: 1.34.1 + v1.31.0: + version: v0.0.27 + busybox-version: 1.34.1 diff --git a/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml b/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml index eaab64c8f..f7d219eaf 100644 --- a/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml @@ -42,6 +42,8 @@ kubeadm: sha1: 85385f4b78b79bd1cd4d1ad5244422d372d5f758 v1.30.3: sha1: f840e75f5dc1001ebdd7e286c0e87e1090df011b + v1.31.0: + sha1: 3ac8ce4f0f17a21e5531213025b0090c42310d08 kubelet: v1.26.3: sha1: 5fe320fedaabb91d3770da19135412b7454bb28b @@ -83,6 +85,8 @@ kubelet: sha1: c62da6ab918b8e56d7c9b77e642ffc73ffdbffac v1.30.3: sha1: fbae53efc43ec715a45b05415294ab991ea087a2 + v1.31.0: + sha1: b5cb543e21faaa600a5aac54a2316a9fd5e5b8c6 kubectl: v1.26.3: sha1: 56916d87c3caef05489db932fd9e48d32ebdf634 @@ -124,6 +128,8 @@ kubectl: sha1: bced94239f1dbdb04d3a661a067bf9587865b6e8 v1.30.3: sha1: 097d6b02fabb284418a9c95ea81fa86fc3c85bb7 + v1.31.0: + sha1: 34214a4983223aa4e8d29063990a265b220bef5b calicoctl: # calicoctl version is duplicated from kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml # It also corresponds to the plugin version in kubemarine/resources/configurations/compatibility/internal/plugins.yaml @@ -187,6 +193,9 @@ calicoctl: v1.30.3: version: v3.28.1 sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 + v1.31.0: + version: v3.28.1 + sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 crictl: # crictl version is duplicated from kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml # for backward compatibility with clusters in a private environment. @@ -250,3 +259,6 @@ crictl: v1.30.3: version: v1.30.0 sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b + v1.31.0: + version: v1.30.0 + sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b diff --git a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml index 69399f00b..82ab730c5 100644 --- a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml +++ b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml @@ -10,6 +10,8 @@ kubernetes_versions: supported: true v1.30: supported: true + v1.31: + supported: true compatibility_map: # This section should be changed manually. v1.26.3: @@ -132,7 +134,12 @@ compatibility_map: kubernetes-dashboard: v2.7.0 local-path-provisioner: v0.0.27 crictl: v1.30.0 - + v1.31.0: + calico: v3.28.1 + nginx-ingress-controller: v1.11.1 + kubernetes-dashboard: v2.7.0 + local-path-provisioner: v0.0.27 + crictl: v1.30.0 # After any change, please run scripts/thirdparties/sync.py # The following optional keys are supported in addition to the 5 required software keys: From a782d5e24b0861b04c46fc3af79b030773f71e3c Mon Sep 17 00:00:00 2001 From: niam0522 Date: Wed, 11 Sep 2024 11:45:17 +0530 Subject: [PATCH 02/17] Convert list of dictionaries --- kubemarine/procedures/check_paas.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kubemarine/procedures/check_paas.py b/kubemarine/procedures/check_paas.py index ca1372061..c50eaedb9 100755 --- a/kubemarine/procedures/check_paas.py +++ b/kubemarine/procedures/check_paas.py @@ -1511,7 +1511,11 @@ def kubernetes_admission_status(cluster: KubernetesCluster) -> None: kubeadm_config = components.KubeadmConfig(cluster) cluster_config = kubeadm_config.load('kubeadm-config', first_control_plane) + # Check if extraArgs is a list of dictionaries or a dictionary apiserver_actual_args = cluster_config["apiServer"]["extraArgs"] + if isinstance(apiserver_actual_args, list): + # Convert list of dictionaries to a single dictionary + apiserver_actual_args = {arg['name']: arg['value'] for arg in apiserver_actual_args} actual_state = "disabled" if "admission-control-config-file" in apiserver_actual_args and ( From c81866480bb6f5c24da493d21e447a1e45a7cf78 Mon Sep 17 00:00:00 2001 From: niam0522 Date: Wed, 11 Sep 2024 11:49:47 +0530 Subject: [PATCH 03/17] revert --- kubemarine/procedures/check_paas.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/kubemarine/procedures/check_paas.py b/kubemarine/procedures/check_paas.py index c50eaedb9..ca1372061 100755 --- a/kubemarine/procedures/check_paas.py +++ b/kubemarine/procedures/check_paas.py @@ -1511,11 +1511,7 @@ def kubernetes_admission_status(cluster: KubernetesCluster) -> None: kubeadm_config = components.KubeadmConfig(cluster) cluster_config = kubeadm_config.load('kubeadm-config', first_control_plane) - # Check if extraArgs is a list of dictionaries or a dictionary apiserver_actual_args = cluster_config["apiServer"]["extraArgs"] - if isinstance(apiserver_actual_args, list): - # Convert list of dictionaries to a single dictionary - apiserver_actual_args = {arg['name']: arg['value'] for arg in apiserver_actual_args} actual_state = "disabled" if "admission-control-config-file" in apiserver_actual_args and ( From 5d33ac54992fb4f52895ed4eadd5adc1b6c9bb2c Mon Sep 17 00:00:00 2001 From: niam0522 Date: Wed, 11 Sep 2024 12:10:35 +0530 Subject: [PATCH 04/17] Convert list of dictionaries_1 --- kubemarine/procedures/check_paas.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kubemarine/procedures/check_paas.py b/kubemarine/procedures/check_paas.py index ca1372061..c50eaedb9 100755 --- a/kubemarine/procedures/check_paas.py +++ b/kubemarine/procedures/check_paas.py @@ -1511,7 +1511,11 @@ def kubernetes_admission_status(cluster: KubernetesCluster) -> None: kubeadm_config = components.KubeadmConfig(cluster) cluster_config = kubeadm_config.load('kubeadm-config', first_control_plane) + # Check if extraArgs is a list of dictionaries or a dictionary apiserver_actual_args = cluster_config["apiServer"]["extraArgs"] + if isinstance(apiserver_actual_args, list): + # Convert list of dictionaries to a single dictionary + apiserver_actual_args = {arg['name']: arg['value'] for arg in apiserver_actual_args} actual_state = "disabled" if "admission-control-config-file" in apiserver_actual_args and ( From 2ddeae825456a4402043f7f8b32c9c46985cf728 Mon Sep 17 00:00:00 2001 From: niam0522 Date: Wed, 11 Sep 2024 12:36:30 +0530 Subject: [PATCH 05/17] Installation.md --- documentation/Installation.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/documentation/Installation.md b/documentation/Installation.md index 911a30e2d..9a3138c00 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -6142,3 +6142,33 @@ The tables below shows the correspondence of versions that are supported and is | | kubernetesui/dashboard | v2.7.0 | v2.7.0 | v2.7.0 | v2.7.0 | v2.7.0 | v2.7.0 | v2.7.0 | Required only if Kubernetes Dashboard plugin is set to be installed. | | | kubernetesui/metrics-scraper | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | Required only if Kubernetes Dashboard plugin is set to be installed. | | | rancher/local-path-provisioner | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | Required only if local-path provisioner plugin is set to be installed. | + +## Default Dependent Components Versions for Kubernetes Versions v1.31.0 +| Type | Name | Versions | | | | | | | Note | +|----------|----------------------------------------------------------------|------------------|------------------------------|--------------|--------------|-------------------|-----------|-----------|------------------------------------------------------------------------------------------------------------| +| | | CentOS RHEL 7.5+ | CentOS RHEL Oracle Linux 8.4 | Ubuntu 20.04 | Ubuntu 22.04 | Oracle Linux 7.5+ | RHEL 8.6+ | RockyLinux 8.6+ | | +| binaries | kubeadm | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | SHA1: 3ac8ce4f0f17a21e5531213025b0090c42310d08 | +| | kubelet | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | SHA1: b5cb543e21faaa600a5aac54a2316a9fd5e5b8c6 | +| | kubectl | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | SHA1: 34214a4983223aa4e8d29063990a265b220bef5b | +| | calicoctl | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | SHA1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 Required only if calico is installed. | +| | crictl | v1.30.0 | v1.30.0 | v1.30.0 | v1.30.0 | v1.30.0 | v1.30.0 | v1.30.0 | SHA1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b | +| rpms | containerd.io | 1.6.* | 1.6.* | 1.6.* | 1.6.* | 1.6.* | 1.6.* | 1.6.* | | +| | haproxy/rh-haproxy | 1.8 | 1.8 | 2.* | 2.* | 1.8 | 1.8 | 1.8 | Required only if balancers are presented in the deployment scheme. | +| | keepalived | 1.3 | 2.1 | 2.* | 2.* | 1.3 | 2.1 | 2.1 | Required only if VRRP is presented in the deployment scheme. | +| images | registry.k8s.io/kube-apiserver | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | | +| | registry.k8s.io/kube-controller-manager | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | | +| | registry.k8s.io/kube-proxy | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | | +| | registry.k8s.io/kube-scheduler | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | | +| | registry.k8s.io/coredns | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | | +| | registry.k8s.io/pause | 3.10 | 3.10 | 3.10 | 3.10 | 3.10 | 3.10 | 3.10 | | +| | registry.k8s.io/etcd | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | | +| | calico/typha | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | Required only if Typha is enabled in Calico config. | +| | calico/cni | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | | +| | calico/node | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | | +| | calico/kube-controllers | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | | +| | calico/apiserver | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | Required only if API server is enabled in Calico config. | +| | registry.k8s.io/ingress-nginx/controller | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | | +| | registry.k8s.io/kube-webhook-certgen | v1.4.1 | v1.4.1 | v1.4.1 | v1.4.1 | v1.4.1 | v1.4.1 | v1.4.1 | | +| | kubernetesui/dashboard | v2.7.0 | v2.7.0 | v2.7.0 | v2.7.0 | v2.7.0 | v2.7.0 | v2.7.0 | Required only if Kubernetes Dashboard plugin is set to be installed. | +| | kubernetesui/metrics-scraper | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | Required only if Kubernetes Dashboard plugin is set to be installed. | +| | rancher/local-path-provisioner | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | Required only if local-path provisioner plugin is set to be installed. | From 2b4f65ceabaf1d7fedb71baa4e7b8e152c71f68b Mon Sep 17 00:00:00 2001 From: niam0522 Date: Wed, 11 Sep 2024 12:52:26 +0530 Subject: [PATCH 06/17] From PR 644 --- kubemarine/kubernetes/components.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kubemarine/kubernetes/components.py b/kubemarine/kubernetes/components.py index e4e24d3d2..1f33342d1 100644 --- a/kubemarine/kubernetes/components.py +++ b/kubemarine/kubernetes/components.py @@ -208,6 +208,10 @@ def kube_proxy_overwrites_higher_system_values(cluster: KubernetesCluster) -> bo return kubernetes_minor_release_at_least(cluster.inventory, "v1.29") +def kubeadm_writes_resolv_conf_per_node(cluster: KubernetesCluster) -> bool: + return kubernetes_minor_release_at_least(cluster.inventory, "v1.31") + + def kubernetes_minor_release_at_least(inventory: dict, minor_version: str) -> bool: kubernetes_version = inventory["services"]["kubeadm"]["kubernetesVersion"] return utils.version_key(kubernetes_version)[0:2] >= utils.minor_version_key(minor_version) @@ -489,8 +493,11 @@ def patch_kubelet_configmap(control_plane: AbstractGroup[RunResult]) -> None: :param control_plane: control plane to operate on """ - # Make sure to check kubeadm_kubelet in the inventory. cluster: KubernetesCluster = control_plane.cluster + if kubeadm_writes_resolv_conf_per_node(cluster): + return + + # Make sure to check kubeadm_kubelet in the inventory. kubeadm_config = KubeadmConfig(cluster) if 'resolvConf' in kubeadm_config.maps['kubelet-config']: return From ca0a9e304392aa717be0a0f3a1dde2dd17444b02 Mon Sep 17 00:00:00 2001 From: Imadzuma Date: Tue, 10 Sep 2024 09:16:47 +0300 Subject: [PATCH 07/17] return to unspecified patch version in base images --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0d669f621..e246482cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ COPY --from=go-build /opt/ipip_check.gz /opt/kubemarine/kubemarine/resources/scr WORKDIR /opt/kubemarine/ RUN apt update && \ + pip3 install --no-cache-dir setuptools wheel && \ pip3 install --no-cache-dir build && \ python3 -m build -n && \ # In any if branch delete source code, but preserve specific directories for different service aims From 5ca2a3301b691f2e514929b7985294f00c02573b Mon Sep 17 00:00:00 2001 From: ilia1243 <8808144+ilia1243@users.noreply.github.com> Date: Thu, 18 Apr 2024 14:20:41 +0300 Subject: [PATCH 08/17] Plan to remove W/A to patch kubelet configmap From 6fa3b8b7aae27a212e5316fc84287e1718956560 Mon Sep 17 00:00:00 2001 From: niam0522 Date: Wed, 11 Sep 2024 19:19:31 +0530 Subject: [PATCH 09/17] Revert "From PR 644" This reverts commit 2b4f65ceabaf1d7fedb71baa4e7b8e152c71f68b. --- kubemarine/kubernetes/components.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/kubemarine/kubernetes/components.py b/kubemarine/kubernetes/components.py index 1f33342d1..e4e24d3d2 100644 --- a/kubemarine/kubernetes/components.py +++ b/kubemarine/kubernetes/components.py @@ -208,10 +208,6 @@ def kube_proxy_overwrites_higher_system_values(cluster: KubernetesCluster) -> bo return kubernetes_minor_release_at_least(cluster.inventory, "v1.29") -def kubeadm_writes_resolv_conf_per_node(cluster: KubernetesCluster) -> bool: - return kubernetes_minor_release_at_least(cluster.inventory, "v1.31") - - def kubernetes_minor_release_at_least(inventory: dict, minor_version: str) -> bool: kubernetes_version = inventory["services"]["kubeadm"]["kubernetesVersion"] return utils.version_key(kubernetes_version)[0:2] >= utils.minor_version_key(minor_version) @@ -493,11 +489,8 @@ def patch_kubelet_configmap(control_plane: AbstractGroup[RunResult]) -> None: :param control_plane: control plane to operate on """ - cluster: KubernetesCluster = control_plane.cluster - if kubeadm_writes_resolv_conf_per_node(cluster): - return - # Make sure to check kubeadm_kubelet in the inventory. + cluster: KubernetesCluster = control_plane.cluster kubeadm_config = KubeadmConfig(cluster) if 'resolvConf' in kubeadm_config.maps['kubelet-config']: return From b60df895dc61747f1749aa724a77002e5abdea49 Mon Sep 17 00:00:00 2001 From: sergey kryazhev Date: Wed, 18 Sep 2024 10:04:09 +0300 Subject: [PATCH 10/17] applied containerd restart fix from !694 --- kubemarine/cri/containerd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kubemarine/cri/containerd.py b/kubemarine/cri/containerd.py index 15f203393..252683a95 100755 --- a/kubemarine/cri/containerd.py +++ b/kubemarine/cri/containerd.py @@ -299,10 +299,12 @@ def configure_containerd(group: NodeGroup) -> RunnersGroupResult: backup=True, sudo=True, mkdir=True) log.debug("Restarting Containerd on %s node..." % node.get_node_name()) + # to restart and wait untill containerd is up&running node.sudo( f"chmod 600 {os_specific_associations['config_location']} && " f"sudo systemctl restart {os_specific_associations['service_name']} && " - f"systemctl status {os_specific_associations['service_name']}", callback=collector) + f"systemctl status {os_specific_associations['service_name']} && " + f"timeout 10 sh -c 'until sudo ctr version 2>&1; do sleep 1; done' ", callback=collector) return collector.result From ed48e630df7074786e1fe83331977f1c6d90a269 Mon Sep 17 00:00:00 2001 From: niam0522 Date: Mon, 30 Sep 2024 13:33:04 +0530 Subject: [PATCH 11/17] cherry-pick --- kubemarine/procedures/upgrade.py | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/kubemarine/procedures/upgrade.py b/kubemarine/procedures/upgrade.py index 5bb6ed472..a55fe8026 100755 --- a/kubemarine/procedures/upgrade.py +++ b/kubemarine/procedures/upgrade.py @@ -17,6 +17,7 @@ from collections import OrderedDict from typing import List, Callable, Dict import uuid +import yaml from kubemarine import kubernetes, plugins, admission, jinja from kubemarine.core import flow, log, resources as res from kubemarine.core import utils @@ -47,6 +48,75 @@ def prepull_images(cluster: KubernetesCluster) -> None: upgrade_group.call(kubernetes.images_grouped_prepull) +def enable_control_plane_kubelet_local_mode(cluster: KubernetesCluster) -> None: + """ + Enable ControlPlaneKubeletLocalMode feature gate to ensure the kubelet communicates + with the local API server on control-plane nodes, but only for Kubernetes v1.31.x and above. + """ + # Check if the target version is v1.31.x or above + target_version = kubernetes.get_procedure_upgrade_version(cluster) + if not target_version.startswith("v1.31"): + cluster.log.debug(f"Skipping ControlPlaneKubeletLocalMode feature gate. Target version is {target_version}.") + return + + cluster.log.debug("Enabling ControlPlaneKubeletLocalMode feature gate in kubeadm config for v1.31.x or above.") + + # Fetch the existing kubeadm-config ConfigMap + control_plane_group = cluster.nodes['control-plane'] + config_map_result = control_plane_group.sudo("kubectl get configmap kubeadm-config -n kube-system -o yaml") + + # Extract the stdout from the RunnersGroupResult object + config_map_yaml = config_map_result.get_simple_out() + + # Log the output to ensure the ConfigMap is fetched correctly + cluster.log.debug(f"Fetched kubeadm-config ConfigMap: {config_map_yaml}") + + # Convert the yaml string into a Python dictionary + config_map = yaml.safe_load(config_map_yaml) + + # Extract the ClusterConfiguration field + cluster_configuration = config_map['data'].get('ClusterConfiguration', "") + + # Parse the ClusterConfiguration to modify it + cluster_config_dict = yaml.safe_load(cluster_configuration) + + # Ensure apiServer.extraArgs exists and is a list + if 'apiServer' not in cluster_config_dict: + cluster_config_dict['apiServer'] = {} + + if 'extraArgs' not in cluster_config_dict['apiServer']: + cluster_config_dict['apiServer']['extraArgs'] = [] + + # Check if feature-gates already exists, and update or add it + feature_gates_exists = False + for arg in cluster_config_dict['apiServer']['extraArgs']: + if arg.get('name') == 'feature-gates': + arg['value'] = 'ControlPlaneKubeletLocalMode=true' + feature_gates_exists = True + break + + if not feature_gates_exists: + # Add the feature-gates setting + cluster_config_dict['apiServer']['extraArgs'].append({ + 'name': 'feature-gates', + 'value': 'ControlPlaneKubeletLocalMode=true' + }) + + # Convert the modified ClusterConfiguration back to a string + updated_cluster_config_yaml = yaml.dump(cluster_config_dict) + + # Update the ConfigMap with the new ClusterConfiguration + config_map['data']['ClusterConfiguration'] = updated_cluster_config_yaml + + # Convert the entire ConfigMap back to yaml for patching + updated_config_map_yaml = yaml.dump(config_map) + + # Apply the updated ConfigMap + control_plane_group.sudo(f"kubectl apply -f - < None: initial_kubernetes_version = kubernetes.get_kubernetes_version(cluster.previous_inventory) From 093ff25c16808f789575c6056ab20205dadb2d37 Mon Sep 17 00:00:00 2001 From: Imadzuma Date: Wed, 9 Oct 2024 08:39:49 +0300 Subject: [PATCH 12/17] support ControlPlaneKubeletLocalMode=True to fix ubgrade to 1.31 --- documentation/Installation.md | 39 ++++----- kubemarine/core/resources.py | 1 + kubemarine/kubernetes/__init__.py | 14 +++- kubemarine/kubernetes/components.py | 4 + kubemarine/procedures/upgrade.py | 83 ++++--------------- .../schemas/definitions/services/kubeadm.json | 14 +++- test/unit/test_kubernetes_components.py | 34 ++++++++ 7 files changed, 99 insertions(+), 90 deletions(-) diff --git a/documentation/Installation.md b/documentation/Installation.md index 9a3138c00..29a24f46e 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -1092,25 +1092,26 @@ In the `services.kubeadm` section, you can override the original settings for ku For more information about these settings, refer to the official Kubernetes documentation at [https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/#config-file](https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/#config-file). By default, the installer uses the following parameters: -| Parameter | Default Value | Description | -|-------------------------------------------------------|----------------------------------------------------------|--------------------------------------------------------------------------------------------------| -|kubernetesVersion | `v1.28.12` | | -|controlPlaneEndpoint | `{{ cluster_name }}:6443` | | -|networking.podSubnet | `10.128.0.0/14` for IPv4 or `fd02::/48` for IPv6 | | -|networking.serviceSubnet | `172.30.0.0/16` for IPv4 or `fd03::/112` for IPv6 | | -|apiServer.certSANs | List with all nodes internal IPs, external IPs and names | Custom SANs are only appended to, but do not override the default list | -|apiServer.extraArgs.enable-admission-plugins | `NodeRestriction` | | -|apiServer.extraArgs.feature-gates | | `PodSecurity=true` is added for Kubernetes < v1.28 if [RBAC pss](#rbac-pss) is enabled | -|apiServer.extraArgs.admission-control-config-file | `/etc/kubernetes/pki/admission.yaml` | Provided default value **overrides** custom value if [RBAC pss](#rbac-pss) is enabled. | -|apiServer.extraArgs.profiling | `false` | | -|apiServer.extraArgs.audit-log-path | `/var/log/kubernetes/audit/audit.log` | | -|apiServer.extraArgs.audit-policy-file | `/etc/kubernetes/audit-policy.yaml` | | -|apiServer.extraArgs.audit-log-maxage | `30` | | -|apiServer.extraArgs.audit-log-maxbackup | `10` | | -|apiServer.extraArgs.audit-log-maxsize | `100` | | -|scheduler.extraArgs.profiling | `false` | | -|controllerManager.extraArgs.profiling | `false` | | -|controllerManager.extraArgs.terminated-pod-gc-threshold| `1000` | | +| Parameter | Default Value | Description | +|---------------------------------------------------------|----------------------------------------------------------|----------------------------------------------------------------------------------------| +| kubernetesVersion | `v1.28.12` | | +| controlPlaneEndpoint | `{{ cluster_name }}:6443` | | +| networking.podSubnet | `10.128.0.0/14` for IPv4 or `fd02::/48` for IPv6 | | +| networking.serviceSubnet | `172.30.0.0/16` for IPv4 or `fd03::/112` for IPv6 | | +| apiServer.certSANs | List with all nodes internal IPs, external IPs and names | Custom SANs are only appended to, but do not override the default list | +| apiServer.extraArgs.enable-admission-plugins | `NodeRestriction` | | +| apiServer.extraArgs.feature-gates | | `PodSecurity=true` is added for Kubernetes < v1.28 if [RBAC pss](#rbac-pss) is enabled | +| apiServer.extraArgs.admission-control-config-file | `/etc/kubernetes/pki/admission.yaml` | Provided default value **overrides** custom value if [RBAC pss](#rbac-pss) is enabled. | +| apiServer.extraArgs.profiling | `false` | | +| apiServer.extraArgs.audit-log-path | `/var/log/kubernetes/audit/audit.log` | | +| apiServer.extraArgs.audit-policy-file | `/etc/kubernetes/audit-policy.yaml` | | +| apiServer.extraArgs.audit-log-maxage | `30` | | +| apiServer.extraArgs.audit-log-maxbackup | `10` | | +| apiServer.extraArgs.audit-log-maxsize | `100` | | +| scheduler.extraArgs.profiling | `false` | | +| controllerManager.extraArgs.profiling | `false` | | +| controllerManager.extraArgs.terminated-pod-gc-threshold | `1000` | | +| featureGates.ControlPlaneKubeletLocalMode | `true` | Provided for Kubernetes version >= 1.31 | The following is an example of kubeadm defaults override: diff --git a/kubemarine/core/resources.py b/kubemarine/core/resources.py index 2f611035a..d5c2471f5 100644 --- a/kubemarine/core/resources.py +++ b/kubemarine/core/resources.py @@ -462,6 +462,7 @@ def enrichment_functions(self) -> List[c.EnrichmentFunction]: # * kubemarine.sysctl.enrich_inventory kubemarine.kubernetes.enrich_inventory, kubemarine.admission.enrich_inventory, + kubemarine.kubernetes.enrich_control_plane_kubelet_local_mode, # Depends on kubemarine.core.defaults.apply_defaults kubemarine.kubernetes_accounts.enrich_inventory, # Depends on kubemarine.kubernetes.enrich_inventory diff --git a/kubemarine/kubernetes/__init__.py b/kubemarine/kubernetes/__init__.py index 4604823be..298deeb81 100644 --- a/kubemarine/kubernetes/__init__.py +++ b/kubemarine/kubernetes/__init__.py @@ -23,7 +23,7 @@ from jinja2 import Template from ordered_set import OrderedSet -from kubemarine import system, admission, etcd, packages, jinja, sysctl +from kubemarine import system, admission, etcd, packages, jinja, sysctl, kubernetes from kubemarine.core import utils, static, summary, log, errors from kubemarine.core.cluster import KubernetesCluster, EnrichmentStage, enrichment from kubemarine.core.executor import Token @@ -188,6 +188,18 @@ def enrich_inventory(cluster: KubernetesCluster) -> None: enrich_kube_proxy(cluster) +@enrichment(EnrichmentStage.FULL) +def enrich_control_plane_kubelet_local_mode(cluster: KubernetesCluster) -> None: + inventory = cluster.inventory + + kubeadm = inventory["services"]["kubeadm"] + if kubernetes.components.control_plane_kubelet_local_mode(cluster): + feature_gates = kubeadm.get("featureGates", {}) + if 'ControlPlaneKubeletLocalMode' not in feature_gates: + feature_gates['ControlPlaneKubeletLocalMode'] = True + kubeadm["featureGates"] = feature_gates + + def enrich_kube_proxy(cluster: KubernetesCluster) -> None: inventory = cluster.inventory diff --git a/kubemarine/kubernetes/components.py b/kubemarine/kubernetes/components.py index e4e24d3d2..388a8cd25 100644 --- a/kubemarine/kubernetes/components.py +++ b/kubemarine/kubernetes/components.py @@ -208,6 +208,10 @@ def kube_proxy_overwrites_higher_system_values(cluster: KubernetesCluster) -> bo return kubernetes_minor_release_at_least(cluster.inventory, "v1.29") +def control_plane_kubelet_local_mode(cluster: KubernetesCluster) -> bool: + return kubernetes_minor_release_at_least(cluster.inventory, "v1.31") + + def kubernetes_minor_release_at_least(inventory: dict, minor_version: str) -> bool: kubernetes_version = inventory["services"]["kubeadm"]["kubernetesVersion"] return utils.version_key(kubernetes_version)[0:2] >= utils.minor_version_key(minor_version) diff --git a/kubemarine/procedures/upgrade.py b/kubemarine/procedures/upgrade.py index a55fe8026..e0c72a33f 100755 --- a/kubemarine/procedures/upgrade.py +++ b/kubemarine/procedures/upgrade.py @@ -48,75 +48,6 @@ def prepull_images(cluster: KubernetesCluster) -> None: upgrade_group.call(kubernetes.images_grouped_prepull) -def enable_control_plane_kubelet_local_mode(cluster: KubernetesCluster) -> None: - """ - Enable ControlPlaneKubeletLocalMode feature gate to ensure the kubelet communicates - with the local API server on control-plane nodes, but only for Kubernetes v1.31.x and above. - """ - # Check if the target version is v1.31.x or above - target_version = kubernetes.get_procedure_upgrade_version(cluster) - if not target_version.startswith("v1.31"): - cluster.log.debug(f"Skipping ControlPlaneKubeletLocalMode feature gate. Target version is {target_version}.") - return - - cluster.log.debug("Enabling ControlPlaneKubeletLocalMode feature gate in kubeadm config for v1.31.x or above.") - - # Fetch the existing kubeadm-config ConfigMap - control_plane_group = cluster.nodes['control-plane'] - config_map_result = control_plane_group.sudo("kubectl get configmap kubeadm-config -n kube-system -o yaml") - - # Extract the stdout from the RunnersGroupResult object - config_map_yaml = config_map_result.get_simple_out() - - # Log the output to ensure the ConfigMap is fetched correctly - cluster.log.debug(f"Fetched kubeadm-config ConfigMap: {config_map_yaml}") - - # Convert the yaml string into a Python dictionary - config_map = yaml.safe_load(config_map_yaml) - - # Extract the ClusterConfiguration field - cluster_configuration = config_map['data'].get('ClusterConfiguration', "") - - # Parse the ClusterConfiguration to modify it - cluster_config_dict = yaml.safe_load(cluster_configuration) - - # Ensure apiServer.extraArgs exists and is a list - if 'apiServer' not in cluster_config_dict: - cluster_config_dict['apiServer'] = {} - - if 'extraArgs' not in cluster_config_dict['apiServer']: - cluster_config_dict['apiServer']['extraArgs'] = [] - - # Check if feature-gates already exists, and update or add it - feature_gates_exists = False - for arg in cluster_config_dict['apiServer']['extraArgs']: - if arg.get('name') == 'feature-gates': - arg['value'] = 'ControlPlaneKubeletLocalMode=true' - feature_gates_exists = True - break - - if not feature_gates_exists: - # Add the feature-gates setting - cluster_config_dict['apiServer']['extraArgs'].append({ - 'name': 'feature-gates', - 'value': 'ControlPlaneKubeletLocalMode=true' - }) - - # Convert the modified ClusterConfiguration back to a string - updated_cluster_config_yaml = yaml.dump(cluster_config_dict) - - # Update the ConfigMap with the new ClusterConfiguration - config_map['data']['ClusterConfiguration'] = updated_cluster_config_yaml - - # Convert the entire ConfigMap back to yaml for patching - updated_config_map_yaml = yaml.dump(config_map) - - # Apply the updated ConfigMap - control_plane_group.sudo(f"kubectl apply -f - < None: initial_kubernetes_version = kubernetes.get_kubernetes_version(cluster.previous_inventory) @@ -163,6 +94,20 @@ def edit_kube_proxy_conntrack_min(kube_proxy_cm: dict) -> dict: preconfigure_components.append('kube-proxy') preconfigure_functions['kube-proxy'] = edit_kube_proxy_conntrack_min + if kubernetes.components.control_plane_kubelet_local_mode(cluster): + + # featureGates field is changed for kubernetes >= 1.31 + # See kubernetes.enrich_control_plane_kubelet_local_mode() for details + def apply_kubelet_local_mode(cluster_config: dict) -> dict: + feature_gates = cluster.inventory["services"]["kubeadm"].get("featureGates") + if feature_gates is not None: + cluster_config["featureGates"] = feature_gates + + return cluster_config + + preconfigure_components.append('kube-apiserver') + preconfigure_functions['kubeadm-config'] = apply_kubelet_local_mode + if preconfigure_components: upgrade_group.call(kubernetes.components.reconfigure_components, components=preconfigure_components, edit_functions=preconfigure_functions) diff --git a/kubemarine/resources/schemas/definitions/services/kubeadm.json b/kubemarine/resources/schemas/definitions/services/kubeadm.json index f9ca992ef..8d3a5ca59 100644 --- a/kubemarine/resources/schemas/definitions/services/kubeadm.json +++ b/kubemarine/resources/schemas/definitions/services/kubeadm.json @@ -48,7 +48,10 @@ "$ref": "#/definitions/Etcd" }, "apiVersion": {"type": ["string"], "default": "kubeadm.k8s.io/v1beta2"}, - "kind": {"enum": ["ClusterConfiguration"], "default": "ClusterConfiguration"} + "kind": {"enum": ["ClusterConfiguration"], "default": "ClusterConfiguration"}, + "featureGates": { + "$ref": "#/definitions/FeatureGates" + } }, "definitions": { "ApiServer": { @@ -172,6 +175,15 @@ }, "required": ["name", "hostPath", "mountPath"], "additionalProperties": false + }, + "FeatureGates": { + "type": "object", + "properties": { + "ControlPlaneKubeletLocalMode": { + "type": "boolean" + } + }, + "additionalProperties": true } } } diff --git a/test/unit/test_kubernetes_components.py b/test/unit/test_kubernetes_components.py index 9a1bdb2f2..476d289d9 100644 --- a/test/unit/test_kubernetes_components.py +++ b/test/unit/test_kubernetes_components.py @@ -123,6 +123,40 @@ def _test_merge_with_inventory(self, config: dict): self.assertEqual(True, config.get('nested', {}).get('untouched')) self.assertEqual([2], config.get('array')) + def test_kubelet_local_mode_enrichment(self): + # No featureGates for kubernetes <= 1.30 + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.30.3' + cluster = demo.new_cluster(inventory) + kubeadm = cluster.inventory['services']['kubeadm'] + self.assertIsNone(kubeadm.get('featureGates')) + + # Enriched featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+ + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.0' + cluster = demo.new_cluster(inventory) + kubeadm = cluster.inventory['services']['kubeadm'] + self.assertIsNotNone(kubeadm.get('featureGates')) + self.assertTrue(kubeadm['featureGates'].get('ControlPlaneKubeletLocalMode')) + + # Enriched featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+ with not empty featureGates + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.0' + inventory['services'].setdefault('kubeadm', {}).setdefault('featureGates', {})['foo'] = 'bar' + cluster = demo.new_cluster(inventory) + kubeadm = cluster.inventory['services']['kubeadm'] + self.assertIsNotNone(kubeadm.get('featureGates')) + self.assertEqual('bar', kubeadm['featureGates'].get('foo')) + self.assertTrue(kubeadm['featureGates'].get('ControlPlaneKubeletLocalMode')) + + # Do not change featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+ if value is overridden + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.0' + inventory['services'].setdefault('kubeadm', {}).setdefault('featureGates', {})['ControlPlaneKubeletLocalMode'] = False + cluster = demo.new_cluster(inventory) + kubeadm = cluster.inventory['services']['kubeadm'] + self.assertIsNotNone(kubeadm.get('featureGates')) + self.assertFalse(kubeadm['featureGates'].get('ControlPlaneKubeletLocalMode')) class WaitForPodsTest(unittest.TestCase): def setUp(self): From 7e50878164f73a950bf11a31a5f5b9ea047f3617 Mon Sep 17 00:00:00 2001 From: Imadzuma Date: Wed, 9 Oct 2024 08:49:51 +0300 Subject: [PATCH 13/17] linter fixes --- kubemarine/kubernetes/__init__.py | 4 ++-- kubemarine/procedures/upgrade.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/kubemarine/kubernetes/__init__.py b/kubemarine/kubernetes/__init__.py index 298deeb81..3d5c7d4e9 100644 --- a/kubemarine/kubernetes/__init__.py +++ b/kubemarine/kubernetes/__init__.py @@ -23,7 +23,7 @@ from jinja2 import Template from ordered_set import OrderedSet -from kubemarine import system, admission, etcd, packages, jinja, sysctl, kubernetes +from kubemarine import system, admission, etcd, packages, jinja, sysctl from kubemarine.core import utils, static, summary, log, errors from kubemarine.core.cluster import KubernetesCluster, EnrichmentStage, enrichment from kubemarine.core.executor import Token @@ -193,7 +193,7 @@ def enrich_control_plane_kubelet_local_mode(cluster: KubernetesCluster) -> None: inventory = cluster.inventory kubeadm = inventory["services"]["kubeadm"] - if kubernetes.components.control_plane_kubelet_local_mode(cluster): + if components.control_plane_kubelet_local_mode(cluster): feature_gates = kubeadm.get("featureGates", {}) if 'ControlPlaneKubeletLocalMode' not in feature_gates: feature_gates['ControlPlaneKubeletLocalMode'] = True diff --git a/kubemarine/procedures/upgrade.py b/kubemarine/procedures/upgrade.py index e0c72a33f..e1265fd05 100755 --- a/kubemarine/procedures/upgrade.py +++ b/kubemarine/procedures/upgrade.py @@ -17,7 +17,6 @@ from collections import OrderedDict from typing import List, Callable, Dict import uuid -import yaml from kubemarine import kubernetes, plugins, admission, jinja from kubemarine.core import flow, log, resources as res from kubemarine.core import utils From d9cab2bdc7e9f3031cf925c72390ead9811b78da Mon Sep 17 00:00:00 2001 From: niam0522 Date: Tue, 15 Oct 2024 13:15:06 +0530 Subject: [PATCH 14/17] v1.31.1_&_dl.k8s.io_instead_of_storage.googleapis --- documentation/Installation.md | 6 +++--- documentation/Maintenance.md | 6 +++--- examples/procedure.yaml/full-restore.yaml | 6 +++--- .../compatibility/internal/kubernetes_images.yaml | 14 ++++++++++++++ .../compatibility/internal/packages.yaml | 6 ++++++ .../compatibility/internal/plugins.yaml | 11 +++++++++++ .../compatibility/internal/thirdparties.yaml | 12 ++++++++++++ .../compatibility/kubernetes_versions.yaml | 6 ++++++ kubemarine/resources/configurations/globals.yaml | 6 +++--- test/unit/core/test_env.py | 4 ++-- 10 files changed, 63 insertions(+), 14 deletions(-) diff --git a/documentation/Installation.md b/documentation/Installation.md index 29a24f46e..cd29c5749 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -2094,11 +2094,11 @@ By default, the installer installs the following thirdparties with the following services: thirdparties: /usr/bin/kubeadm: - source: 'https://storage.googleapis.com/kubernetes-release/release/{{k8s-version}}/bin/linux/amd64/kubeadm' + source: 'https://dl.k8s.io/{{k8s-version}}/bin/linux/amd64/kubeadm' /usr/bin/kubelet: - source: 'https://storage.googleapis.com/kubernetes-release/release/{{k8s-version}}/bin/linux/amd64/kubelet' + source: 'https://dl.k8s.io/{{k8s-version}}/bin/linux/amd64/kubelet' /usr/bin/kubectl: - source: 'https://storage.googleapis.com/kubernetes-release/release/{{k8s-version}}/bin/linux/amd64/kubectl' + source: 'https://dl.k8s.io/{{k8s-version}}/bin/linux/amd64/kubectl' group: control-plane /usr/bin/calicoctl: source: 'https://github.com/projectcalico/calico/releases/download/{{calico-version}}/calicoctl-linux-amd64' diff --git a/documentation/Maintenance.md b/documentation/Maintenance.md index d2b8f43bd..4a2aed1dc 100644 --- a/documentation/Maintenance.md +++ b/documentation/Maintenance.md @@ -752,11 +752,11 @@ The procedure recovers thirdparties based on the `cluster.yaml`. If rpm thirdpar restore_plan: thirdparties: /usr/bin/kubeadm: - source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubeadm + source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubeadm /usr/bin/kubelet: - source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubelet + source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubelet /usr/bin/kubectl: - source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubectl + source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubectl /usr/bin/calicoctl: source: https://github.com/projectcalico/calicoctl/releases/download/v3.14.1/calicoctl-linux-amd64 ``` diff --git a/examples/procedure.yaml/full-restore.yaml b/examples/procedure.yaml/full-restore.yaml index 7d47be3fe..cb2210354 100644 --- a/examples/procedure.yaml/full-restore.yaml +++ b/examples/procedure.yaml/full-restore.yaml @@ -12,11 +12,11 @@ restore_plan: peer_cacert: /etc/kubernetes/pki/etcd/ca.crt thirdparties: /usr/bin/kubeadm: - source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubeadm + source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubeadm /usr/bin/kubelet: - source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubelet + source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubelet /usr/bin/kubectl: - source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubectl + source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubectl /opt/cni/cni-plugins-linux.tgz: source: https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz /usr/bin/calicoctl: diff --git a/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml b/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml index f0ed2fc2b..744c0acc2 100644 --- a/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml @@ -43,6 +43,8 @@ kube-apiserver: version: v1.30.3 v1.31.0: version: v1.31.0 + v1.31.1: + version: v1.31.1 kube-controller-manager: v1.26.3: version: v1.26.3 @@ -86,6 +88,8 @@ kube-controller-manager: version: v1.30.3 v1.31.0: version: v1.31.0 + v1.31.1: + version: v1.31.1 kube-scheduler: v1.26.3: version: v1.26.3 @@ -129,6 +133,8 @@ kube-scheduler: version: v1.30.3 v1.31.0: version: v1.31.0 + v1.31.1: + version: v1.31.1 kube-proxy: v1.26.3: version: v1.26.3 @@ -172,6 +178,8 @@ kube-proxy: version: v1.30.3 v1.31.0: version: v1.31.0 + v1.31.1: + version: v1.31.1 pause: v1.26.3: version: '3.9' @@ -215,6 +223,8 @@ pause: version: '3.9' v1.31.0: version: '3.10' + v1.31.1: + version: '3.10' etcd: v1.26.3: version: 3.5.6-0 @@ -258,6 +268,8 @@ etcd: version: 3.5.12-0 v1.31.0: version: 3.5.15-0 + v1.31.1: + version: 3.5.15-0 coredns/coredns: v1.26.3: version: v1.9.3 @@ -301,3 +313,5 @@ coredns/coredns: version: v1.11.1 v1.31.0: version: v1.11.1 + v1.31.1: + version: v1.11.3 diff --git a/kubemarine/resources/configurations/compatibility/internal/packages.yaml b/kubemarine/resources/configurations/compatibility/internal/packages.yaml index beaa8456d..0aa91e072 100644 --- a/kubemarine/resources/configurations/compatibility/internal/packages.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/packages.yaml @@ -47,6 +47,8 @@ containerd: version_debian: 1.6.* v1.31.0: version_debian: 1.6.* + v1.31.1: + version_debian: 1.6.* containerdio: v1.26.3: version_rhel: 1.6* @@ -132,6 +134,10 @@ containerdio: version_rhel: 1.6* version_rhel8: 1.6* version_rhel9: 1.6* + v1.31.1: + version_rhel: 1.6* + version_rhel8: 1.6* + version_rhel9: 1.6* haproxy: version_rhel: 1.8* version_rhel8: 1.8* diff --git a/kubemarine/resources/configurations/compatibility/internal/plugins.yaml b/kubemarine/resources/configurations/compatibility/internal/plugins.yaml index 06941477f..b7b44b3b3 100644 --- a/kubemarine/resources/configurations/compatibility/internal/plugins.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/plugins.yaml @@ -47,6 +47,8 @@ calico: version: v3.28.1 v1.31.0: version: v3.28.1 + v1.31.1: + version: v3.28.1 nginx-ingress-controller: v1.26.3: version: v1.4.0 @@ -111,6 +113,9 @@ nginx-ingress-controller: v1.31.0: version: v1.11.1 webhook-version: v1.4.1 + v1.31.1: + version: v1.11.1 + webhook-version: v1.4.1 kubernetes-dashboard: v1.26.3: version: v2.7.0 @@ -175,6 +180,9 @@ kubernetes-dashboard: v1.31.0: version: v2.7.0 metrics-scraper-version: v1.0.8 + v1.31.1: + version: v2.7.0 + metrics-scraper-version: v1.0.8 local-path-provisioner: v1.26.3: version: v0.0.23 @@ -239,3 +247,6 @@ local-path-provisioner: v1.31.0: version: v0.0.27 busybox-version: 1.34.1 + v1.31.1: + version: v0.0.27 + busybox-version: 1.34.1 diff --git a/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml b/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml index f7d219eaf..f04228d34 100644 --- a/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml @@ -44,6 +44,8 @@ kubeadm: sha1: f840e75f5dc1001ebdd7e286c0e87e1090df011b v1.31.0: sha1: 3ac8ce4f0f17a21e5531213025b0090c42310d08 + v1.31.1: + sha1: 0a682af6436ce4e7188f93ddeebff5c2f3be1592 kubelet: v1.26.3: sha1: 5fe320fedaabb91d3770da19135412b7454bb28b @@ -87,6 +89,8 @@ kubelet: sha1: fbae53efc43ec715a45b05415294ab991ea087a2 v1.31.0: sha1: b5cb543e21faaa600a5aac54a2316a9fd5e5b8c6 + v1.31.1: + sha1: fc7d0a9859c97ec2a2a4ac9ec1814b131e8d875f kubectl: v1.26.3: sha1: 56916d87c3caef05489db932fd9e48d32ebdf634 @@ -130,6 +134,8 @@ kubectl: sha1: 097d6b02fabb284418a9c95ea81fa86fc3c85bb7 v1.31.0: sha1: 34214a4983223aa4e8d29063990a265b220bef5b + v1.31.1: + sha1: a0fd9dc942f533e2bdeaa4b2691fc408e334f922 calicoctl: # calicoctl version is duplicated from kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml # It also corresponds to the plugin version in kubemarine/resources/configurations/compatibility/internal/plugins.yaml @@ -196,6 +202,9 @@ calicoctl: v1.31.0: version: v3.28.1 sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 + v1.31.1: + version: v3.28.1 + sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 crictl: # crictl version is duplicated from kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml # for backward compatibility with clusters in a private environment. @@ -262,3 +271,6 @@ crictl: v1.31.0: version: v1.30.0 sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b + v1.31.1: + version: v1.30.0 + sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b diff --git a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml index 82ab730c5..0918585c0 100644 --- a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml +++ b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml @@ -140,6 +140,12 @@ compatibility_map: kubernetes-dashboard: v2.7.0 local-path-provisioner: v0.0.27 crictl: v1.30.0 + v1.31.1: + calico: v3.28.1 + nginx-ingress-controller: v1.11.1 + kubernetes-dashboard: v2.7.0 + local-path-provisioner: v0.0.27 + crictl: v1.30.0 # After any change, please run scripts/thirdparties/sync.py # The following optional keys are supported in addition to the 5 required software keys: diff --git a/kubemarine/resources/configurations/globals.yaml b/kubemarine/resources/configurations/globals.yaml index 8ed0df7eb..b14e1409a 100644 --- a/kubemarine/resources/configurations/globals.yaml +++ b/kubemarine/resources/configurations/globals.yaml @@ -128,19 +128,19 @@ thirdparties: /usr/bin/kubeadm: software_name: kubeadm source_prefix: - public: https://storage.googleapis.com/kubernetes-release/release + public: https://dl.k8s.io private: '{registry}/kubernetes/kubeadm' relative_path: '{version}/bin/linux/amd64/kubeadm' /usr/bin/kubelet: software_name: kubelet source_prefix: - public: https://storage.googleapis.com/kubernetes-release/release + public: https://dl.k8s.io private: '{registry}/kubernetes/kubelet' relative_path: '{version}/bin/linux/amd64/kubelet' /usr/bin/kubectl: software_name: kubectl source_prefix: - public: https://storage.googleapis.com/kubernetes-release/release + public: https://dl.k8s.io private: '{registry}/kubernetes/kubectl' relative_path: '{version}/bin/linux/amd64/kubectl' /usr/bin/calicoctl: diff --git a/test/unit/core/test_env.py b/test/unit/core/test_env.py index 825a4d5bc..b345d2a86 100644 --- a/test/unit/core/test_env.py +++ b/test/unit/core/test_env.py @@ -156,7 +156,7 @@ def test_kubernetes_version_env_variable(self): inventory = self.resources.working_inventory self.assertEqual(kubernetes_version, inventory['services']['kubeadm']['kubernetesVersion']) - expected_source = (f'https://storage.googleapis.com/kubernetes-release/release/' + expected_source = (f'https://dl.k8s.io/' f'{kubernetes_version}/bin/linux/amd64/kubeadm') self.assertEqual(expected_source, inventory['services']['thirdparties']['/usr/bin/kubeadm']['source']) @@ -183,7 +183,7 @@ def test_kubernetes_version_upgrade_env_variable(self): inventory = self.resources.working_inventory self.assertEqual(after, inventory['services']['kubeadm']['kubernetesVersion']) - self.assertEqual(f'https://storage.googleapis.com/kubernetes-release/release/{after}/bin/linux/amd64/kubeadm', + self.assertEqual(f'https://dl.k8s.io/{after}/bin/linux/amd64/kubeadm', inventory['services']['thirdparties']['/usr/bin/kubeadm']['source']) self.assertEqual('containerd_new', inventory['services']['packages']['associations']['rhel']['containerd']['package_name']) From 92b3a5f9c7902a9c216174e121dd9497ada3a745 Mon Sep 17 00:00:00 2001 From: niam0522 Date: Tue, 15 Oct 2024 16:24:40 +0530 Subject: [PATCH 15/17] removed_v1.31.0 --- .../configurations/compatibility/kubernetes_versions.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml index 0918585c0..bf178d8c2 100644 --- a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml +++ b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml @@ -134,12 +134,6 @@ compatibility_map: kubernetes-dashboard: v2.7.0 local-path-provisioner: v0.0.27 crictl: v1.30.0 - v1.31.0: - calico: v3.28.1 - nginx-ingress-controller: v1.11.1 - kubernetes-dashboard: v2.7.0 - local-path-provisioner: v0.0.27 - crictl: v1.30.0 v1.31.1: calico: v3.28.1 nginx-ingress-controller: v1.11.1 From db384c39bef3e3a2c3ca5bf1e3a4a17c26e6c346 Mon Sep 17 00:00:00 2001 From: sekr0614 Date: Tue, 15 Oct 2024 16:44:38 +0300 Subject: [PATCH 16/17] removed v1.31.0 --- .../compatibility/internal/kubernetes_images.yaml | 14 -------------- .../compatibility/internal/packages.yaml | 6 ------ .../compatibility/internal/plugins.yaml | 11 ----------- .../compatibility/internal/thirdparties.yaml | 12 ------------ test/unit/test_kubernetes_components.py | 6 +++--- 5 files changed, 3 insertions(+), 46 deletions(-) diff --git a/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml b/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml index 744c0acc2..d28333de5 100644 --- a/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml @@ -41,8 +41,6 @@ kube-apiserver: version: v1.30.1 v1.30.3: version: v1.30.3 - v1.31.0: - version: v1.31.0 v1.31.1: version: v1.31.1 kube-controller-manager: @@ -86,8 +84,6 @@ kube-controller-manager: version: v1.30.1 v1.30.3: version: v1.30.3 - v1.31.0: - version: v1.31.0 v1.31.1: version: v1.31.1 kube-scheduler: @@ -131,8 +127,6 @@ kube-scheduler: version: v1.30.1 v1.30.3: version: v1.30.3 - v1.31.0: - version: v1.31.0 v1.31.1: version: v1.31.1 kube-proxy: @@ -176,8 +170,6 @@ kube-proxy: version: v1.30.1 v1.30.3: version: v1.30.3 - v1.31.0: - version: v1.31.0 v1.31.1: version: v1.31.1 pause: @@ -221,8 +213,6 @@ pause: version: '3.9' v1.30.3: version: '3.9' - v1.31.0: - version: '3.10' v1.31.1: version: '3.10' etcd: @@ -266,8 +256,6 @@ etcd: version: 3.5.12-0 v1.30.3: version: 3.5.12-0 - v1.31.0: - version: 3.5.15-0 v1.31.1: version: 3.5.15-0 coredns/coredns: @@ -311,7 +299,5 @@ coredns/coredns: version: v1.11.1 v1.30.3: version: v1.11.1 - v1.31.0: - version: v1.11.1 v1.31.1: version: v1.11.3 diff --git a/kubemarine/resources/configurations/compatibility/internal/packages.yaml b/kubemarine/resources/configurations/compatibility/internal/packages.yaml index 0aa91e072..e34f949c2 100644 --- a/kubemarine/resources/configurations/compatibility/internal/packages.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/packages.yaml @@ -45,8 +45,6 @@ containerd: version_debian: 1.6.* v1.30.3: version_debian: 1.6.* - v1.31.0: - version_debian: 1.6.* v1.31.1: version_debian: 1.6.* containerdio: @@ -130,10 +128,6 @@ containerdio: version_rhel: 1.6* version_rhel8: 1.6* version_rhel9: 1.6* - v1.31.0: - version_rhel: 1.6* - version_rhel8: 1.6* - version_rhel9: 1.6* v1.31.1: version_rhel: 1.6* version_rhel8: 1.6* diff --git a/kubemarine/resources/configurations/compatibility/internal/plugins.yaml b/kubemarine/resources/configurations/compatibility/internal/plugins.yaml index b7b44b3b3..f36c893df 100644 --- a/kubemarine/resources/configurations/compatibility/internal/plugins.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/plugins.yaml @@ -45,8 +45,6 @@ calico: version: v3.27.3 v1.30.3: version: v3.28.1 - v1.31.0: - version: v3.28.1 v1.31.1: version: v3.28.1 nginx-ingress-controller: @@ -110,9 +108,6 @@ nginx-ingress-controller: v1.30.3: version: v1.11.1 webhook-version: v1.4.1 - v1.31.0: - version: v1.11.1 - webhook-version: v1.4.1 v1.31.1: version: v1.11.1 webhook-version: v1.4.1 @@ -177,9 +172,6 @@ kubernetes-dashboard: v1.30.3: version: v2.7.0 metrics-scraper-version: v1.0.8 - v1.31.0: - version: v2.7.0 - metrics-scraper-version: v1.0.8 v1.31.1: version: v2.7.0 metrics-scraper-version: v1.0.8 @@ -244,9 +236,6 @@ local-path-provisioner: v1.30.3: version: v0.0.27 busybox-version: 1.34.1 - v1.31.0: - version: v0.0.27 - busybox-version: 1.34.1 v1.31.1: version: v0.0.27 busybox-version: 1.34.1 diff --git a/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml b/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml index f04228d34..8ca725d20 100644 --- a/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml @@ -42,8 +42,6 @@ kubeadm: sha1: 85385f4b78b79bd1cd4d1ad5244422d372d5f758 v1.30.3: sha1: f840e75f5dc1001ebdd7e286c0e87e1090df011b - v1.31.0: - sha1: 3ac8ce4f0f17a21e5531213025b0090c42310d08 v1.31.1: sha1: 0a682af6436ce4e7188f93ddeebff5c2f3be1592 kubelet: @@ -87,8 +85,6 @@ kubelet: sha1: c62da6ab918b8e56d7c9b77e642ffc73ffdbffac v1.30.3: sha1: fbae53efc43ec715a45b05415294ab991ea087a2 - v1.31.0: - sha1: b5cb543e21faaa600a5aac54a2316a9fd5e5b8c6 v1.31.1: sha1: fc7d0a9859c97ec2a2a4ac9ec1814b131e8d875f kubectl: @@ -132,8 +128,6 @@ kubectl: sha1: bced94239f1dbdb04d3a661a067bf9587865b6e8 v1.30.3: sha1: 097d6b02fabb284418a9c95ea81fa86fc3c85bb7 - v1.31.0: - sha1: 34214a4983223aa4e8d29063990a265b220bef5b v1.31.1: sha1: a0fd9dc942f533e2bdeaa4b2691fc408e334f922 calicoctl: @@ -199,9 +193,6 @@ calicoctl: v1.30.3: version: v3.28.1 sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 - v1.31.0: - version: v3.28.1 - sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 v1.31.1: version: v3.28.1 sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 @@ -268,9 +259,6 @@ crictl: v1.30.3: version: v1.30.0 sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b - v1.31.0: - version: v1.30.0 - sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b v1.31.1: version: v1.30.0 sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b diff --git a/test/unit/test_kubernetes_components.py b/test/unit/test_kubernetes_components.py index 476d289d9..012c4279e 100644 --- a/test/unit/test_kubernetes_components.py +++ b/test/unit/test_kubernetes_components.py @@ -133,7 +133,7 @@ def test_kubelet_local_mode_enrichment(self): # Enriched featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+ inventory = demo.generate_inventory(**demo.ALLINONE) - inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.0' + inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.1' cluster = demo.new_cluster(inventory) kubeadm = cluster.inventory['services']['kubeadm'] self.assertIsNotNone(kubeadm.get('featureGates')) @@ -141,7 +141,7 @@ def test_kubelet_local_mode_enrichment(self): # Enriched featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+ with not empty featureGates inventory = demo.generate_inventory(**demo.ALLINONE) - inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.0' + inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.1' inventory['services'].setdefault('kubeadm', {}).setdefault('featureGates', {})['foo'] = 'bar' cluster = demo.new_cluster(inventory) kubeadm = cluster.inventory['services']['kubeadm'] @@ -151,7 +151,7 @@ def test_kubelet_local_mode_enrichment(self): # Do not change featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+ if value is overridden inventory = demo.generate_inventory(**demo.ALLINONE) - inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.0' + inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.1' inventory['services'].setdefault('kubeadm', {}).setdefault('featureGates', {})['ControlPlaneKubeletLocalMode'] = False cluster = demo.new_cluster(inventory) kubeadm = cluster.inventory['services']['kubeadm'] From 129f010e4e956263e7b4e1a6b40f51c7759b8e80 Mon Sep 17 00:00:00 2001 From: niam0522 Date: Wed, 16 Oct 2024 13:37:08 +0530 Subject: [PATCH 17/17] Doc_update --- documentation/Installation.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/documentation/Installation.md b/documentation/Installation.md index cd29c5749..d4ab661cf 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -6144,22 +6144,22 @@ The tables below shows the correspondence of versions that are supported and is | | kubernetesui/metrics-scraper | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | v1.0.8 | Required only if Kubernetes Dashboard plugin is set to be installed. | | | rancher/local-path-provisioner | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | v0.0.27 | Required only if local-path provisioner plugin is set to be installed. | -## Default Dependent Components Versions for Kubernetes Versions v1.31.0 +## Default Dependent Components Versions for Kubernetes Versions v1.31.1 | Type | Name | Versions | | | | | | | Note | |----------|----------------------------------------------------------------|------------------|------------------------------|--------------|--------------|-------------------|-----------|-----------|------------------------------------------------------------------------------------------------------------| | | | CentOS RHEL 7.5+ | CentOS RHEL Oracle Linux 8.4 | Ubuntu 20.04 | Ubuntu 22.04 | Oracle Linux 7.5+ | RHEL 8.6+ | RockyLinux 8.6+ | | -| binaries | kubeadm | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | SHA1: 3ac8ce4f0f17a21e5531213025b0090c42310d08 | -| | kubelet | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | SHA1: b5cb543e21faaa600a5aac54a2316a9fd5e5b8c6 | -| | kubectl | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | SHA1: 34214a4983223aa4e8d29063990a265b220bef5b | +| binaries | kubeadm | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | SHA1: 0a682af6436ce4e7188f93ddeebff5c2f3be1592 | +| | kubelet | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | SHA1: fc7d0a9859c97ec2a2a4ac9ec1814b131e8d875f | +| | kubectl | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | SHA1: a0fd9dc942f533e2bdeaa4b2691fc408e334f922 | | | calicoctl | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | v3.28.1 | SHA1: 80f164a8248b8fa501f35f12cbcac0b059e665e2 Required only if calico is installed. | | | crictl | v1.30.0 | v1.30.0 | v1.30.0 | v1.30.0 | v1.30.0 | v1.30.0 | v1.30.0 | SHA1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b | | rpms | containerd.io | 1.6.* | 1.6.* | 1.6.* | 1.6.* | 1.6.* | 1.6.* | 1.6.* | | | | haproxy/rh-haproxy | 1.8 | 1.8 | 2.* | 2.* | 1.8 | 1.8 | 1.8 | Required only if balancers are presented in the deployment scheme. | | | keepalived | 1.3 | 2.1 | 2.* | 2.* | 1.3 | 2.1 | 2.1 | Required only if VRRP is presented in the deployment scheme. | -| images | registry.k8s.io/kube-apiserver | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | | -| | registry.k8s.io/kube-controller-manager | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | | -| | registry.k8s.io/kube-proxy | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | | -| | registry.k8s.io/kube-scheduler | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | v1.31.0 | | +| images | registry.k8s.io/kube-apiserver | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | | +| | registry.k8s.io/kube-controller-manager | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | | +| | registry.k8s.io/kube-proxy | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | | +| | registry.k8s.io/kube-scheduler | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | v1.31.1 | | | | registry.k8s.io/coredns | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | v1.11.1 | | | | registry.k8s.io/pause | 3.10 | 3.10 | 3.10 | 3.10 | 3.10 | 3.10 | 3.10 | | | | registry.k8s.io/etcd | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | 3.5.15-0 | |