[CPDEV-96929] Support of granular 3rd party updates per specific Kubernetes version

documentation/Maintenance.md

@@ -216,7 +216,10 @@ The configuration format for the plugins is the same.
   * https://kubernetes.io/docs/tasks/run-application/configure-pdb/#unhealthy-pod-eviction-policy is configured to _AlwaysAllow_
 * API versions `extensions/v1beta1` and `networking.k8s.io/v1beta1` are not supported starting from Kubernetes 1.22 and higher. Need to update ingress to the new API `networking.k8s.io/v1`. More info: https://kubernetes.io/docs/reference/using-api/deprecation-guide/#ingress-v122
 * Before starting the upgrade, make sure you make a backup. For more information, see the section [Backup Procedure](#backup-procedure).
-* The upgrade procedure only maintains upgrading from one `supported` version to the next `supported` version. For example, from 1.18 to 1.20 or from 1.20 to 1.21.
+* The upgrade procedure only maintains upgrading from one `supported` version to the higher `supported` version.
+  The target version must also be the latest patch version supported by Kubemarine.
+  For example, upgrade is allowed from v1.26.7 to v1.26.11, or from v1.26.7 to v1.27.8, or from v1.26.7 to v1.28.4 through v1.27.8,
+  but not from v1.26.7 to v1.27.1 as v1.27.1 is not the latest supported patch version of Kubernetes v1.27.
 * Since Kubernetes v1.25 doesn't support PSP, any clusters with `PSP` enabled must be migrated to `PSS` **before the upgrade** procedure running. For more information see the [Admission Migration Procedure](#admission-migration-procedure). The migration procedure is very important for Kubernetes cluster. If the solution doesn't have appropriate description about what `PSS` profile should be used for every namespace, it is better not to migrate from PSP for a while.  
 
 ### Upgrade Procedure Parameters

kubemarine/kubernetes/__init__.py

@@ -19,7 +19,7 @@
 import uuid
 from contextlib import contextmanager
 from copy import deepcopy
-from typing import List, Dict, Tuple, Iterator, Any, Optional
+from typing import List, Dict, Iterator, Any, Optional
 
 import yaml
 from jinja2 import Template
@@ -40,6 +40,7 @@
 ERROR_SAME='Kubernetes old version \"%s\" is the same as new one \"%s\"'
 ERROR_MAJOR_RANGE_EXCEEDED='Major version \"%s\" rises to new \"%s\" more than one'
 ERROR_MINOR_RANGE_EXCEEDED='Minor version \"%s\" rises to new \"%s\" more than one'
+ERROR_NOT_LATEST_PATCH='New version \"%s\" is not the latest supported patch version \"%s\"'
 
 
 def is_container_runtime_not_configurable(cluster: KubernetesCluster) -> bool:
@@ -1034,31 +1035,45 @@ def expect_kubernetes_version(cluster: KubernetesCluster, version: str,
     raise Exception('In the expected time, the nodes did not receive correct Kubernetes version')
 
 
-def test_version_upgrade_possible(old: str, new: str, skip_equal: bool = False) -> None:
-    versions_unchanged = {
-        'old': old.strip(),
-        'new': new.strip()
-    }
-    versions: Dict[str, Tuple[int, int, int]] = {}
+def is_version_upgrade_possible(old: str, new: str) -> bool:
+    try:
+        test_version_upgrade_possible(old, new)
+        return True
+    except Exception:
+        return False
+
 
-    for v_type, version in versions_unchanged.items():
-        versions[v_type] = utils.version_key(version)
+def test_version_upgrade_possible(old: str, new: str, skip_equal: bool = False) -> None:
+    old = old.strip()
+    new = new.strip()
+    old_version_key = utils.version_key(old)
+    new_version_key = utils.version_key(new)
 
     # test new is greater than old
-    if versions['old'] > versions['new']:
-        raise Exception(ERROR_DOWNGRADE % (versions_unchanged['old'], versions_unchanged['new']))
+    if old_version_key > new_version_key:
+        raise Exception(ERROR_DOWNGRADE % (old, new))
 
     # test new is the same as old
-    if versions['old'] == versions['new'] and not skip_equal:
-        raise Exception(ERROR_SAME % (versions_unchanged['old'], versions_unchanged['new']))
+    if old_version_key == new_version_key and not skip_equal:
+        raise Exception(ERROR_SAME % (old, new))
 
     # test major step is not greater than 1
-    if versions['new'][0] - versions['old'][0] > 1:
-        raise Exception(ERROR_MAJOR_RANGE_EXCEEDED % (versions_unchanged['old'], versions_unchanged['new']))
+    if new_version_key[0] - old_version_key[0] > 1:
+        raise Exception(ERROR_MAJOR_RANGE_EXCEEDED % (old, new))
 
     # test minor step is not greater than 1
-    if versions['new'][1] - versions['old'][1] > 1:
-        raise Exception(ERROR_MINOR_RANGE_EXCEEDED % (versions_unchanged['old'], versions_unchanged['new']))
+    if new_version_key[1] - old_version_key[1] > 1:
+        raise Exception(ERROR_MINOR_RANGE_EXCEEDED % (old, new))
+
+    # test the target version is the latest supported patch version
+    new_minor_version = utils.minor_version(new)
+    latest_supported_patch_version = max(
+        (v for v in static.KUBERNETES_VERSIONS['compatibility_map']
+         if utils.minor_version(v) == new_minor_version),
+        key=utils.version_key)
+
+    if new != latest_supported_patch_version:
+        raise Exception(ERROR_NOT_LATEST_PATCH % (new, latest_supported_patch_version))
 
 
 def recalculate_proper_timeout(cluster: KubernetesCluster, timeout: int) -> int:

scripts/thirdparties/src/software/plugins.py

@@ -11,13 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import itertools
 import os
 import tarfile
 import tempfile
 from typing import List, Tuple, Dict, Any
 
-from kubemarine import demo
+from kubemarine import demo, kubernetes
 from kubemarine.core import static, utils, log
 from kubemarine.plugins import builtin
 from kubemarine.plugins.manifest import Manifest, get_default_manifest_path, Identity
@@ -131,6 +131,9 @@ def validate_plugin_versions(kubernetes_versions: Dict[str, Dict[str, str]], plu
     for i, older_k8s_version in enumerate(k8s_versions):
         for j in range(i + 1, len(k8s_versions)):
             newer_k8s_version = k8s_versions[j]
+            if not kubernetes.is_version_upgrade_possible(older_k8s_version, newer_k8s_version):
+                continue
+
             older_version = kubernetes_versions[older_k8s_version][plugin_name]
             newer_version = kubernetes_versions[newer_k8s_version][plugin_name]
             if key(newer_version) < key(older_version):
@@ -144,6 +147,8 @@ def validate_plugin_versions(kubernetes_versions: Dict[str, Dict[str, str]], plu
 def validate_compatibility_map(compatibility_map: CompatibilityMap, plugin_name: str) -> None:
     plugin_mapping: dict = compatibility_map.compatibility_map[plugin_name]
     k8s_versions = list(plugin_mapping)
+    latest_patch_k8s_versions = [sorted(versions, key=utils.version_key)[-1]
+                                 for _, versions in itertools.groupby(k8s_versions, key=utils.minor_version)]
 
     extra_images = []
     if plugin_name == 'nginx-ingress-controller':
@@ -155,10 +160,13 @@ def validate_compatibility_map(compatibility_map: CompatibilityMap, plugin_name:
 
     for extra_image in extra_images:
         version_key = f"{extra_image}-version"
-        for i, newer_k8s_version in enumerate(k8s_versions):
-            for j in range(i - 1):
-                k8s_version = k8s_versions[i - 1]
-                older_k8s_version = k8s_versions[j]
+        for i, older_k8s_version in enumerate(k8s_versions):
+            newer_latest_patch_versions = [v for v in latest_patch_k8s_versions
+                                           if utils.version_key(v) > utils.version_key(older_k8s_version)]
+
+            for j in range(len(newer_latest_patch_versions) - 1):
+                k8s_version = newer_latest_patch_versions[j]
+                newer_k8s_version = newer_latest_patch_versions[j + 1]
                 version_A = plugin_mapping[older_k8s_version][version_key]
                 version_B = plugin_mapping[k8s_version][version_key]
                 version_A1 = plugin_mapping[newer_k8s_version][version_key]

scripts/thirdparties/src/software/thirdparties.py

@@ -15,7 +15,7 @@
 import os
 from typing import List, Tuple, Dict
 
-from kubemarine import thirdparties
+from kubemarine import thirdparties, kubernetes
 from kubemarine.core import utils
 from ..shell import curl, TEMP_FILE, SYNC_CACHE
 from ..tracker import SummaryTracker, ComposedTracker
@@ -90,6 +90,9 @@ def validate_thirdparty_versions(kubernetes_versions: Dict[str, Dict[str, str]],
     for i, older_k8s_version in enumerate(k8s_versions):
         for j in range(i + 1, len(k8s_versions)):
             newer_k8s_version = k8s_versions[j]
+            if not kubernetes.is_version_upgrade_possible(older_k8s_version, newer_k8s_version):
+                continue
+
             older_version = kubernetes_versions[older_k8s_version][thirdparty_name]
             newer_version = kubernetes_versions[newer_k8s_version][thirdparty_name]
             if key(newer_version) < key(older_version):

test/unit/test_upgrade.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import itertools
 import random
 import re
 import unittest
@@ -28,7 +29,7 @@
 class UpgradeVerifyUpgradePlan(unittest.TestCase):
 
     def test_valid_upgrade_plan(self):
-        upgrade.verify_upgrade_plan(self.k8s_versions()[0], self.k8s_versions()[1:])
+        upgrade.verify_upgrade_plan(self.k8s_versions()[0], self.latest_patch_k8s_versions()[1:])
 
     def test_invalid_upgrade_plan(self):
         k8s_oldest = self.k8s_versions()[0]
@@ -66,17 +67,30 @@ def test_incorrect_inventory_same_version(self):
                                                % (re.escape(old_kubernetes_version), re.escape(new_kubernetes_version))):
             upgrade.verify_upgrade_plan(old_kubernetes_version, [new_kubernetes_version])
 
+    def test_incorrect_inventory_not_latest_patch_version(self):
+        old_kubernetes_version = 'v1.27.1'
+        new_kubernetes_version = 'v1.28.0'
+        latest_supported_patch_version = next(v for v in self.latest_patch_k8s_versions()
+                                              if kutils.minor_version(v) == kutils.minor_version(new_kubernetes_version))
+        with self.assertRaisesRegex(Exception, kubernetes.ERROR_NOT_LATEST_PATCH
+                                               % (re.escape(new_kubernetes_version), re.escape(latest_supported_patch_version))):
+            upgrade.verify_upgrade_plan(old_kubernetes_version, [new_kubernetes_version])
+
     def test_upgrade_plan_sort(self):
         k8s_oldest = self.k8s_versions()[0]
-        k8s_versions = list(self.k8s_versions())[1:]
+        k8s_versions = list(self.latest_patch_k8s_versions())[1:]
         random.shuffle(k8s_versions)
         result = upgrade.verify_upgrade_plan(k8s_oldest, k8s_versions)
 
-        self.assertEqual(self.k8s_versions()[1:], result)
+        self.assertEqual(self.latest_patch_k8s_versions()[1:], result)
 
     def k8s_versions(self) -> List[str]:
         return sorted(list(static.KUBERNETES_VERSIONS['compatibility_map']), key=kutils.version_key)
 
+    def latest_patch_k8s_versions(self) -> List[str]:
+        return [sorted(versions, key=kutils.version_key)[-1]
+                for _, versions in itertools.groupby(self.k8s_versions(), key=kutils.minor_version)]
+
 
 def generate_upgrade_environment(old) -> Tuple[dict, dict]:
     inventory = demo.generate_inventory(**demo.MINIHA_KEEPALIVED)

test/unit/tools/thirdparties/test_sync.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import io
+import itertools
 import re
 import unittest
 from contextlib import contextmanager
@@ -343,8 +344,23 @@ def test_mapped_software_not_ascending_order(self):
                 with self.assertRaisesRegex(Exception, error_msg_pattern.format(**kwargs)):
                     self.run_sync()
 
+    def test_mapped_software_latest_patch_ascending_order(self):
+        for software_name in ('calico', 'nginx-ingress-controller', 'kubernetes-dashboard', 'local-path-provisioner',
+                              'crictl'):
+            with self.subTest(software_name):
+                k8s_latest = self.k8s_versions()[-1]
+                self.kubernetes_versions = FakeKubernetesVersions()
+                software_version = self.compatibility_map()[k8s_latest][software_name]
+                for latest_patch_k8s_version in self.latest_patch_k8s_versions():
+                    software_version = test_utils.increment_version(software_version)
+                    self.compatibility_map()[latest_patch_k8s_version][software_name] = software_version
+
+                # No error should be raised
+                self.run_sync()
+
     def test_plugins_suspicious_aba_extra_images(self):
-        if len(self.k8s_versions()) < 3:
+        latest_patch_k8s_versions = self.latest_patch_k8s_versions()
+        if len(latest_patch_k8s_versions) < 3:
             self.skipTest("Cannot check suspicions A -> B -> A versions of extra images,")
 
         plugin_images = {
@@ -355,16 +371,14 @@ def test_plugins_suspicious_aba_extra_images(self):
         for plugin, extra_image in plugin_images.items():
             with self.subTest(plugin):
                 self.kubernetes_versions = FakeKubernetesVersions()
-                self.compatibility_map()[self.k8s_versions()[0]][extra_image] = 'A'
-                self.compatibility_map()[self.k8s_versions()[1]][extra_image] = 'B'
-                self.compatibility_map()[self.k8s_versions()[-1]][extra_image] = 'A'
+                self.compatibility_map()[latest_patch_k8s_versions[0]][extra_image] = 'A'
+                self.compatibility_map()[latest_patch_k8s_versions[1]][extra_image] = 'B'
+                self.compatibility_map()[latest_patch_k8s_versions[-1]][extra_image] = 'A'
 
                 kwargs = {
                     'image': re.escape(extra_image), 'plugin': re.escape(plugin),
-                    'version_A': 'A', 'version_B': '.*',
-                    'older_k8s_version': re.escape(self.k8s_versions()[0]),
-                    'newer_k8s_version': re.escape(self.k8s_versions()[-1]),
-                    'k8s_version': '.*',
+                    'version_A': '.*', 'version_B': '.*',
+                    'older_k8s_version': '.*', 'newer_k8s_version': '.*', 'k8s_version': '.*',
                 }
                 with self.assertRaisesRegex(Exception, ERROR_SUSPICIOUS_ABA_VERSIONS.format(**kwargs)):
                     self.run_sync()
@@ -535,6 +549,10 @@ def compatibility_map(self) -> dict:
     def k8s_versions(self) -> List[str]:
         return sorted(self.compatibility_map(), key=utils.version_key)
 
+    def latest_patch_k8s_versions(self) -> List[str]:
+        return [sorted(versions, key=utils.version_key)[-1]
+                for _, versions in itertools.groupby(self.k8s_versions(), key=utils.minor_version)]
+
 
 if __name__ == '__main__':
     unittest.main()