From f79532ca97540f71db2a93f709e386d7c32b6bc3 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Tue, 2 Jul 2024 18:08:53 +0200 Subject: [PATCH 01/49] list-modules. Add upstream_name attribute schema --- .../cluster/actions/list-modules/validate-output.json | 5 +++++ .../var/lib/nethserver/cluster/repodata-schema.json | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index a49fe0906..be6a5f8e3 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -99,6 +99,11 @@ "type": "string", "description": "Unique name of a package" }, + "upstream_name": { + "type": "string", + "description": "The alternative software name and version number, if they differ from the package name and version", + "example": "Nextcloud Hub 7" + }, "description": { "type": "object", "description": "A map of language codes (eg. en, it) with the translated description" diff --git a/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json b/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json index 332401aef..591b7d736 100644 --- a/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json +++ b/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json @@ -74,6 +74,11 @@ "type": "string", "description": "Common name of a package" }, + "upstream_name": { + "type": "string", + "description": "The alternative software name and version number, if they differ from the package name and version", + "example": "Nextcloud Hub 7" + }, "description": { "type": "object", "description": "A map of language code and description of the packaged translated in the indexed language" From 4e4813dd55c3e77420aa52a24f5d506d0c81b511 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Tue, 2 Jul 2024 18:11:00 +0200 Subject: [PATCH 02/49] list-modules. Add terms_url attribute schema --- .../cluster/actions/list-modules/validate-output.json | 4 ++++ .../imageroot/var/lib/nethserver/cluster/repodata-schema.json | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index be6a5f8e3..42f0cf43c 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -140,6 +140,10 @@ "docs": { "type": "object", "parameters": { + "terms_url": { + "type": "uri", + "description": "Optional link to the application Terms & Conditions document" + }, "documentation_url": { "type": "uri", "description": "Link to the package documentation" diff --git a/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json b/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json index 591b7d736..668c376e2 100644 --- a/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json +++ b/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json @@ -115,6 +115,10 @@ "docs": { "type": "object", "parameters": { + "terms_url": { + "type": "uri", + "description": "Optional link to the application Terms & Conditions document" + }, "documentation_url": { "type": "uri", "description": "Link to the package documentation" From b484bc638218aad202d28571e19009cbc1f384f7 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Wed, 3 Jul 2024 16:32:09 +0200 Subject: [PATCH 03/49] Implementation of testing_update attribute --- .../usr/local/agent/pypkg/cluster/modules.py | 85 +++++++++++-------- .../cluster/actions/list-modules/50read | 2 +- .../actions/list-modules/validate-output.json | 6 +- 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py index f0088578b..54b05bb70 100644 --- a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py +++ b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py @@ -238,6 +238,10 @@ def _fetch_metadata_json(module_id, image_name): def list_available(rdb, skip_core_modules = False): """Iterate over enabled repositories and return available modules respecting the repository priority.""" + modules = _get_available_modules(rdb, skip_core_modules) + return list(modules.values()) + +def _get_available_modules(rdb, skip_core_modules = False): modules = {} repositories = [] # List all modules from enabled repositories @@ -269,7 +273,7 @@ def list_available(rdb, skip_core_modules = False): vmetadata["updates"] = [] vmetadata["id"] = image_name modules[module_source] = vmetadata - return list(modules.values()) + return modules def list_installed(rdb, skip_core_modules = False): installed = {} @@ -315,43 +319,54 @@ def list_installed_core(rdb): return installed -def list_updates(rdb, skip_core_modules = False): +def list_updates(rdb, skip_core_modules=False, with_testing_update=False): updates = [] - installed = list_installed(rdb, skip_core_modules) - available = list_available(rdb, skip_core_modules) - - for module in available: - if module["source"] not in installed.keys(): - continue - newest_version = None - for version in module["versions"]: - try: - # skip bogus version tag - v = semver.Version.parse(version["tag"]) - except: - continue - # Skip testing versions if testing is disabled - testing = rdb.hget(f'cluster/repository/{module["repository"]}', 'testing') - if testing != "1" and not v.prerelease is None: - continue - newest_version = version["tag"] - break - - # Handle multiple instances of the same module - for instance in installed[module["source"]]: + installed_modules = list_installed(rdb, skip_core_modules) + available_modules = _get_available_modules(rdb, skip_core_modules) + + repo_testing_cache = {} + def repo_has_testing_flag(repo_name): + if repo_name not in repo_testing_cache: + repo_testing_cache[repo_name] = rdb.hget(f'cluster/repository/{repo_name}', 'testing') == "1" + return repo_testing_cache[repo_name] + + flat_instance_list = list(mi for module_instances in installed_modules.values() for mi in module_instances) + for instance in flat_instance_list: + if not instance['source'] in available_modules: + continue # skip instance if is not available from any repository + try: + current_version = semver.parse_version_info(instance['version']) + except: + continue # skip development version: instance must be updated manually + # Assuming the versions array is sorted in decreasing oreder, look + # up update candidates for both stable and testing updates: + update_candidate = None + testing_update_candidate = None + available_module = available_modules[instance['source']] + repository_name = available_module['repository'] + for atag in list(aver['tag'] for aver in available_module['versions']): try: - cur = semver.Version.parse(instance["version"]) + available_version = semver.parse_version_info(atag) except: - # skip installed instanced with dev version - continue - - # Version are already sorted - # First match is the newest release - if v > cur: - # Create a copy to not change original object - update = instance.copy() - update["update"] = version["tag"] - updates.append(update) + continue # skip non-semver available tag + if available_version <= current_version: + continue # ignore tags that do not update the current one + if update_candidate is None and ( + repo_has_testing_flag(repository_name) + or not available_version.prerelease + ): + update_candidate = available_version + if testing_update_candidate is None and with_testing_update and available_version.prerelease: + testing_update_candidate = available_version + + # If a stable or testing candidate has been found, add this + # instance to the updates list. + if update_candidate: + instance['update'] = str(update_candidate) + if testing_update_candidate: + instance['testing_update'] = str(testing_update_candidate) + if 'update' in instance or 'testing_update' in instance: + updates.append(instance) return updates diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read index c0f8f5f02..4772b4e6d 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read @@ -35,7 +35,7 @@ def get_module(source, modules): rdb = agent.redis_connect(privileged=True) installed = cluster.modules.list_installed(rdb, skip_core_modules = True) available = cluster.modules.list_available(rdb, skip_core_modules = True) -updates = cluster.modules.list_updates(rdb, skip_core_modules = True) +updates = cluster.modules.list_updates(rdb, skip_core_modules=True, with_testing_update=True) # Prepare variables for later use for a in available: diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index 42f0cf43c..4c899f062 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -226,9 +226,13 @@ "type": "string", "description": "A valid semantic version extracted from image tag" }, + "testing_update": { + "type": "string", + "description": "A semver prerelease tag, greater than 'version' field" + }, "update": { "type": "string", - "description": "A valid semantic version extracted from image tag wich should be greater than 'version' field" + "description": "A semver tag greater than 'version' field" } }, "required": [ From 3657e91cd3774475d435803badedd4940929af50 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Jul 2024 16:39:42 +0200 Subject: [PATCH 04/49] Add terms_url to get-subscription action --- .../cluster/actions/get-subscription/10get_subscription | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription b/core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription index 5ea2743e6..e1c535d52 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription +++ b/core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription @@ -12,6 +12,8 @@ import sys import json import requests, urllib3.util +terms_url = "https://docs.nethserver.org/projects/ns8/en/latest/subscription.html#terms-and-conditions" + def _get_http_session(): osession = requests.Session() osession.timeout = 15 # Timout for HTTP connections @@ -95,4 +97,4 @@ elif hsubscription["provider"] == "nsent": elif hsubscription["provider"] == "nscom": hsubscription.update(fetch_subscription_info_nscom(rdb, hsubscription)) -json.dump({"subscription": hsubscription}, fp=sys.stdout) +json.dump({"subscription": hsubscription, "terms_url": terms_url}, fp=sys.stdout) From 85dc80d0a6107dfcccf7e44c60ac2a6b4d5ba896 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Jul 2024 17:58:27 +0200 Subject: [PATCH 05/49] Definition of certification_level attribute --- .../usr/local/agent/pypkg/cluster/modules.py | 30 ++++++++++++++++--- .../actions/list-modules/validate-output.json | 6 ++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py index 54b05bb70..4d6ac5f79 100644 --- a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py +++ b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py @@ -51,7 +51,21 @@ def _get_downloaded_logos(): logos[os.path.basename(app)] = logo return logos -def _parse_repository_metadata(repository_name, repository_url, repository_updated, repodata, skip_core_modules = False, skip_testing_versions = False): +def _calc_certification_level(repository_authority, package, hsubscription=None): + if repository_authority in ["distfeed.nethserver.org", "subscription.nethserver.com"]: + certification_level = 3 + elif repository_authority == "forge.nethserver.org": + certification_level = 2 + else: + certification_level = 1 + # If we trust the repo metadata, elevate up to level 5 + if certification_level == 3: + if package["source"].startswith("ghcr.io/nethserver/") or package["source"].startswith("ghcr.io/nethesis/"): + certification_level = 4 if hsubscription is None else 5 + + return certification_level + +def _parse_repository_metadata(repository_name, repository_url, repository_updated, repodata, skip_core_modules=False, skip_testing_versions=False, hsubscription=None): modules = [] try: @@ -59,6 +73,12 @@ def _parse_repository_metadata(repository_name, repository_url, repository_updat except: return modules + try: + repository_authority = urllib.parse.urlparse(repository_url).hostname + except Exception as ex: + repository_authority = None + print(agent.SD_WARNING + f"Unable to parse repository {repository_name} URL: {repository_url}", ex, file=sys.stderr) + def ignore_testing(version): if skip_testing_versions and version["testing"] is True: return False @@ -74,6 +94,7 @@ def ignore_testing(version): package["repository"] = repository_name package["repository_updated"] = repository_updated + package["certification_level"] = _calc_certification_level(repository_authority, package, hsubscription) # Set absolute path for logo if package["logo"]: @@ -110,12 +131,12 @@ def _get_http_session(): def _list_repository_modules(rdb, repository_name, repository_url, skip_core_modules = False, skip_testing_versions=False): key = f'cluster/repository_cache/{repository_name}' cache = rdb.hgetall(key) + hsubscription = rdb.hgetall("cluster/subscription") or None if cache: - return _parse_repository_metadata(repository_name, repository_url, cache["updated"], cache["data"], skip_core_modules, skip_testing_versions) + return _parse_repository_metadata(repository_name, repository_url, cache["updated"], cache["data"], skip_core_modules, skip_testing_versions, hsubscription=hsubscription) url = _urljoin(repository_url, "repodata.json") try: - hsubscription = rdb.hgetall("cluster/subscription") with _get_http_session() as osession: if hsubscription and url.startswith("https://subscription.nethserver.com/"): # Send system_id for HTTP Basic authentication @@ -128,7 +149,7 @@ def _list_repository_modules(rdb, repository_name, repository_url, skip_core_mod # If repository is not accessible or invalid, just return an empty array return [] - modules = _parse_repository_metadata(repository_name, repository_url, updated, repodata, skip_core_modules, skip_testing_versions) + modules = _parse_repository_metadata(repository_name, repository_url, updated, repodata, skip_core_modules, skip_testing_versions, hsubscription=hsubscription) # Save inside the cache if data is valid if modules: # Save also repodata file date @@ -229,6 +250,7 @@ def _fetch_metadata_json(module_id, image_name): ometadata.setdefault("screenshots", []) ometadata.setdefault("repository", "__local__") ometadata.setdefault("repository_updated", repository_updated_timestamp) + ometadata["certification_level"] = 0 try: ometadata['logo'] = glob(f'{path_prefix}apps/{module_id}/img/*logo*png')[0].removeprefix(path_prefix) except Exception as ex: diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index 4c899f062..0d36e313e 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -60,6 +60,7 @@ ], "repository": "t3", "repository_updated": "Mon, 28 Jun 2021 14:42:44 GMT", + "certification_level": 2, "updates": [ { "id": "dokuwiki2", @@ -104,6 +105,10 @@ "description": "The alternative software name and version number, if they differ from the package name and version", "example": "Nextcloud Hub 7" }, + "certification_level": { + "type": "integer", + "description": "The higher, the better: 0=unknown certification, 5=max" + }, "description": { "type": "object", "description": "A map of language codes (eg. en, it) with the translated description" @@ -288,6 +293,7 @@ "authors", "docs", "source", + "certification_level", "versions", "installed", "updates" From 1315972853391d914c29a59b843e3c6cd81f7d3e Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Jul 2024 10:26:53 +0200 Subject: [PATCH 06/49] Relax org.nethserver.rootfull parsing If a non-numeric value is used, do not abort the install but consider it a False value. --- .../var/lib/nethserver/cluster/actions/add-module/50update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update b/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update index de0b8248a..0dd57b815 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update +++ b/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update @@ -100,7 +100,7 @@ with subprocess.Popen(['podman', 'image', 'inspect', image_url], stdout=subproce inspect_image_repodigest = inspect[0]['RepoDigests'][0] if 'org.nethserver.rootfull' in inspect_labels: - is_rootfull = int(inspect_labels['org.nethserver.rootfull']) == 1 + is_rootfull = inspect_labels['org.nethserver.rootfull'] == "1" else: is_rootfull = False From a06735745d5e601dd491f8b8d4fc2e88b8dbab88 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Jul 2024 10:28:44 +0200 Subject: [PATCH 07/49] Definition of "rootfull" attribute --- .../imageroot/usr/local/agent/pypkg/cluster/modules.py | 10 ++++++++++ .../cluster/actions/list-modules/validate-output.json | 9 +++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py index 4d6ac5f79..60a31798c 100644 --- a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py +++ b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py @@ -110,6 +110,15 @@ def ignore_testing(version): package["versions"] = list(filter(ignore_testing, package["versions"])) if len(package["versions"]) > 0: + try: + package_is_rootfull = package["versions"][0]["labels"]["org.nethserver.rootfull"] == "1" + except: + package_is_rootfull = False + # Ignore untrusted rootfull application, if a subscription is active + if hsubscription and package_is_rootfull and package["certification_level"] < 3: + print(agent.SD_WARNING + f"Ignoring image of rootfull application {package['source']}: certification_level {package['certification_level']} is too low", file=sys.stderr) + continue # skip package + package['rootfull'] = package_is_rootfull modules.append(package) return modules @@ -251,6 +260,7 @@ def _fetch_metadata_json(module_id, image_name): ometadata.setdefault("repository", "__local__") ometadata.setdefault("repository_updated", repository_updated_timestamp) ometadata["certification_level"] = 0 + ometadata["rootfull"] = False try: ometadata['logo'] = glob(f'{path_prefix}apps/{module_id}/img/*logo*png')[0].removeprefix(path_prefix) except Exception as ex: diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index 0d36e313e..0d8bdd7a9 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -61,6 +61,7 @@ "repository": "t3", "repository_updated": "Mon, 28 Jun 2021 14:42:44 GMT", "certification_level": 2, + "rootfull": false, "updates": [ { "id": "dokuwiki2", @@ -109,6 +110,10 @@ "type": "integer", "description": "The higher, the better: 0=unknown certification, 5=max" }, + "rootfull": { + "type": "boolean", + "description": "True if the application gains full OS privileges when installed" + }, "description": { "type": "object", "description": "A map of language codes (eg. en, it) with the translated description" @@ -245,8 +250,7 @@ "node", "digest", "source", - "version", - "update" + "version" ] } }, @@ -294,6 +298,7 @@ "docs", "source", "certification_level", + "rootfull", "versions", "installed", "updates" From 587482409b02b1e6a6ac497819636370cfc68997 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Jul 2024 16:46:32 +0200 Subject: [PATCH 08/49] docs. Add attributes to metadata.json example --- docs/modules/metadata.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/modules/metadata.md b/docs/modules/metadata.md index 18cf45985..2bb03fb7d 100644 --- a/docs/modules/metadata.md +++ b/docs/modules/metadata.md @@ -22,11 +22,12 @@ without `id`, `logo` and `versions` fields. Example of `metadata.json`: ```json { - "name": "kickstart", + "name": "Kickstart", + "upstream_name": "Kickstart 12", "description": { "en": "My kickstart module" }, - "categories": [], + "categories": ["somecategory"], "authors": [ { "name": "Name Surname", @@ -34,6 +35,7 @@ Example of `metadata.json`: } ], "docs": { + "terms_url": "https://docs.kickstart.com/terms/", "documentation_url": "https://docs.kickstart.com/", "bug_url": "https://github.com/NethServer/dev", "code_url": "https://github.com/author/ns8-kickstart" From 7a20080fbdb8f067b3ce288376867092b9cc766e Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Jul 2024 18:00:34 +0200 Subject: [PATCH 09/49] list-modules. Add install_destinations attribute --- .../cluster/actions/list-modules/50read | 33 +++++++++- .../actions/list-modules/validate-output.json | 60 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read index c0f8f5f02..306cfba01 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read @@ -24,6 +24,7 @@ import sys import json import agent import cluster.modules +import copy def get_module(source, modules): ret = [] @@ -37,14 +38,42 @@ installed = cluster.modules.list_installed(rdb, skip_core_modules = True) available = cluster.modules.list_available(rdb, skip_core_modules = True) updates = cluster.modules.list_updates(rdb, skip_core_modules = True) +install_destinations = [] +for node_id in set(rdb.hvals("cluster/module_node")): + install_destinations.append({ + "node_id": int(node_id), + "node_ui_name": rdb.get(f"node/{node_id}/ui_name") or "", + "instances": 0, + "eligible": True, + "reject_reason": None, + }) +install_destinations.sort(key=lambda n: n["node_id"]) + +def calculate_node_install_destinations(module): + module_destinations = copy.deepcopy(install_destinations) + try: + max_per_node = int(module["versions"][0]["labels"]["org.nethserver.max-per-node"]) + except: + return module_destinations + for mdest in module_destinations: + count_instances = len(list(filter(lambda m: m["node"] == str(mdest["node_id"]), module["installed"]))) + mdest["instances"] = count_instances + if count_instances < max_per_node: + continue # node is eligible, nothing to do + mdest["eligible"] = False + mdest["reject_reason"] = { + "message": "max_per_node_limit", + "parameter": str(max_per_node), + } + return module_destinations + # Prepare variables for later use for a in available: a["updates"] = [] a["installed"] = [] - if a["source"] in installed.keys(): a["installed"] = installed[a["source"]] - a["updates"] = get_module(a["source"], updates) + a["install_destinations"] = calculate_node_install_destinations(a) json.dump(available, fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index a49fe0906..6e35fadf8 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -60,6 +60,15 @@ ], "repository": "t3", "repository_updated": "Mon, 28 Jun 2021 14:42:44 GMT", + "install_destinations": [ + { + "node_id": 1, + "node_ui_name": "Node One", + "instances": 2, + "eligible": true, + "reject_reason": null + } + ], "updates": [ { "id": "dokuwiki2", @@ -232,6 +241,57 @@ ] } }, + "install_destinations": { + "description": "Describe for each node of the cluster if the node is eligible or not to install a new module instance. If not, a reject reason is returned.", + "type": "array", + "items": { + "type": "object", + "required": [ + "node_id", + "node_ui_name", + "instances", + "eligible", + "reject_reason" + ], + "properties": { + "node_id": { + "type": "integer", + "description": "Node identifier" + }, + "node_ui_name": { + "type": "string", + "description": "Label of the node assigned by the admin" + }, + "instances": { + "type": "integer", + "description": "Number of module instances currently installed on the node" + }, + "eligible": { + "type": "boolean", + "description": "True if another instance of the module can be installed on the node" + }, + "reject_reason": { + "type": [ + "object", + "null" + ], + "descripton": "If it is an object, it tells why the node is not eligible to host a module instance", + "properties": { + "message": { + "type": "string" + }, + "paramter": { + "type": "string" + } + }, + "required": [ + "message", + "parameter" + ] + } + } + } + }, "installed": { "type": "array", "items": { From 7f41ae8ba727103ac5eee5aa717f08dd48da4da8 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Jul 2024 18:24:22 +0200 Subject: [PATCH 10/49] docs. Document label org.nethserver.max-per-node --- docs/modules/images.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/modules/images.md b/docs/modules/images.md index d3ccc12e3..01a0a7fb3 100644 --- a/docs/modules/images.md +++ b/docs/modules/images.md @@ -55,6 +55,7 @@ Module images can use a list of well-known labels to configure the system: - `no_data_backup`: if present, the modules will need no data backup - `rootless`: if present, the module is rootless (calculated from `org.nethserver.rootfull` label) - `rootfull`: if present, the module is rootfull (calculated from `org.nethserver.rootfull` label) +- `org.nethserver.max-per-node`: maximum number of module instances installed on the same node Labels are set by `build-images.sh`, when the images are built. From 46f8f908350775f9906546093599b831ed25005d Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Mon, 8 Jul 2024 18:12:54 +0200 Subject: [PATCH 11/49] Refactor of testing version filtering --- .../usr/local/agent/pypkg/cluster/modules.py | 170 ++++++++---------- 1 file changed, 79 insertions(+), 91 deletions(-) diff --git a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py index 60a31798c..018e6c70a 100644 --- a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py +++ b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py @@ -32,6 +32,14 @@ import subprocess import datetime +_repo_testing_cache = {} +def _repo_has_testing_flag(rdb, repo_name): + """Retrieve from Redis the testing flag of repo_name one time, then read it from a cache.""" + global _repo_testing_cache + if repo_name not in _repo_testing_cache: + _repo_testing_cache[repo_name] = rdb.hget(f'cluster/repository/{repo_name}', 'testing') == "1" + return _repo_testing_cache[repo_name] + def _urljoin(base_path, *args): '''replace urllib.parse.joinurl because it doesn't handle multiple parameters ''' @@ -51,76 +59,43 @@ def _get_downloaded_logos(): logos[os.path.basename(app)] = logo return logos -def _calc_certification_level(repository_authority, package, hsubscription=None): - if repository_authority in ["distfeed.nethserver.org", "subscription.nethserver.com"]: +def _calc_certification_level(package, has_subscription=False): + if package['repository_authority'] in ["distfeed.nethserver.org", "subscription.nethserver.com"]: certification_level = 3 - elif repository_authority == "forge.nethserver.org": + elif package['repository_authority'] == "forge.nethserver.org": certification_level = 2 else: certification_level = 1 # If we trust the repo metadata, elevate up to level 5 if certification_level == 3: if package["source"].startswith("ghcr.io/nethserver/") or package["source"].startswith("ghcr.io/nethesis/"): - certification_level = 4 if hsubscription is None else 5 - + certification_level = 5 if has_subscription else 4 return certification_level -def _parse_repository_metadata(repository_name, repository_url, repository_updated, repodata, skip_core_modules=False, skip_testing_versions=False, hsubscription=None): - modules = [] - - try: - repodata = json.loads(repodata) - except: - return modules - +def _parse_repository_metadata(repository_name, repository_url, repository_updated, metadata): try: - repository_authority = urllib.parse.urlparse(repository_url).hostname + repodata = json.loads(metadata) except Exception as ex: - repository_authority = None - print(agent.SD_WARNING + f"Unable to parse repository {repository_name} URL: {repository_url}", ex, file=sys.stderr) - - def ignore_testing(version): - if skip_testing_versions and version["testing"] is True: - return False - else: - return True - + print(agent.SD_WARNING + f"Unable to parse metadata of repository {repository_name} at {repository_url}:", ex, file=sys.stderr) + return [] + modules = [] for package in repodata: - # Skip core modules if flag is enabled - if skip_core_modules and package['versions']: - version = package['versions'][0] - if 'org.nethserver.flags' in version['labels'] and 'core_module' in version['labels']['org.nethserver.flags']: - continue - package["repository"] = repository_name package["repository_updated"] = repository_updated - package["certification_level"] = _calc_certification_level(repository_authority, package, hsubscription) - + try: + package['repository_authority'] = urllib.parse.urlparse(repository_url).hostname + except Exception as ex: + package['repository_authority'] = "" + print(agent.SD_WARNING + f"Unable to parse repository {repository_name} URL: {repository_url}", ex, file=sys.stderr) # Set absolute path for logo if package["logo"]: package["logo"] = _urljoin(repository_url, package["id"], package["logo"]) - # Set absolute path for screenshots screenshots = [] for s in package["screenshots"]: screenshots.append(_urljoin(repository_url, package["id"], s)) package["screenshots"] = screenshots - - # Filter - package["versions"] = list(filter(ignore_testing, package["versions"])) - - if len(package["versions"]) > 0: - try: - package_is_rootfull = package["versions"][0]["labels"]["org.nethserver.rootfull"] == "1" - except: - package_is_rootfull = False - # Ignore untrusted rootfull application, if a subscription is active - if hsubscription and package_is_rootfull and package["certification_level"] < 3: - print(agent.SD_WARNING + f"Ignoring image of rootfull application {package['source']}: certification_level {package['certification_level']} is too low", file=sys.stderr) - continue # skip package - package['rootfull'] = package_is_rootfull - modules.append(package) - + modules.append(package) return modules def _get_http_session(): @@ -137,35 +112,32 @@ def _get_http_session(): osession.mount('https://', requests.adapters.HTTPAdapter(max_retries=oretries)) return osession -def _list_repository_modules(rdb, repository_name, repository_url, skip_core_modules = False, skip_testing_versions=False): - key = f'cluster/repository_cache/{repository_name}' - cache = rdb.hgetall(key) - hsubscription = rdb.hgetall("cluster/subscription") or None - if cache: - return _parse_repository_metadata(repository_name, repository_url, cache["updated"], cache["data"], skip_core_modules, skip_testing_versions, hsubscription=hsubscription) - - url = _urljoin(repository_url, "repodata.json") - try: - with _get_http_session() as osession: - if hsubscription and url.startswith("https://subscription.nethserver.com/"): - # Send system_id for HTTP Basic authentication - osession.auth = (hsubscription["system_id"], hashlib.sha256(hsubscription["auth_token"].encode()).hexdigest()) - resp = osession.get(url) - repodata = resp.text - updated = resp.headers.get('Last-Modified', "") - except Exception as ex: - print(f"Fetching {url}:", ex, file=sys.stderr) - # If repository is not accessible or invalid, just return an empty array - return [] - - modules = _parse_repository_metadata(repository_name, repository_url, updated, repodata, skip_core_modules, skip_testing_versions, hsubscription=hsubscription) +def _list_repository_modules(rdb, repository_name, repository_url): + cache_key = f'cluster/repository_cache/{repository_name}' + hcache = rdb.hgetall(cache_key) + if not hcache: + url = _urljoin(repository_url, "repodata.json") + hsubscription = rdb.hgetall("cluster/subscription") or None + try: + with _get_http_session() as osession: + if hsubscription and url.startswith("https://subscription.nethserver.com/"): + # Send system_id for HTTP Basic authentication + osession.auth = (hsubscription["system_id"], hashlib.sha256(hsubscription["auth_token"].encode()).hexdigest()) + resp = osession.get(url) + repodata_raw = resp.text + updated = resp.headers.get('Last-Modified', "") + except Exception as ex: + print(f"Fetching {url}:", ex, file=sys.stderr) + # If repository is not accessible or invalid, just return an empty array + return [] + hcache = {"data": repodata_raw, "updated": updated} + modules = _parse_repository_metadata(repository_name, repository_url, hcache['updated'], hcache['data']) # Save inside the cache if data is valid if modules: # Save also repodata file date - rdb.hset(key, mapping={"data": repodata, "updated": updated}) + rdb.hset(cache_key, mapping=hcache) # Set cache expiration to 3600 seconds - rdb.expire(key, 3600) - + rdb.expire(cache_key, 3600) return modules class LatestModuleLookupError(Exception): @@ -259,6 +231,7 @@ def _fetch_metadata_json(module_id, image_name): ometadata.setdefault("screenshots", []) ometadata.setdefault("repository", "__local__") ometadata.setdefault("repository_updated", repository_updated_timestamp) + ometadata.setdefault("repository_authority", "__local__") ometadata["certification_level"] = 0 ometadata["rootfull"] = False try: @@ -268,12 +241,36 @@ def _fetch_metadata_json(module_id, image_name): print(agent.SD_INFO + "_fetch_metadata_json/glob:", ex, file=sys.stderr) return ometadata -def list_available(rdb, skip_core_modules = False): +def list_available(rdb, skip_core_modules=False): """Iterate over enabled repositories and return available modules respecting the repository priority.""" - modules = _get_available_modules(rdb, skip_core_modules) - return list(modules.values()) + hsubscription = rdb.hgetall("cluster/subscription") or None + modules = [] + for omod in _get_available_modules(rdb).values(): + if not _repo_has_testing_flag(rdb, omod["repository"]): + # Ignore testing releases for new installations: + omod["versions"] = list(filter(lambda v: v["testing"] is False, omod["versions"])) + if not omod["versions"]: + continue # Ignore modules with no versions + omod["certification_level"] = _calc_certification_level(omod, bool(hsubscription)) + try: + if skip_core_modules and 'core_module' in omod["versions"][0]['labels']['org.nethserver.flags']: + continue # core modules are ignored + except: + pass + try: + package_is_rootfull = omod["versions"][0]["labels"]["org.nethserver.rootfull"] == "1" + except: + package_is_rootfull = False + # Ignore untrusted rootfull application, if a subscription is active + if hsubscription and package_is_rootfull and omod["certification_level"] < 3: + print(agent.SD_WARNING + f"Ignoring image of rootfull application {omod['source']}: certification_level {omod['certification_level']} is too low", file=sys.stderr) + continue # skip package + else: + omod['rootfull'] = package_is_rootfull + modules.append(omod) + return modules -def _get_available_modules(rdb, skip_core_modules = False): +def _get_available_modules(rdb): modules = {} repositories = [] # List all modules from enabled repositories @@ -286,23 +283,21 @@ def _get_available_modules(rdb, skip_core_modules = False): # Skip non-enabled repositories if repo.get("status", "0") != "1": continue - skip_testing_versions = repo.get("testing", "0") != "1" - for rmod in _list_repository_modules(rdb, nrepo, repo["url"], skip_core_modules, skip_testing_versions): + for rmod in _list_repository_modules(rdb, nrepo, repo["url"]): if rmod["source"] in modules: continue # skip duplicated images from lower priority modules modules[rmod["source"]] = rmod rmod['versions'].sort(key=lambda v: _parse_version_object(v["tag"]), reverse=True) # Integrate the available set with instances that do not belong to any # repository. They can be found in the "installed" dict: - for module_source, module_instances in list_installed(rdb, skip_core_modules).items(): + for module_source, module_instances in list_installed(rdb).items(): if module_source in modules: continue _, image_name = module_source.rsplit("/", 1) vmetadata = _fetch_metadata_json(module_instances[0]['id'], image_name) vmetadata["versions"] = list(_synthesize_module_version(oinst) for oinst in module_instances) - vmetadata["installed"] = module_instances + vmetadata["versions"].sort(key=lambda v: _parse_version_object(v["tag"]), reverse=True) vmetadata["source"] = module_source - vmetadata["updates"] = [] vmetadata["id"] = image_name modules[module_source] = vmetadata return modules @@ -350,17 +345,10 @@ def list_installed_core(rdb): return installed - def list_updates(rdb, skip_core_modules=False, with_testing_update=False): updates = [] installed_modules = list_installed(rdb, skip_core_modules) - available_modules = _get_available_modules(rdb, skip_core_modules) - - repo_testing_cache = {} - def repo_has_testing_flag(repo_name): - if repo_name not in repo_testing_cache: - repo_testing_cache[repo_name] = rdb.hget(f'cluster/repository/{repo_name}', 'testing') == "1" - return repo_testing_cache[repo_name] + available_modules = _get_available_modules(rdb) flat_instance_list = list(mi for module_instances in installed_modules.values() for mi in module_instances) for instance in flat_instance_list: @@ -384,7 +372,7 @@ def repo_has_testing_flag(repo_name): if available_version <= current_version: continue # ignore tags that do not update the current one if update_candidate is None and ( - repo_has_testing_flag(repository_name) + _repo_has_testing_flag(rdb, repository_name) or not available_version.prerelease ): update_candidate = available_version From db2652875d468c222716677421a77b587a44c72c Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Jul 2024 18:24:22 +0200 Subject: [PATCH 12/49] docs. Document label org.nethserver.max-per-node --- docs/modules/images.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/modules/images.md b/docs/modules/images.md index d3ccc12e3..01a0a7fb3 100644 --- a/docs/modules/images.md +++ b/docs/modules/images.md @@ -55,6 +55,7 @@ Module images can use a list of well-known labels to configure the system: - `no_data_backup`: if present, the modules will need no data backup - `rootless`: if present, the module is rootless (calculated from `org.nethserver.rootfull` label) - `rootfull`: if present, the module is rootfull (calculated from `org.nethserver.rootfull` label) +- `org.nethserver.max-per-node`: maximum number of module instances installed on the same node Labels are set by `build-images.sh`, when the images are built. From af0c8f2c73585ecb124e8f4f9db3b569fb8c2a91 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Tue, 9 Jul 2024 12:50:54 +0200 Subject: [PATCH 13/49] list-modules. Fix install_destinations attribute value The node_ui_name attribute is not used and can be removed --- .../var/lib/nethserver/cluster/actions/list-modules/50read | 1 - .../cluster/actions/list-modules/validate-output.json | 6 ------ 2 files changed, 7 deletions(-) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read index 306cfba01..71affc14b 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read @@ -42,7 +42,6 @@ install_destinations = [] for node_id in set(rdb.hvals("cluster/module_node")): install_destinations.append({ "node_id": int(node_id), - "node_ui_name": rdb.get(f"node/{node_id}/ui_name") or "", "instances": 0, "eligible": True, "reject_reason": None, diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index 6e35fadf8..a18a4476c 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -63,7 +63,6 @@ "install_destinations": [ { "node_id": 1, - "node_ui_name": "Node One", "instances": 2, "eligible": true, "reject_reason": null @@ -248,7 +247,6 @@ "type": "object", "required": [ "node_id", - "node_ui_name", "instances", "eligible", "reject_reason" @@ -258,10 +256,6 @@ "type": "integer", "description": "Node identifier" }, - "node_ui_name": { - "type": "string", - "description": "Label of the node assigned by the admin" - }, "instances": { "type": "integer", "description": "Number of module instances currently installed on the node" From e329818a0a5e0be1406bb82f536dde3c6251a12a Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Tue, 9 Jul 2024 12:58:38 +0200 Subject: [PATCH 14/49] Fix JSON schema syntax Attribute definitions must be moved under the "properties" object, otherwise they are ignored. --- .../actions/list-modules/validate-output.json | 249 +++++++++--------- 1 file changed, 125 insertions(+), 124 deletions(-) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index 8194a79e3..1d0745f2e 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -193,6 +193,130 @@ "type": "string", "description": "Date and time of last modification to remote repodata" }, + "updates": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique name of a module instance" + }, + "node": { + "type": "string", + "description": "Id of the node where the instance is running" + }, + "digest": { + "type": "string", + "description": "Image digest" + }, + "source": { + "type": "string", + "description": "The URL of the container image registry" + }, + "version": { + "type": "string", + "description": "A valid semantic version extracted from image tag" + }, + "testing_update": { + "type": "string", + "description": "A semver prerelease tag, greater than 'version' field" + }, + "update": { + "type": "string", + "description": "A semver tag greater than 'version' field" + } + }, + "required": [ + "id", + "node", + "digest", + "source", + "version" + ] + } + }, + "install_destinations": { + "description": "Describe for each node of the cluster if the node is eligible or not to install a new module instance. If not, a reject reason is returned.", + "type": "array", + "items": { + "type": "object", + "required": [ + "node_id", + "instances", + "eligible", + "reject_reason" + ], + "properties": { + "node_id": { + "type": "integer", + "description": "Node identifier" + }, + "instances": { + "type": "integer", + "description": "Number of module instances currently installed on the node" + }, + "eligible": { + "type": "boolean", + "description": "True if another instance of the module can be installed on the node" + }, + "reject_reason": { + "type": [ + "object", + "null" + ], + "descripton": "If it is an object, it tells why the node is not eligible to host a module instance", + "properties": { + "message": { + "type": "string" + }, + "paramter": { + "type": "string" + } + }, + "required": [ + "message", + "parameter" + ] + } + } + } + }, + "installed": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique name of a module instance" + }, + "node": { + "type": "string", + "description": "Id of the node where the instance is running" + }, + "digest": { + "type": "string", + "description": "Image digest" + }, + "source": { + "type": "string", + "description": "The URL of the container image registry" + }, + "version": { + "type": "string", + "description": "A valid semantic version extracted from image tag" + } + }, + "required": [ + "id", + "node", + "digest", + "source", + "version" + ] + } + }, "versions": { "type": "array", "items": { @@ -219,130 +343,6 @@ } } }, - "updates": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "Unique name of a module instance" - }, - "node": { - "type": "string", - "description": "Id of the node where the instance is running" - }, - "digest": { - "type": "string", - "description": "Image digest" - }, - "source": { - "type": "string", - "description": "The URL of the container image registry" - }, - "version": { - "type": "string", - "description": "A valid semantic version extracted from image tag" - }, - "testing_update": { - "type": "string", - "description": "A semver prerelease tag, greater than 'version' field" - }, - "update": { - "type": "string", - "description": "A semver tag greater than 'version' field" - } - }, - "required": [ - "id", - "node", - "digest", - "source", - "version" - ] - } - }, - "install_destinations": { - "description": "Describe for each node of the cluster if the node is eligible or not to install a new module instance. If not, a reject reason is returned.", - "type": "array", - "items": { - "type": "object", - "required": [ - "node_id", - "instances", - "eligible", - "reject_reason" - ], - "properties": { - "node_id": { - "type": "integer", - "description": "Node identifier" - }, - "instances": { - "type": "integer", - "description": "Number of module instances currently installed on the node" - }, - "eligible": { - "type": "boolean", - "description": "True if another instance of the module can be installed on the node" - }, - "reject_reason": { - "type": [ - "object", - "null" - ], - "descripton": "If it is an object, it tells why the node is not eligible to host a module instance", - "properties": { - "message": { - "type": "string" - }, - "paramter": { - "type": "string" - } - }, - "required": [ - "message", - "parameter" - ] - } - } - } - }, - "installed": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "Unique name of a module instance" - }, - "node": { - "type": "string", - "description": "Id of the node where the instance is running" - }, - "digest": { - "type": "string", - "description": "Image digest" - }, - "source": { - "type": "string", - "description": "The URL of the container image registry" - }, - "version": { - "type": "string", - "description": "A valid semantic version extracted from image tag" - } - }, - "required": [ - "id", - "node", - "digest", - "source", - "version" - ] - } - }, "required": [ "name", "description", @@ -354,6 +354,7 @@ "certification_level", "rootfull", "versions", + "install_destinations", "installed", "updates" ] From f9ef2b9c4dff67329acc4ed4300fc8a2593819d0 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Tue, 9 Jul 2024 16:22:48 +0200 Subject: [PATCH 15/49] api. Fix repository management API (#663) - Make the "testing" parameter optional. - Fix the output type: boolean is redundant. Refs NethServer/dev#6956 --- .../nethserver/cluster/actions/add-repository/50update | 4 +--- .../cluster/actions/add-repository/validate-input.json | 4 ++-- .../actions/add-repository/validate-output.json | 10 ---------- .../cluster/actions/alter-repository/50update | 4 +--- .../actions/alter-repository/validate-input.json | 3 +-- .../actions/alter-repository/validate-output.json | 10 ---------- .../cluster/actions/remove-repository/50update | 2 -- .../actions/remove-repository/validate-output.json | 10 ---------- 8 files changed, 5 insertions(+), 42 deletions(-) delete mode 100644 core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-output.json delete mode 100644 core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-output.json delete mode 100644 core/imageroot/var/lib/nethserver/cluster/actions/remove-repository/validate-output.json diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/50update b/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/50update index 5a6fe08a1..d634cb97e 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/50update +++ b/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/50update @@ -32,11 +32,9 @@ name = request['name'] url = request['url'] # convert boolean to integers status = 1 if request['status'] else 0 -testing = 1 if request['testing'] else 0 +testing = 1 if request.get('testing') else 0 # Add the repository and enable it. # Access to testing packages is disabled by default. if not rdb.hset(f'cluster/repository/{name}', mapping={'url': url, 'status': status, 'testing': testing}): sys.exit(1) - -json.dump(True, fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-input.json b/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-input.json index 2530c23fa..1bd0d6c16 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-input.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-input.json @@ -6,6 +6,7 @@ "examples": [ { "name": "repository1", + "status": true, "url": "https://repository1.nethserver.org/" } ], @@ -20,7 +21,7 @@ "type": "string" }, "testing": { - "description": "Enable or disable access to testing images", + "description": "Use testing releases to install new instances and update existing ones.", "type": "boolean" }, "status": { @@ -31,7 +32,6 @@ "required": [ "name", "url", - "testing", "status" ] } diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-output.json deleted file mode 100644 index 7b48931cd..000000000 --- a/core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-output.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "add-repository output", - "$id": "http://schema.nethserver.org/cluster/add-repository-output.json", - "description": "Output schema of the add-repository action", - "examples": [ - true - ], - "type": "boolean" -} diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/50update b/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/50update index 305c51ee3..670f6749e 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/50update +++ b/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/50update @@ -31,11 +31,9 @@ rdb = agent.redis_connect(privileged=True) name = request['name'] # convert boolean to integers status = 1 if request['status'] else 0 -testing = 1 if request['testing'] else 0 +testing = 1 if request.get('testing') else 0 # Add the repository and enable it. # Access to testing packages is disabled by default rdb.hset(f'cluster/repository/{name}', mapping={'status': status, 'testing': testing}) - -json.dump(True, fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-input.json b/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-input.json index 1e985e6a1..d94f75341 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-input.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-input.json @@ -17,7 +17,7 @@ "type": "string" }, "testing": { - "description": "Enable or disable access to testing images", + "description": "Use testing releases to install new instances and update existing ones.", "type": "boolean" }, "status": { @@ -27,7 +27,6 @@ }, "required": [ "name", - "testing", "status" ] } diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-output.json deleted file mode 100644 index bd6efa0f9..000000000 --- a/core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-output.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "alter-repository output", - "$id": "http://schema.nethserver.org/cluster/alter-repository-output.json", - "description": "Output schema of the alter-repository action", - "examples": [ - true - ], - "type": "boolean" -} diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/remove-repository/50update b/core/imageroot/var/lib/nethserver/cluster/actions/remove-repository/50update index 8a2ec832e..fe0820cd9 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/remove-repository/50update +++ b/core/imageroot/var/lib/nethserver/cluster/actions/remove-repository/50update @@ -32,5 +32,3 @@ name = request['name'] if not rdb.delete(f'cluster/repository/{name}'): sys.exit(1) - -json.dump(True, fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/remove-repository/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/remove-repository/validate-output.json deleted file mode 100644 index 020657b53..000000000 --- a/core/imageroot/var/lib/nethserver/cluster/actions/remove-repository/validate-output.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "remove-repository output", - "$id": "http://schema.nethserver.org/cluster/remove-repository-output.json", - "description": "Output schema of the remove-repository action", - "examples": [ - true - ], - "type": "boolean" -} From bea3dea20bf3c4e6fda8bc9f0a1a8fc43147a6bc Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Wed, 10 Jul 2024 09:24:26 +0200 Subject: [PATCH 16/49] fix: software center improvements ui (#660) --- core/ui/package.json | 2 +- core/ui/public/i18n/en/translation.json | 46 +++- core/ui/src/components/nodes/NodeSelector.vue | 17 +- .../software-center/AppInfoModal.vue | 44 +++- .../components/software-center/AppList.vue | 38 ++-- .../CertificationLevelBadge.vue | 94 +++++++++ .../software-center/CloneOrMoveAppModal.vue | 8 +- .../software-center/InstallAppModal.vue | 98 ++++++++- .../software-center/UpdateAppModal.vue | 53 ++++- core/ui/src/stories/NsComboBox.stories.js | 12 -- .../stories/NsDangerDeleteModal.stories.js | 2 +- core/ui/src/styles/_core.scss | 6 + core/ui/src/views/SoftwareCenter.vue | 20 +- .../src/views/SoftwareCenterAppInstances.vue | 96 ++++++--- .../settings/SettingsSoftwareRepositories.vue | 196 +++++++----------- .../views/settings/SettingsSubscription.vue | 97 +++++++-- core/ui/yarn.lock | 16 +- 17 files changed, 608 insertions(+), 237 deletions(-) create mode 100644 core/ui/src/components/software-center/CertificationLevelBadge.vue diff --git a/core/ui/package.json b/core/ui/package.json index 57e9d2598..4e7c28aeb 100644 --- a/core/ui/package.json +++ b/core/ui/package.json @@ -16,7 +16,7 @@ "@carbon/icons-vue": "^10.37.0", "@carbon/themes": "^10.34.0", "@carbon/vue": "^2.40.0", - "@nethserver/ns8-ui-lib": "^1.0.1", + "@nethserver/ns8-ui-lib": "^1.0.2", "await-to-js": "^3.0.0", "axios": "^0.21.2", "carbon-components": "^10.41.0", diff --git a/core/ui/public/i18n/en/translation.json b/core/ui/public/i18n/en/translation.json index ccfdedf99..f4d697471 100644 --- a/core/ui/public/i18n/en/translation.json +++ b/core/ui/public/i18n/en/translation.json @@ -84,7 +84,9 @@ "click_here_to_upload": "Click here to upload", "not_active": "Inactive", "active": "Active", - "configure": "Configure" + "configure": "Configure", + "terms_and_conditions": "Terms and Conditions", + "terms_required": "Please read and agree to @:common.terms_and_conditions" }, "error": { "error": "Error", @@ -632,7 +634,8 @@ "unknown_token": "Unknown token", "must_be_32_chars_but_less_than_128": "Must be between 32 and 128 characters", "os_not_supported": "Operating system is not supported", - "subscription_cannot_be_enabled": "Subscription is not available" + "subscription_cannot_be_enabled": "Subscription is not available", + "agree_terms_before_register": "I have read and agree to subscription {terms}." }, "settings_sw_repositories": { "title": "Software repositories", @@ -647,7 +650,17 @@ "repository_same_name_already_exists": "A repository with this name already exists", "repository_same_url_already_exists": "A repository with this URL already exists", "repository_not_accessible": "Repository not accessible", - "repository_is_going_to_be_deleted": "Repository {object} is going to be deleted..." + "repository_is_going_to_be_deleted": "Repository {object} is going to be deleted...", + "enable_repository": "Enable repository {name}", + "disable_repository": "Disable repository {name}", + "create_repository_warning_title": "Security risks", + "create_repository_warning_description_1": "Adding a third-party software repository can pose significant risks:", + "create_repository_warning_li_1": "Potential security threats", + "create_repository_warning_li_2": "Possible system instability", + "create_repository_warning_li_3": "Privacy concerns", + "create_repository_warning_li_4": "Uncertain update and support status", + "create_repository_warning_li_5": "Granting system access to unknown sources", + "create_repository_warning_description_2": "Only proceed if you trust the repository source and understand these risks. Research the repository's reputation before adding it to the cluster." }, "settings_http_routes": { "title": "HTTP routes", @@ -798,7 +811,7 @@ "software_updates": "Updates", "you_have_updates": "You have {numUpdates} app to update | You have {numUpdates} apps to update", "testing_warning_title": "Testing repositories enabled", - "testing_warning_description": "Some software repositories include non-stable versions of applications", + "testing_warning_description": "{repos} software repository includes non-stable versions of applications | The following software repositories include non-stable versions of applications: {repos}", "testing_warning_action_label": "Go to Software repositories", "all": "All", "installed": "Installed", @@ -833,8 +846,8 @@ "source_package": "Source package", "authors": "Author | Authors", "app_installation": "Install {app}", - "choose_node_for_installation": "Select installation node for {app} {version}", - "about_to_install_app": "{app} {version} will be installed on {node}", + "choose_node_for_installation": "Select installation node for {app} {version}:", + "about_to_install_app": "{app} {version} will be installed on {node}.", "installing_on_node": "Installing on {node}", "instance_installed_on_node": "{module_id} installed on {node}", "app_instances": "{app} instances", @@ -868,7 +881,9 @@ "app_now_renamed": "{instance} (now {module_id})", "uninstall_app": "Uninstall {name}? App data will be deleted too. This action is NOT reversible", "update_app": "Update {app}", - "about_to_update_app": "{app} {version} will be updated to {newVersion}", + "about_to_update_app": "{app} {version} will be updated to {newVersion}.", + "about_to_update_app_to_testing_version": "{app} {version} will be updated to testing version {newVersion}.", + "update_to_testing_version_warning": "Testing versions may contain bugs and are not recommended for production use. Update only if you are aware of the risks.", "updating_to_version": "Updating to version {version}", "core_app_name": "{productName} core", "core_app_description": "Collection of {productName} core apps", @@ -901,7 +916,22 @@ "communication": "Communication", "security": "Security" }, - "images_label": "Images" + "images_label": "Images", + "rootfull_app_warning_title": "Administrative privileges", + "rootfull_app_warning_description": "After installation {appName} will have full access to the cluster: this can pose significant security risks.", + "agree_terms_before_install": "I have read and agree to {appName} {terms}.", + "level_level": "Level {level}", + "certification": "Certification", + "level_0_description": "Unknown level", + "level_1_description": "App developed by third-parties, not certified", + "level_2_description": "App developed by third-parties, certified by NethServer 8 community", + "level_3_description": "App developed by third-parties, certified by Nethesis", + "level_4_description": "App developed and certified by Nethesis", + "level_5_description": "App developed, certified and supported by Nethesis", + "update_to_stable_version": "Update to stable version", + "update_to_testing_version": "Update to testing version", + "num_instances_installed": "{num} instance installed | {num} instances installed", + "reason_max_per_node_limit": "Limit of {param} instance reached | Limit of {param} instances reached" }, "system_logs": { "title": "System logs", diff --git a/core/ui/src/components/nodes/NodeSelector.vue b/core/ui/src/components/nodes/NodeSelector.vue index c880058f8..b7294207c 100644 --- a/core/ui/src/components/nodes/NodeSelector.vue +++ b/core/ui/src/components/nodes/NodeSelector.vue @@ -23,18 +23,13 @@ :disabled="disabledNodes.includes(node.id)" >
- {{ - node.ui_name ? node.ui_name : $t("common.node") + " " + node.id - }} + + {{ node.ui_name }} ({{ $t("common.node") }} {{ node.id }}) + + {{ $t("common.node") }} {{ node.id }}
-
- {{ $t("common.node") }} {{ node.id }} -
-
- {{ extraInfoLabel }} +
+
diff --git a/core/ui/src/components/software-center/AppInfoModal.vue b/core/ui/src/components/software-center/AppInfoModal.vue index 5c1e27fba..e47b08d9f 100644 --- a/core/ui/src/components/software-center/AppInfoModal.vue +++ b/core/ui/src/components/software-center/AppInfoModal.vue @@ -26,7 +26,7 @@ />
-

{{ app.name }}

+

{{ app.name }}

@@ -38,6 +38,26 @@
{{ getApplicationDescription(app) }}
+ + +
+ + {{ $t("software_center.certification") }} + + + + +
{{ @@ -54,7 +74,16 @@ {{ $t("software_center.latest_version") }} - {{ app.versions.length ? app.versions[0].tag : "-" }} + {{ app.versions.length ? app.versions[0].tag : "latest" }} + ({{ app.upstream_name }}) + +
+
+
+
+ {{ $t("software_center.repository") }} + + {{ app.repository || "-" }}
@@ -129,6 +158,14 @@ {{ $t("software_center.source_code") }} + @@ -140,10 +177,11 @@ + + diff --git a/core/ui/src/components/software-center/CloneOrMoveAppModal.vue b/core/ui/src/components/software-center/CloneOrMoveAppModal.vue index 7a99d8f2a..74feaf81b 100644 --- a/core/ui/src/components/software-center/CloneOrMoveAppModal.vue +++ b/core/ui/src/components/software-center/CloneOrMoveAppModal.vue @@ -31,10 +31,12 @@ + > + +