Skip to content

Commit

Permalink
Merge branch 'feat-sci'
Browse files Browse the repository at this point in the history 
Refs NethServer/dev#6970 NethServer/dev#6955 NethServer/dev#6956
  • Loading branch information
@DavidePrincipi
DavidePrincipi committed Sep 26, 2024
2 parents ce88ec6 + 9a3a365 commit df1fc0f
Show file tree
Hide file tree
Showing 48 changed files with 1,852 additions and 909 deletions.
476 changes: 310 additions & 166 deletions core/imageroot/usr/local/agent/pypkg/cluster/modules.py
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ with subprocess.Popen(['podman', 'image', 'inspect', image_url], stdout=subproce
inspect_image_repodigest = inspect[0]['RepoDigests'][0]

if 'org.nethserver.rootfull' in inspect_labels:
is_rootfull = int(inspect_labels['org.nethserver.rootfull']) == 1
is_rootfull = inspect_labels['org.nethserver.rootfull'] == "1"
else:
is_rootfull = False

Expand Down
4 changes: 1 addition & 3 deletions core/imageroot/var/lib/nethserver/cluster/actions/add-repository/50update
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,9 @@ name = request['name']
url = request['url']
# convert boolean to integers
status = 1 if request['status'] else 0
testing = 1 if request['testing'] else 0
testing = 1 if request.get('testing') else 0

# Add the repository and enable it.
# Access to testing packages is disabled by default.
if not rdb.hset(f'cluster/repository/{name}', mapping={'url': url, 'status': status, 'testing': testing}):
sys.exit(1)

json.dump(True, fp=sys.stdout)
4 changes: 2 additions & 2 deletions core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-input.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"examples": [
{
"name": "repository1",
"status": true,
"url": "https://repository1.nethserver.org/"
}
],
Expand All @@ -20,7 +21,7 @@
"type": "string"
},
"testing": {
"description": "Enable or disable access to testing images",
"description": "Use testing releases to install new instances and update existing ones.",
"type": "boolean"
},
"status": {
Expand All @@ -31,7 +32,6 @@
"required": [
"name",
"url",
"testing",
"status"
]
}
10 changes: 0 additions & 10 deletions core/imageroot/var/lib/nethserver/cluster/actions/add-repository/validate-output.json
View file

This file was deleted.

4 changes: 1 addition & 3 deletions core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/50update
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,9 @@ rdb = agent.redis_connect(privileged=True)
name = request['name']
# convert boolean to integers
status = 1 if request['status'] else 0
testing = 1 if request['testing'] else 0
testing = 1 if request.get('testing') else 0

# Add the repository and enable it.
# Access to testing packages is disabled by default

rdb.hset(f'cluster/repository/{name}', mapping={'status': status, 'testing': testing})

json.dump(True, fp=sys.stdout)
3 changes: 1 addition & 2 deletions core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-input.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"type": "string"
},
"testing": {
"description": "Enable or disable access to testing images",
"description": "Use testing releases to install new instances and update existing ones.",
"type": "boolean"
},
"status": {
Expand All @@ -27,7 +27,6 @@
},
"required": [
"name",
"testing",
"status"
]
}
10 changes: 0 additions & 10 deletions core/imageroot/var/lib/nethserver/cluster/actions/alter-repository/validate-output.json
View file

This file was deleted.

4 changes: 3 additions & 1 deletion core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import sys
import json
import requests, urllib3.util

terms_url = "https://docs.nethserver.org/projects/ns8/en/latest/subscription.html#terms-and-conditions"

def _get_http_session():
osession = requests.Session()
osession.timeout = 15 # Timout for HTTP connections
Expand Down Expand Up @@ -95,4 +97,4 @@ elif hsubscription["provider"] == "nsent":
elif hsubscription["provider"] == "nscom":
hsubscription.update(fetch_subscription_info_nscom(rdb, hsubscription))

json.dump({"subscription": hsubscription}, fp=sys.stdout)
json.dump({"subscription": hsubscription, "terms_url": terms_url}, fp=sys.stdout)
26 changes: 25 additions & 1 deletion core/imageroot/var/lib/nethserver/cluster/actions/list-core-modules/50read
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,29 @@ import cluster.modules
import agent

rdb = agent.redis_connect(privileged=True)
list_core_modules = cluster.modules.list_core_modules(rdb)
available = cluster.modules.list_available(rdb, skip_core_modules = False)

json.dump(cluster.modules.list_core_modules(rdb), fp=sys.stdout)
# Create a dictionary from available for quick lookup by name
dict_available = {item['name']: item for item in available}

# Update list_core_modules with data from dict_available
for item in list_core_modules:
if item['name'] in dict_available:
matched_item = dict_available[item['name']]
# each installed modules must get its information
for instance in item['instances']:
instance.update({
"description": matched_item["description"],
"logo": matched_item["logo"],
"certification_level": matched_item["certification_level"] ,
"repository": matched_item["repository"],
"versions": matched_item["versions"],
"screenshots": matched_item["screenshots"],
"categories": matched_item["categories"],
"authors": matched_item["authors"],
"docs": matched_item["docs"],
"source": matched_item["source"]
})

json.dump(list_core_modules, fp=sys.stdout)
58 changes: 40 additions & 18 deletions core/imageroot/var/lib/nethserver/cluster/actions/list-core-modules/validate-output.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,52 +11,64 @@
{
"id": "core",
"version": "updates_from_repo",
"update": ""
"update": "",
"node_id": "1",
"node_ui_name": "node1"
}
]
},
{
"name": "promtail",
"instances": [
{
"id": "promtail1",
"version": "latest",
"update": ""
"id": "promtail1",
"version": "latest",
"update": "",
"node_id": "1",
"node_ui_name": "node1"
}
]
},
{
"name": "traefik",
"instances": [
{
"id": "traefik1",
"version": "0.0.1",
"update": ""
"id": "traefik1",
"version": "0.0.1",
"update": "",
"node_id": "1",
"node_ui_name": "node1"
}
]
},
{
"name": "loki",
"instances": [
{
"id": "loki1",
"version": "latest",
"update": ""
"id": "loki1",
"version": "latest",
"update": "",
"node_id": "1",
"node_ui_name": "node1"
},
{
"id": "loki2",
"version": "0.0.1-alpha1",
"update": "0.0.1"
"id": "loki2",
"version": "0.0.1-alpha1",
"update": "0.0.1",
"node_id": "2",
"node_ui_name": "node2"
}
]
},
{
"name": "ldapproxy",
"instances": [
{
"id": "ldapproxy1",
"version": "latest",
"update": ""
"id": "ldapproxy1",
"version": "latest",
"update": "",
"node_id": "1",
"node_ui_name": "node1"
}
]
}
Expand Down Expand Up @@ -86,12 +98,22 @@
"update": {
"type": "string",
"description": "Available version update, can be empty"
}
},
"node_id": {
"type": "string",
"description": "the node ID where the module is installed"
},
"node_ui_name": {
"type": "string",
"description": "the node UI name where the module is installed"
}
},
"required": [
"id",
"version",
"update"
"update",
"node_id",
"node_ui_name"
]
}
}
Expand Down
53 changes: 50 additions & 3 deletions core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import sys
import json
import agent
import cluster.modules
import copy
import semver

def get_module(source, modules):
ret = []
Expand All @@ -35,16 +37,61 @@ def get_module(source, modules):
rdb = agent.redis_connect(privileged=True)
installed = cluster.modules.list_installed(rdb, skip_core_modules = True)
available = cluster.modules.list_available(rdb, skip_core_modules = True)
updates = cluster.modules.list_updates(rdb, skip_core_modules = True)
updates = cluster.modules.list_updates(rdb, skip_core_modules=True, with_testing_update=True)
node_core_versions = cluster.modules.get_node_core_versions(rdb)

install_destinations = []
for node_id in set(rdb.hvals("cluster/module_node")):
install_destinations.append({
"node_id": int(node_id),
"instances": 0,
"eligible": True,
"reject_reason": None,
})
install_destinations.sort(key=lambda n: n["node_id"])

def calculate_node_install_destinations(module):
global node_core_versions
module_destinations = copy.deepcopy(install_destinations)
# Parse labels of the array first element
try:
max_per_node = int(module["versions"][0]["labels"]["org.nethserver.max-per-node"])
except:
max_per_node = 9999
try:
min_core = semver.Version.parse(module["versions"][0]["labels"]["org.nethserver.min-core"])
except:
min_core = semver.Version(0,0,0)
# Find reject reasons in this loop:
for mdest in module_destinations:
# max-per-node label check:
count_instances = len(list(filter(lambda m: m["node"] == str(mdest["node_id"]), module["installed"])))
mdest["instances"] = count_instances
if count_instances >= max_per_node:
mdest["eligible"] = False
mdest["reject_reason"] = {
"message": "max_per_node_limit",
"parameter": str(max_per_node),
}
continue
# min-core label check:
snode_id = str(mdest["node_id"])
if snode_id in node_core_versions and node_core_versions[snode_id] < min_core:
mdest["eligible"] = False
mdest["reject_reason"] = {
"message": "min_core_requirement",
"parameter": str(min_core),
}
continue
return module_destinations

# Prepare variables for later use
for a in available:
a["updates"] = []
a["installed"] = []

if a["source"] in installed.keys():
a["installed"] = installed[a["source"]]

a["updates"] = get_module(a["source"], updates)
a["install_destinations"] = calculate_node_install_destinations(a)

json.dump(available, fp=sys.stdout)
Loading

0 comments on commit df1fc0f

Please sign in to comment.