Merge pull request #749 from NethServer/feat-7072

List backup repositories of a module

Refs NethServer/dev#7072

core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import sys
+import json
+import agent
+import asyncio
+import os
+import time
+from datetime import datetime, timezone
+
+rdb = agent.redis_connect(privileged=False)
+module_id = os.environ['MODULE_ID']
+module_uuid = os.environ['MODULE_UUID']
+module_ui_name = rdb.get(f'module/{module_id}/ui_name') or ""
+image_name = agent.get_image_name_from_url(os.environ["IMAGE_URL"])
+cluster_uuid = rdb.get("cluster/uuid") or ""
+odests = {}
+for krepo in rdb.scan_iter('cluster/backup_repository/*'):
+    dest_uuid = krepo.removeprefix('cluster/backup_repository/')
+    odests[dest_uuid] = rdb.hgetall(krepo)
+rdb.close()
+
+#
+# Fetch data from all backup destinations
+#
+
+async def read_destination_repo(dest_uuid, dest_path):
+    proc = await asyncio.create_subprocess_exec('rclone-wrapper', dest_uuid, 'lsjson', f'REMOTE_PATH/{dest_path}/config', stdout=asyncio.subprocess.PIPE)
+    # Return the first and only element of the expected JSON array
+    out, _ = await proc.communicate()
+    if out == b'[\n]\n' or not out:
+        data = {}
+    else:
+        try:
+            data = json.loads(out)[0]
+        except Exception as ex:
+            print(agent.SD_DEBUG + f"Ignored output from rclone-wrapper. Does the Restic repository configuration file, {dest_path}/config, exist in destination {dest_uuid}?", repr(ex), 'Data read:', out, file=sys.stderr)
+            data = {}
+    return data
+
+async def read_destination_meta(dest_uuid, dest_path):
+    proc = await asyncio.create_subprocess_exec('rclone-wrapper', dest_uuid, 'cat', f'REMOTE_PATH/{dest_path}.json', stdout=asyncio.subprocess.PIPE)
+    out, _ = await proc.communicate()
+    if out:
+        try:
+            data = json.loads(out)
+        except Exception as ex:
+            print(agent.SD_DEBUG + f"Ignored output from rclone-wrapper. Does {dest_path}.json file exist in destination {dest_uuid}?", repr(ex), 'Data read:', out, file=sys.stderr)
+            data = {}
+    else:
+        data = {}
+    return data
+
+
+async def get_destination_info(dest_uuid, odest):
+    global cluster_uuid, module_id, module_uuid, module_ui_name, image_name
+
+    dest_path = f"{image_name}/{module_uuid}"
+
+    async with asyncio.TaskGroup() as tg:
+        task_repo = tg.create_task(read_destination_repo(dest_uuid, dest_path))
+        task_meta = tg.create_task(read_destination_meta(dest_uuid, dest_path))
+
+    info = {
+        "module_id": module_id,
+        "module_ui_name": module_ui_name,
+        "node_fqdn": "",
+        "path": dest_path,
+        "name": image_name,
+        "uuid": module_uuid,
+        "timestamp": 0,
+        "repository_id" : dest_uuid,
+        "repository_name": odest["name"],
+        "repository_provider": odest["provider"],
+        "repository_url": odest["url"],
+        "installed_instance": module_id,
+        "installed_instance_ui_name": module_ui_name,
+        "is_generated_locally": False,
+    }
+
+    result_repo = task_repo.result()
+    if not result_repo:
+        return None
+
+    try:
+        # Obtain from lsjson the repository creation timestamp
+        info['timestamp'] = int(time.mktime(datetime.fromisoformat(result_repo["ModTime"]).timetuple()))
+    except:
+        info['timestamp'] = int(time.time())
+
+    result_meta = task_meta.result()
+    if "cluster_uuid" in result_meta and result_meta["cluster_uuid"] == cluster_uuid:
+        info['is_generated_locally'] = True
+    info.update(result_meta) # merge two dictionaries
+
+    return info
+
+async def print_destinations(odests):
+    tasks = []
+    async with asyncio.TaskGroup() as tg:
+        for dest_uuid, odest in odests.items():
+            tasks.append(tg.create_task(get_destination_info(dest_uuid, odest)))
+    destinations = list(filter(lambda r: r, [task.result() for task in tasks]))
+    json.dump(destinations, fp=sys.stdout)
+
+asyncio.run(print_destinations(odests))

core/imageroot/usr/local/agent/actions/list-backup-repositories/validate-output.json

@@ -0,0 +1,107 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "list-backup-repositories output",
+    "$id": "http://schema.nethserver.org/module/list-backup-repositories-output.json",
+    "description": "Return a list of the module's Restic backup repositories. The format is the same of cluster/read-backup-repositories.",
+    "examples": [
+        [
+            {
+                "module_id": "loki1",
+                "module_ui_name": "My Loki",
+                "node_fqdn": "rl1.dp.nethserver.net",
+                "path": "loki/35f45b73-f81e-467b-b622-96ec3b7fec19",
+                "name": "loki",
+                "uuid": "35f45b73-f81e-467b-b622-96ec3b7fec19",
+                "timestamp": 1721405723,
+                "repository_id": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8",
+                "repository_name": "BackBlaze repo1",
+                "repository_provider": "backblaze",
+                "repository_url": "b2:ns8-davidep",
+                "installed_instance": "loki1",
+                "installed_instance_ui_name": "My Loki",
+                "is_generated_locally": true
+            }
+        ]
+    ],
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "module_id": {
+                "type": "string",
+                "description": "Original module ID value."
+            },
+            "module_ui_name": {
+                "type": "string",
+                "description": "Original module label, assigned by the user."
+            },
+            "node_fqdn": {
+                "type": "string",
+                "description": "The FQDN of the node where the module of the backup is hosted."
+            },
+            "path": {
+                "type": "string",
+                "description": "Path of the repository, relative to the backup destination."
+            },
+            "name": {
+                "type": "string",
+                "description": "Name of the module. It is equal to the module image name."
+            },
+            "uuid": {
+                "type": "string",
+                "description": "Universal, unique identifier of the module instance."
+            },
+            "timestamp": {
+                "type": "integer",
+                "description": "Unix timestamp of the last backup run."
+            },
+            "repository_id": {
+                "type": "string",
+                "description": "UUID of the backup destination."
+            },
+            "repository_name": {
+                "type": "string",
+                "description": "Human readable name of the backup destination."
+            },
+            "repository_provider": {
+                "type": "string",
+                "description": "Type of backup destination provider, e.g. SMB, S3..."
+            },
+            "repository_url": {
+                "type": "string",
+                "description": "Restic URL of the backup destination."
+            },
+            "installed_instance": {
+                "type": "string",
+                "description": "If the backup belongs to an installed module instance this is its module ID."
+            },
+            "installed_instance_ui_name": {
+                "type": "string",
+                "description": "If the backup belongs to an installed module instance this is its module friendly name."
+            },
+            "is_generated_locally": {
+                "type": [
+                    "boolean",
+                    "null"
+                ],
+                "description": "Tells if the backup originates from the local cluster or from another cluster. The null value is returned if this information is missing completely, as it happens in old backups."
+            }
+        },
+        "required": [
+            "module_id",
+            "module_ui_name",
+            "node_fqdn",
+            "path",
+            "name",
+            "uuid",
+            "timestamp",
+            "repository_id",
+            "repository_name",
+            "repository_provider",
+            "repository_url",
+            "installed_instance",
+            "installed_instance_ui_name",
+            "is_generated_locally"
+        ]
+    }
+}

core/imageroot/usr/local/agent/actions/restore-module/00progress

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import agent
+
+agent.set_weight('00progress', '0')
+agent.set_weight('05replace', '0')
+agent.set_weight('10restore', '4')
+agent.set_weight('20label', '0')

core/imageroot/usr/local/agent/actions/restore-module/10restore

@@ -23,7 +23,7 @@
 import agent
 import json
 import sys
-import os, os.path
+import os
 
 request = json.load(sys.stdin)
 
@@ -37,8 +37,25 @@ rdb = agent.redis_connect(host='127.0.0.1') # Connect to local replica
 podman_args = ["--workdir=/srv"]
 podman_args.extend(agent.get_state_volume_args()) # get volumes from state-include.conf
 
-restic_args = ["restore", snapshot,
+restic_args = ["restore", "--json", snapshot,
     "--target", ".", # workdir should be /srv
     "--exclude", "state/environment", # special core file exception
 ]
-agent.run_restic(rdb, repository, repopath, podman_args, restic_args, stdout=sys.stderr).check_returncode()
+
+# Prepare progress callback function that captures non-progress messages too:
+last_restic_message = {}
+def build_restore_progress_callback():
+    restore_progress = agent.get_progress_callback(1, 100)
+    def fprog(omessage):
+        global last_restic_message
+        last_restic_message = omessage
+        if omessage['message_type'] == 'status':
+            fpercent = float(omessage['percent_done'])
+            restore_progress(int(fpercent * 100))
+    return fprog
+
+prestore = agent.run_restic(rdb, repository, repopath, podman_args, restic_args, progress_callback=build_restore_progress_callback())
+json.dump(last_restic_message, fp=open("restic_restore.json", "w"))
+if prestore.returncode != 0:
+    print(agent.SD_ERR + "Restic restore failed", last_restic_message, file=sys.stderr)
+    sys.exit(1)

core/imageroot/usr/local/agent/actions/run-backup/50run_backup

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+BACKUP_ID=$(jq -r .id)
+exec module-backup "${BACKUP_ID}"

core/imageroot/usr/local/agent/bin/module-backup

@@ -130,9 +130,22 @@ else:
         backup_status['errors'] += 1
         sys.exit(1)
 
+agent_progress_callback = agent.get_progress_callback(1, 95)
+def backup_progress_callback(omessage):
+    global agent_progress_callback
+    if omessage['message_type'] == 'status':
+        fpercent = float(omessage['percent_done'])
+        agent_progress_callback(int(fpercent * 100))
+
 try:
     # Run the backup
-    agent.run_restic(rdb, repository, repopath, podman_args, ["backup"] + backup_args).check_returncode()
+    if os.getenv('AGENT_TASK_ID'):
+        pbackup = agent.run_restic(rdb, repository, repopath, podman_args, ["backup", "--json"] + backup_args, progress_callback=backup_progress_callback)
+        if pbackup.returncode != 0:
+            print(agent.SD_ERR + f"Restic restore command failed with exit code {pbackup.returncode}.", file=sys.stderr)
+            sys.exit(1)
+    else:
+        agent.run_restic(rdb, repository, repopath, podman_args, ["backup", "--no-scan"] + backup_args).check_returncode()
 
     # Apply retention policy
     agent.run_restic(rdb, repository, repopath, [], ["forget", "--prune", "--keep-last=" + obackup['retention']]).check_returncode()

core/imageroot/usr/local/agent/pypkg/agent/__init__.py

@@ -187,7 +187,7 @@ def run_helper(*args, log_command=True, **kwargs):
 
     return subprocess.CompletedProcess(args, proc.returncode)
 
-def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs):
+def prepare_restic_command(rdb, repository, repo_path, podman_args, restic_args):
     core_env = read_envfile('/etc/nethserver/core.env') # Import URLs of core images
     orepo = rdb.hgetall(f"cluster/backup_repository/{repository}")
     assert_exp(len(orepo) > 0) # Check the repository exists
@@ -240,6 +240,11 @@ def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs):
     podman_cmd.append(core_env["RESTIC_IMAGE"])
     podman_cmd.extend(restic_args)
 
+    return (podman_cmd, restic_env)
+
+def run_restic(rdb, repository, repo_path, podman_args, restic_args, progress_callback=None, **kwargs):
+    podman_cmd, restic_env = prepare_restic_command(rdb, repository, repo_path, podman_args, restic_args)
+
     penv = os.environ.copy()
     penv.update(restic_env)
     if os.getenv('DEBUG', False):
@@ -251,8 +256,22 @@ def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs):
     kwargs.setdefault('env', penv)
     kwargs.setdefault('stdout', sys.stdout)
     kwargs.setdefault('stderr', sys.stderr)
-
-    return subprocess.run(podman_cmd, **kwargs)
+    if progress_callback and '--json' in restic_args:
+        kwargs['stdout'] = subprocess.PIPE
+        kwargs.setdefault('errors', 'replace')
+        kwargs.setdefault('text', True)
+        with subprocess.Popen(podman_cmd, **kwargs) as prestic:
+            while True:
+                line = prestic.stdout.readline()
+                if not line:
+                    break
+                try:
+                    progress_callback(json.loads(line))
+                except Exception as ex:
+                    print(SD_DEBUG + "Error decoding Restic status message", ex, file=kwargs['stderr'])
+    else:
+        prestic = subprocess.run(podman_cmd, **kwargs)
+    return prestic
 
 def get_existing_volume_args():
     """Return a list of --volume arguments for Podman run and similar. The argument values

core/imageroot/var/lib/nethserver/cluster/actions/list-backup-repositories/validate-output.json

@@ -2,7 +2,7 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "list-backup-repositories output",
   "$id": "http://schema.nethserver.org/cluster/list-backup-repositories-output.json",
-  "description": "Get the list of available backup repositories and the status of cluster backup password",
+  "description": "Get the list of available backup destinations and the status of cluster backup password",
   "examples": [
     {
       "repositories": [