From e7ee0897fbec5184b9b8486d48d7364e0273e1e3 Mon Sep 17 00:00:00 2001 From: Christian Theune Date: Fri, 7 Jul 2023 17:14:54 +0200 Subject: [PATCH] nixos/paperless: add backup feature Paperless includes a document exporter that can be used for backups. This extends the module to provide a way to enable and configure a timer, the backup parameters and allow providing a post-processing script (e.g. to ship the backup somewhere else, clean up, ...). It works out of the box when just enabling it but can be customized. Includes suitable tests. --- .../manual/release-notes/rl-2505.section.md | 3 + nixos/modules/services/misc/paperless.nix | 111 +++++++++++++++++- nixos/tests/paperless.nix | 35 ++++++ 3 files changed, 145 insertions(+), 4 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index 9b39fb1869ea76..f80ab7905fd96a 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -308,6 +308,9 @@ - `bind.cacheNetworks` now only controls access for recursive queries, where it previously controlled access for all queries. +- The paperless module now has an option for regular automatic export of + documents data using the integrated document exporter. + - Caddy can now be built with plugins by using `caddy.withPlugins`, a `passthru` function that accepts an attribute set as a parameter. The `plugins` argument represents a list of Caddy plugins, with each Caddy plugin being a versioned module. The `hash` argument represents the `vendorHash` of the resulting Caddy source code with the plugins added. Example: diff --git a/nixos/modules/services/misc/paperless.nix b/nixos/modules/services/misc/paperless.nix index 7f34f8d704ea44..57993c3c2149a8 100644 --- a/nixos/modules/services/misc/paperless.nix +++ b/nixos/modules/services/misc/paperless.nix @@ -1,4 +1,4 @@ -{ config, pkgs, lib, ... }: +{ config, options, pkgs, lib, ... }: let cfg = config.services.paperless; @@ -82,7 +82,7 @@ let }; in { - meta.maintainers = with lib.maintainers; [ leona SuperSandro2000 erikarvstedt ]; + meta.maintainers = with lib.maintainers; [ leona SuperSandro2000 erikarvstedt atemu theuni ]; imports = [ (lib.mkRenamedOptionModule [ "services" "paperless-ng" ] [ "services" "paperless" ]) @@ -252,9 +252,60 @@ in ''; }; }; + + exporter = { + + enable = lib.mkEnableOption "regular automatic document exports"; + + directory = lib.mkOption { + type = lib.types.str; + default = cfg.dataDir + "/exports"; + defaultText = "\${dataDir}/exports"; + description = "Directory to store exports."; + }; + + onCalendar = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "01:30:00"; + description = '' + When to run the exporter. + + Fills in the OnCalendar section of a timer and uses a systemd.time(7) + format. + + `null` disables the timer; allowing you to trigger the + `paperless-exporter` service through other means. + ''; + }; + + options = lib.mkOption { + type = lib.types.attrs; + default = { + "no-progress-bar" = true; + "no-color" = true; + "compare-checksums" = true; + "delete" = true; + }; + description = "Options to pass to the document exporter"; + }; + + preScript = lib.mkOption { + type = lib.types.lines; + default = ""; + description = "Script to run right before the export"; + }; + + postScript = lib.mkOption { + type = lib.types.lines; + default = ""; + description = "Script to run after finishing the export"; + }; + + }; + }; - config = lib.mkIf cfg.enable { + config = lib.mkIf cfg.enable (lib.mkMerge [ { services.redis.servers.paperless.enable = lib.mkIf enableRedis true; services.postgresql = lib.mkIf cfg.database.createLocally { @@ -439,5 +490,57 @@ in gid = config.ids.gids.paperless; }; }; - }; + } + + (lib.mkIf cfg.exporter.enable { + systemd.tmpfiles.rules = [ + "d '${cfg.exporter.directory}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -" + ]; + + systemd.services.paperless-exporter = { + description = "paperless exporter service"; + + startAt = cfg.exporter.onCalendar; + + serviceConfig = { + User = cfg.user; + WorkingDirectory = cfg.dataDir; + }; + + unitConfig = let + services = [ + "paperless-consumer.service" + "paperless-scheduler.service" + "paperless-task-queue.service" + "paperless-web.service" ]; + in { + # Shut down the paperless services while the exporter runs + Conflicts = services; + After = services; + # Bring them back up afterwards, regardless of pass/fail + OnFailure = services; + OnSuccess = services; + }; + enableStrictShellChecks = true; + script = let + # Allowing to override individual values in attrsets without loosing + # the defaults is a bit hairy. We wanted to avoid the complexity + # of a submodule here. + options' = options.services.paperless.exporter.options.default // cfg.exporter.options; + args = lib.cli.toGNUCommandLineShell {} options'; + in + '' + echo "Running pre script ..." + ${cfg.exporter.preScript} + + echo "Exporting documents ..." + ./paperless-manage document_exporter ${cfg.exporter.directory} ${args} + + echo "Running post script ..." + ${cfg.exporter.postScript} + ''; + }; + }) + ]); + } diff --git a/nixos/tests/paperless.nix b/nixos/tests/paperless.nix index 2a2a74eeb24140..efebe90ee2f3a7 100644 --- a/nixos/tests/paperless.nix +++ b/nixos/tests/paperless.nix @@ -8,6 +8,19 @@ import ./make-test-python.nix ({ lib, ... }: { services.paperless = { enable = true; passwordFile = builtins.toFile "password" "admin"; + + exporter.enable = true; + + exporter.options = { + "no-color" = false; # override a default option + "no-thumbnail" = true; # add a new option + }; + exporter.preScript = '' + echo "Hello World" + ''; + exporter.postScript = '' + echo "Goodbye World" + ''; }; }; postgres = { config, pkgs, ... }: { @@ -73,6 +86,28 @@ import ./make-test-python.nix ({ lib, ... }: { metadata = json.loads(node.succeed("curl -u admin:admin -fs localhost:28981/api/documents/3/metadata/")) assert "original_checksum" in metadata + # Check exporter config looks good + with subtest("Exporter config is good"): + node.succeed("systemctl start --wait paperless-exporter") + node.wait_for_unit("paperless-web.service") + node.wait_for_unit("paperless-consumer.service") + node.wait_for_unit("paperless-scheduler.service") + node.wait_for_unit("paperless-task-queue.service") + output = node.succeed("journalctl -u paperless-exporter.service") + print(output) + assert "Hello World" in output, "Missing pre script output" + assert "Goodbye World" in output, "Missing post script output" + node.succeed("ls -lah /var/lib/paperless/exports/manifest.json") + timers = node.succeed("systemctl list-timers paperless-exporter") + print(timers) + assert "paperless-exporter.timer paperless-exporter.service" in timers, "missing timer" + assert "1 timers listed." in timers, "incorrect number of timers" + + # Double check that our attrset option override works as expected + cmdline = node.succeed("grep 'paperless-manage' $(systemctl cat paperless-exporter | grep ExecStart | cut -f 2 -d=)") + print(f"Exporter command line {cmdline!r}") + assert cmdline.strip() == "./paperless-manage document_exporter /var/lib/paperless/exports --compare-checksums --delete --no-progress-bar --no-thumbnail", "Unexpected exporter command line" + test_paperless(simple) simple.send_monitor_command("quit") simple.wait_for_shutdown()