Merge pull request restic#5194 from darkdragon-001/json-check

Json check

changelog/unreleased/issue-1378

@@ -0,0 +1,7 @@
+Enhancement: Add JSON support to check
+
+Restic `check` now also supports the `--json` option and gives all
+statistics in JSON format.
+
+https://github.com/restic/restic/issues/1378
+https://github.com/restic/restic/pull/5194

cmd/restic/cmd_check.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"math/rand"
 	"os"
 	"strconv"
@@ -45,7 +46,14 @@ Exit status is 12 if the password is incorrect.
 	RunE: func(cmd *cobra.Command, args []string) error {
 		term, cancel := setupTermstatus()
 		defer cancel()
-		return runCheck(cmd.Context(), checkOptions, globalOptions, args, term)
+		summary, err := runCheck(cmd.Context(), checkOptions, globalOptions, args, term)
+		if globalOptions.JSON {
+			if err != nil && summary.NumErrors == 0 {
+				summary.NumErrors = 1
+			}
+			term.Print(ui.ToJSONString(summary))
+		}
+		return err
 	},
 	PreRunE: func(_ *cobra.Command, _ []string) error {
 		return checkFlags(checkOptions)
@@ -210,12 +218,18 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress
 	return cleanup
 }
 
-func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error {
+func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) (checkSummary, error) {
+	summary := checkSummary{MessageType: "summary"}
 	if len(args) != 0 {
-		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
+		return summary, errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
 	}
 
-	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+	var printer progress.Printer
+	if !gopts.JSON {
+		printer = newTerminalProgressPrinter(gopts.verbosity, term)
+	} else {
+		printer = newJSONErrorPrinter(term)
+	}
 
 	cleanup := prepareCheckCache(opts, &gopts, printer)
 	defer cleanup()
@@ -225,44 +239,42 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
-		return err
+		return summary, err
 	}
 	defer unlock()
 
 	chkr := checker.New(repo, opts.CheckUnused)
 	err = chkr.LoadSnapshots(ctx)
 	if err != nil {
-		return err
+		return summary, err
 	}
 
 	printer.P("load indexes\n")
 	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
 	hints, errs := chkr.LoadIndex(ctx, bar)
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	errorsFound := false
-	suggestIndexRebuild := false
-	mixedFound := false
 	for _, hint := range hints {
 		switch hint.(type) {
 		case *checker.ErrDuplicatePacks:
 			term.Print(hint.Error())
-			suggestIndexRebuild = true
+			summary.HintRepairIndex = true
 		case *checker.ErrMixedPack:
 			term.Print(hint.Error())
-			mixedFound = true
+			summary.HintPrune = true
 		default:
 			printer.E("error: %v\n", hint)
 			errorsFound = true
 		}
 	}
 
-	if suggestIndexRebuild {
+	if summary.HintRepairIndex {
 		term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
 	}
-	if mixedFound {
+	if summary.HintPrune {
 		term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
 	}
 
@@ -271,8 +283,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 			printer.E("error: %v\n", err)
 		}
 
+		summary.NumErrors += len(errs)
+		summary.HintRepairIndex = true
 		printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n")
-		return errors.Fatal("repository contains errors")
+		return summary, errors.Fatal("repository contains errors")
 	}
 
 	orphanedPacks := 0
@@ -293,6 +307,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 					salvagePacks.Insert(packErr.ID)
 				}
 				errorsFound = true
+				summary.NumErrors++
 				printer.E("%v\n", err)
 			}
 		} else {
@@ -301,12 +316,15 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		}
 	}
 
-	if orphanedPacks > 0 && !errorsFound {
-		// hide notice if repository is damaged
-		printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
+	if orphanedPacks > 0 {
+		summary.HintPrune = true
+		if !errorsFound {
+			// hide notice if repository is damaged
+			printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
+		}
 	}
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	printer.P("check snapshots, trees and blobs\n")
@@ -316,7 +334,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		bar := newTerminalProgressMax(!gopts.Quiet, 0, "snapshots", term)
+		bar := printer.NewCounter("snapshots")
 		defer bar.Done()
 		chkr.Structure(ctx, bar, errChan)
 	}()
@@ -326,9 +344,11 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		if e, ok := err.(*checker.TreeError); ok {
 			printer.E("error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
+				summary.NumErrors++
 				printer.E("  %v\n", treeErr)
 			}
 		} else {
+			summary.NumErrors++
 			printer.E("error: %v\n", err)
 		}
 	}
@@ -338,13 +358,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	// deadlocking in the case of errors.
 	wg.Wait()
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	if opts.CheckUnused {
 		unused, err := chkr.UnusedBlobs(ctx)
 		if err != nil {
-			return err
+			return summary, err
 		}
 		for _, id := range unused {
 			printer.P("unused blob %v\n", id)
@@ -353,15 +373,15 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	doReadData := func(packs map[restic.ID]int64) {
-		packCount := uint64(len(packs))
-
-		p := newTerminalProgressMax(!gopts.Quiet, packCount, "packs", term)
+		p := printer.NewCounter("packs")
+		p.SetMax(uint64(len(packs)))
 		errChan := make(chan error)
 
 		go chkr.ReadPacks(ctx, packs, p, errChan)
 
 		for err := range errChan {
 			errorsFound = true
+			summary.NumErrors++
 			printer.E("%v\n", err)
 			if err, ok := err.(*repository.ErrPackData); ok {
 				salvagePacks.Insert(err.PackID)
@@ -396,7 +416,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 				repoSize += size
 			}
 			if repoSize == 0 {
-				return errors.Fatal("Cannot read from a repository having size 0")
+				return summary, errors.Fatal("Cannot read from a repository having size 0")
 			}
 			subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
 			if subsetSize > repoSize {
@@ -406,34 +426,32 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 			printer.P("read %d bytes of data packs\n", subsetSize)
 		}
 		if packs == nil {
-			return errors.Fatal("internal error: failed to select packs to check")
+			return summary, errors.Fatal("internal error: failed to select packs to check")
 		}
 		doReadData(packs)
 	}
 
 	if len(salvagePacks) > 0 {
 		printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
-		var strIDs []string
 		for id := range salvagePacks {
-			strIDs = append(strIDs, id.String())
+			summary.BrokenPacks = append(summary.BrokenPacks, id.String())
 		}
-		printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
+		printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(summary.BrokenPacks, " "))
 		printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
 	}
 
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	if errorsFound {
 		if len(salvagePacks) == 0 {
 			printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n")
 		}
-		return errors.Fatal("repository contains errors")
+		return summary, errors.Fatal("repository contains errors")
 	}
 	printer.P("no errors were found\n")
-
-	return nil
+	return summary, nil
 }
 
 // selectPacksByBucket selects subsets of packs by ranges of buckets.
@@ -479,3 +497,41 @@ func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64,
 	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
 	return packs
 }
+
+type checkSummary struct {
+	MessageType     string   `json:"message_type"` // "summary"
+	NumErrors       int      `json:"num_errors"`
+	BrokenPacks     []string `json:"broken_packs"`         // run "restic repair packs ID..." and "restic repair snapshots --forget" to remove damaged files
+	HintRepairIndex bool     `json:"suggest_repair_index"` // run "restic repair index"
+	HintPrune       bool     `json:"suggest_prune"`        // run "restic prune"
+}
+
+type checkError struct {
+	MessageType string `json:"message_type"` // "error"
+	Message     string `json:"message"`
+}
+
+type jsonErrorPrinter struct {
+	term ui.Terminal
+}
+
+func newJSONErrorPrinter(term ui.Terminal) *jsonErrorPrinter {
+	return &jsonErrorPrinter{
+		term: term,
+	}
+}
+
+func (*jsonErrorPrinter) NewCounter(_ string) *progress.Counter {
+	return nil
+}
+
+func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
+	status := checkError{
+		MessageType: "error",
+		Message:     fmt.Sprintf(msg, args...),
+	}
+	p.term.Error(ui.ToJSONString(status))
+}
+func (*jsonErrorPrinter) P(_ string, _ ...interface{})  {}
+func (*jsonErrorPrinter) V(_ string, _ ...interface{})  {}
+func (*jsonErrorPrinter) VV(_ string, _ ...interface{}) {}

cmd/restic/cmd_check_integration_test.go

@@ -32,7 +32,8 @@ func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
 			ReadData:    true,
 			CheckUnused: checkUnused,
 		}
-		return runCheck(context.TODO(), opts, gopts, nil, term)
+		_, err := runCheck(context.TODO(), opts, gopts, nil, term)
+		return err
 	})
 	return buf.String(), err
 }

cmd/restic/cmd_migrate.go

@@ -105,7 +105,7 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio
 					// the repository is already locked
 					checkGopts.NoLock = true
 
-					err = runCheck(ctx, checkOptions, checkGopts, []string{}, term)
+					_, err = runCheck(ctx, checkOptions, checkGopts, []string{}, term)
 					if err != nil {
 						return err
 					}

cmd/restic/cmd_prune_integration_test.go

@@ -112,7 +112,8 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
 	createPrunableRepo(t, env)
 	testRunPrune(t, env.gopts, pruneOpts)
 	rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
-		return runCheck(context.TODO(), checkOpts, env.gopts, nil, term)
+		_, err := runCheck(context.TODO(), checkOpts, env.gopts, nil, term)
+		return err
 	}))
 }
 
@@ -220,7 +221,8 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 		testRunCheck(t, env.gopts)
 	} else {
 		rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
-			return runCheck(context.TODO(), optionsCheck, env.gopts, nil, term)
+			_, err := runCheck(context.TODO(), optionsCheck, env.gopts, nil, term)
+			return err
 		}) != nil,
 			"check should have reported an error")
 	}

cmd/restic/integration_test.go

@@ -88,7 +88,8 @@ func TestListOnce(t *testing.T) {
 	createPrunableRepo(t, env)
 	testRunPrune(t, env.gopts, pruneOpts)
 	rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
-		return runCheck(context.TODO(), checkOpts, env.gopts, nil, term)
+		_, err := runCheck(context.TODO(), checkOpts, env.gopts, nil, term)
+		return err
 	}))
 	rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
 		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)

doc/075_scripting.rst

@@ -245,6 +245,38 @@ are stored in JSON form. Specifying ``--json``  or ``--quiet`` will suppress any
 non-JSON messages the command generates.
 
 
+check
+-----
+
+The ``check`` command uses the JSON lines format with the following message types.
+
+Status
+^^^^^^
+
++--------------------------+------------------------------------------------------------------------------------------------+
+| ``message_type``         | Always "summary"                                                                               |
++--------------------------+------------------------------------------------------------------------------------------------+
+| ``num_errors``           | Number of errors                                                                               |
++--------------------------+------------------------------------------------------------------------------------------------+
+| ``broken_packs``         | Run "restic repair packs ID..." and "restic repair snapshots --forget" to remove damaged files |
++--------------------------+------------------------------------------------------------------------------------------------+
+| ``suggest_repair_index`` | Run "restic repair index"                                                                      |
++--------------------------+------------------------------------------------------------------------------------------------+
+| ``suggest_prune``        | Run "restic prune"                                                                             |
++--------------------------+------------------------------------------------------------------------------------------------+
+
+Error
+^^^^^
+
+These errors are printed on ``stderr``.
+
++----------------------+---------------------------------------------------------------------+
+| ``message_type``     | Always "error"                                                      |
++----------------------+---------------------------------------------------------------------+
+| ``message``          | Error message. May change in arbitrary ways across restic versions. |
++----------------------+---------------------------------------------------------------------+
+
+
 diff
 ----