Merge pull request #605 from Tarsnap/dry-run-metadata

Add --dry-run-metadata

NEWS.md

@@ -21,6 +21,11 @@
   command-line usage.
 - If the server-side state was modified and tarsnap exits with an error, it
   will now have an exit code of 2.
+- tarsnap -c now accepts --dry-run-metadata, which simulates creating an
+  archive without reading any file data.  This is significantly faster than a
+  regular --dry-run, and is suitable for checking which filesystem entries
+  will be archived (with -v) or checking the total archive size (with --totals
+  or --progress-bytes).
 
 
 Tarsnap Releases

misc/bash_completion.d/tarsnap

@@ -52,13 +52,13 @@ _tarsnap ()
 	longopts="--aggressive-networking --archive-names --cachedir \
 		  --check-links --checkpoint-bytes --chroot --configfile \
 		  --creationtime --csv-file --disk-pause --dry-run \
-		  --dump-config --exclude --fast-read --force-resources \
-		  --fsck --fsck-prune --hashes --humanize-numbers \
-		  --include --initialize-cachedir --insane-filesystems \
-		  --iso-dates --keep-going --keep-newer-files --keyfile \
-		  --list-archives --lowmem --maxbw --maxbw-rate \
-		  --maxbw-rate-down --maxbw-rate-up --newer \
-		  --newer-mtime --newer-than --newer-mtime-than \
+		  --dry-run-metadata --dump-config --exclude --fast-read \
+		  --force-resources --fsck --fsck-prune --hashes \
+		  --humanize-numbers --include --initialize-cachedir \
+		  --insane-filesystems --iso-dates --keep-going \
+		  --keep-newer-files --keyfile --list-archives --lowmem \
+		  --maxbw --maxbw-rate --maxbw-rate-down --maxbw-rate-up \
+		  --newer --newer-mtime --newer-than --newer-mtime-than \
 		  --no-aggressive-networking --no-config-exclude \
 		  --no-config-include --no-default-config \
 		  --no-disk-pause --no-force-resources \

misc/describe-options.txt

@@ -14,6 +14,7 @@
 --csv-file		write statistics in CSV format to a file
 --disk-pause		often pause for ARG ms while creating an archive
 --dry-run		do not really create an archive; just simulate doing so
+--dry-run-metadata	do not really create an archive; just simulate the metadata
 --dump-config		print all config lines
 --exclude		do not process certain files or directories
 --fast-read		stop after the first entry which matches the ARG	DUP-q

misc/zsh_completion/_tarsnap

@@ -527,6 +527,7 @@ _shtab_tarsnap_c_options=(
   "--creationtime[manually specify a creation time for the archive]:X:"
   "--disk-pause[often pause for ARG ms while creating an archive]:X:"
   "--dry-run[do not really create an archive\; just simulate doing so]"
+  "--dry-run-metadata[do not really create an archive\; just simulate the metadata]"
   "--exclude[do not process certain files or directories]:pattern:"
   "-f[specify name of archive to operate on]:archive-name:(${archive_list}):"
   "-H[store the targets of a symbolic links]"

tar/bsdtar.c

@@ -463,8 +463,17 @@ main(int argc, char **argv)
 			optq_push(bsdtar, "disk-pause", bsdtar->optarg);
 			break;
 		case OPTION_DRYRUN: /* tarsnap */
+			if (bsdtar->option_dryrun != 0)
+				bsdtar_errc(bsdtar, 1, 0,
+				    "Can only specify one --dry-run* option");
 			bsdtar->option_dryrun = 1;
 			break;
+		case OPTION_DRYRUN_METADATA: /* tarsnap */
+			if (bsdtar->option_dryrun != 0)
+				bsdtar_errc(bsdtar, 1, 0,
+				    "Can only specify one --dry-run* option");
+			bsdtar->option_dryrun = 2;
+			break;
 		case OPTION_EXCLUDE: /* GNU tar */
 			optq_push(bsdtar, "exclude", bsdtar->optarg);
 			break;
@@ -988,6 +997,9 @@ main(int argc, char **argv)
 			tarsnap_opt_aggressive_networking = 0;
 		}
 	}
+	if ((bsdtar->option_dryrun == 2) && bsdtar->option_print_stats)
+		bsdtar_errc(bsdtar, 1, 0, "--dry-run-metadata is "
+		    "incompatible with --print-stats");
 
 	/*
 	 * The -f option doesn't make sense for --fsck, --fsck-prune, or
@@ -1044,8 +1056,10 @@ main(int argc, char **argv)
 		only_mode(bsdtar, "-U", "x");
 	if (bsdtar->option_warn_links)
 		only_mode(bsdtar, "--check-links", "c");
-	if (bsdtar->option_dryrun)
+	if (bsdtar->option_dryrun == 1)
 		only_mode(bsdtar, "--dry-run", "c");
+	if (bsdtar->option_dryrun == 2)
+		only_mode(bsdtar, "--dry-run-metadata", "c");
 
 	/* Check other parameters only permitted in certain modes. */
 	if (bsdtar->symlink_mode != '\0') {

tar/bsdtar.h

@@ -75,7 +75,7 @@ struct bsdtar {
 	char		  option_chroot; /* --chroot */
 	char		 *option_csv_filename; /* --csv-filename */
 	char		  option_dont_traverse_mounts; /* --one-file-system */
-	char		  option_dryrun; /* --dry-run */
+	char		  option_dryrun; /* --dry-run / --dry-run-metadata */
 	char		  option_fast_read; /* --fast-read */
 	int               option_hashes;
 	char		  option_honor_nodump; /* --nodump */
@@ -207,6 +207,7 @@ enum {
 	OPTION_DEBUG_NETWORK_STATS,
 	OPTION_DISK_PAUSE,
 	OPTION_DRYRUN,
+	OPTION_DRYRUN_METADATA,
 	OPTION_DUMP_CONFIG,
 	OPTION_EXCLUDE,
 	OPTION_FORCE_RESOURCES,

tar/cmdline.c

@@ -85,6 +85,7 @@ static struct option {
 	{ "directory",            1, 'C' },
 	{ "disk-pause",		  1, OPTION_DISK_PAUSE },
 	{ "dry-run",		  0, OPTION_DRYRUN },
+	{ "dry-run-metadata",	  0, OPTION_DRYRUN_METADATA },
 	{ "dump-config",	  0, OPTION_DUMP_CONFIG },
 	{ "exclude",              1, OPTION_EXCLUDE },
 	{ "exclude-from",         1, 'X' },

tar/multitape/chunkify.c

@@ -314,6 +314,8 @@ chunkify_init(uint32_t meanlen, uint32_t maxlen,
  *
  * The value returned is zero, or the first nonzero value returned by the
  * callback function.
+ *
+ * If ${c} is NULL, do nothing.
  */
 int
 chunkify_write(CHUNKIFIER * c, const uint8_t * buf, size_t buflen)
@@ -323,6 +325,10 @@ chunkify_write(CHUNKIFIER * c, const uint8_t * buf, size_t buflen)
 	size_t i;
 	int rc;
 
+	/* Bail if we don't have a chunkifier. */
+	if (c == NULL)
+		return (0);
+
 	for (i = 0; i < buflen; i++) {
 		/* Add byte to buffer. */
 		c->buf[c->k] = buf[i];
@@ -432,12 +438,18 @@ chunkify_write(CHUNKIFIER * c, const uint8_t * buf, size_t buflen)
  *
  * The value returned is zero or the nonzero value returned by the callback
  * function.
+ *
+ * If ${c} is NULL, do nothing.
  */
 int
 chunkify_end(CHUNKIFIER * c)
 {
 	int rc;
 
+	/* Bail if we don't have a chunkifier. */
+	if (c == NULL)
+		return (0);
+
 	/* If we haven't started the chunk yet, don't end it either. */
 	if (c->k == 0)
 		return (0);

tar/multitape/chunkify.h

@@ -44,6 +44,8 @@ CHUNKIFIER * chunkify_init(uint32_t, uint32_t, chunkify_callback *, void *);
  *
  * The value returned is zero, or the first nonzero value returned by the
  * callback function.
+ *
+ * If ${c} is NULL, do nothing.
  */
 int chunkify_write(CHUNKIFIER *, const uint8_t *, size_t);
 
@@ -54,6 +56,8 @@ int chunkify_write(CHUNKIFIER *, const uint8_t *, size_t);
  *
  * The value returned is zero or the nonzero value returned by the callback
  * function.
+ *
+ * If ${c} is NULL, do nothing.
  */
 int chunkify_end(CHUNKIFIER *);
 

tar/multitape/multitape.h

@@ -52,7 +52,9 @@ int readtape_close(TAPE_R *);
  * writetape_open(machinenum, cachedir, tapename, argc, argv, printstats,
  *     dryrun, creationtime, csv_filename, storage_modified):
  * Create a tape with the given name, and return a cookie which can be used
- * for accessing it.  The argument vector must be long-lived.
+ * for accessing it.  The argument vector must be long-lived.  If ${dryrun}
+ * is 2, do not pass any data to the chunkifier or chunk layer; and in this
+ * case, ${printstats} cannot be non-zero.
  */
 TAPE_W * writetape_open(uint64_t, const char *, const char *, int, char **,
     int, int, time_t, const char *, int *);

tar/multitape/multitape_write.c

@@ -106,13 +106,17 @@ static int flushtape(TAPE_W *, int);
 
 /* Initialize stream. */
 static int
-stream_init(struct stream * S, chunkify_callback callback, void * cookie)
+stream_init(struct stream * S, chunkify_callback callback, void * cookie,
+    int no_chunkifier)
 {
 
 	/* Create chunkifier. */
-	if ((S->c =
-	    chunkify_init(MEANCHUNK, MAXCHUNK, callback, cookie)) == NULL)
+	if (!no_chunkifier) {
+		if ((S->c = chunkify_init(MEANCHUNK, MAXCHUNK, callback,
+		    cookie)) == NULL)
 		goto err0;
+	} else
+		S->c = NULL;
 
 	/* Allocate elastic array to hold chunk headers. */
 	if ((S->index = chunklist_init(0)) == NULL)
@@ -403,7 +407,9 @@ endentry(TAPE_W * d)
  * writetape_open(machinenum, cachedir, tapename, argc, argv, printstats,
  *     dryrun, creationtime, csv_filename, storage_modified):
  * Create a tape with the given name, and return a cookie which can be used
- * for accessing it.  The argument vector must be long-lived.
+ * for accessing it.  The argument vector must be long-lived.  If ${dryrun}
+ * is 2, do not pass any data to the chunkifier or chunk layer; and in this
+ * case, ${printstats} cannot be non-zero.
  */
 TAPE_W *
 writetape_open(uint64_t machinenum, const char * cachedir,
@@ -414,6 +420,13 @@ writetape_open(uint64_t machinenum, const char * cachedir,
 	struct multitape_write_internal * d;
 	uint8_t lastseq[32];
 	size_t argvlen;
+	int no_chunkifiers;
+
+	/* Sanity check options. */
+	assert(!((dryrun == 2) && printstats));
+
+	/* We don't want the chunkifiers in this case. */
+	no_chunkifiers = (dryrun == 2);
 
 	/* Allocate memory. */
 	if ((d = malloc(sizeof(struct multitape_write_internal))) == NULL)
@@ -501,21 +514,24 @@ writetape_open(uint64_t machinenum, const char * cachedir,
 		goto err6;
 
 	/* Initialize streams. */
-	if (stream_init(&d->h, &callback_h, (void *)d))
+	if (stream_init(&d->h, &callback_h, (void *)d, no_chunkifiers))
 		goto err6;
-	if (stream_init(&d->c, &callback_c, (void *)d))
+	if (stream_init(&d->c, &callback_c, (void *)d, no_chunkifiers))
 		goto err7;
-	if (stream_init(&d->t, &callback_t, (void *)d))
+	if (stream_init(&d->t, &callback_t, (void *)d, no_chunkifiers))
 		goto err8;
 
 	/* Initialize header buffer. */
 	if ((d->hbuf = bytebuf_init(0)) == NULL)
 		goto err9;
 
 	/* Initialize file chunkifier. */
-	if ((d->c_file = chunkify_init(MEANCHUNK, MAXCHUNK, &callback_file,
-	    (void *)d)) == NULL)
-		goto err10;
+	if (!no_chunkifiers) {
+		if ((d->c_file = chunkify_init(MEANCHUNK, MAXCHUNK,
+		    &callback_file, (void *)d)) == NULL)
+			goto err10;
+	} else
+		d->c_file = NULL;
 
 	/* No data has entered or exited c_file. */
 	d->c_file_in = d->c_file_out = 0;

tar/tarsnap.1-man.in

@@ -354,6 +354,26 @@ Furthermore,
 \fB\--dry-run\fP
 will not check whether the cache directory is out of sync.
 .TP
+\fB\--dry-run-metadata\fP
+(c mode only)
+Don't really create an archive; just simulate doing so.
+This is similar to
+\fB\--dry-run\fP,
+except that it doesn't read any files; it only processes filesystem metadata.
+.PP
+This is significantly faster than a regular
+\fB\--dry-run\fP,
+but it is still suitable for checking which filesystem entries will be
+archived (with
+\fB\-v\fP),
+or getting the total uncompressed archive size (via
+\fB\--totals\fP
+or
+\fB\--progress-bytes\fP).
+.PP
+This option cannot be used with
+\fB\--print-stats\fP.
+.TP
 \fB\--dump-config\fP
 Print out the command-line and all non-blank lines read from config files.
 .TP

tar/tarsnap.1-mdoc.in

@@ -339,6 +339,25 @@ from the tarsnap server.
 Furthermore,
 .Fl -dry-run
 will not check whether the cache directory is out of sync.
+.It Fl -dry-run-metadata
+(c mode only)
+Don't really create an archive; just simulate doing so.
+This is similar to
+.Fl -dry-run ,
+except that it doesn't read any files; it only processes filesystem metadata.
+.Pp
+This is significantly faster than a regular
+.Fl -dry-run ,
+but it is still suitable for checking which filesystem entries will be
+archived (with
+.Fl v ) ,
+or getting the total uncompressed archive size (via
+.Fl -totals
+or
+.Fl -progress-bytes ) .
+.Pp
+This option cannot be used with
+.Fl -print-stats .
 .It Fl -dump-config
 Print out the command-line and all non-blank lines read from config files.
 .It Fl -exclude Ar pattern

tar/write.c

@@ -1217,6 +1217,10 @@ write_file_data(struct bsdtar *bsdtar, struct archive *a,
 	ssize_t	bytes_written;
 	off_t	progress = 0;
 
+	/* If we have --dry-run-metadata, don't read any file data. */
+	if (bsdtar->option_dryrun == 2)
+		return (0);
+
 	bytes_read = read(fd, bsdtar->buff, FILEDATABUFLEN);
 	while (bytes_read > 0) {
 		disk_pause(bsdtar->disk_pause);