ftr: Simplify interface (remove staging/grace opts)

This commit leverages a new feature
available in the upstream/sister utility Photein
to encode/import files to both desktop and web libraries in parallel
(i.e., using multi-threaded logic),
thus eliminating the need for a staging directory entirely.

This commit also removes the `--grace-period` CLI option
with the rationale that the slow upload/synchronization of files
from a remote device (e.g., a cell phone) to the host system
provides a sufficient "grace period" on its own.

Dockerfile

@@ -13,4 +13,4 @@ RUN apk add --no-cache --update \
 
 RUN gem install xferase
 
-ENTRYPOINT xferase --inbox "$INBOX" --staging "$STAGING" --library "$LIBRARY" --library-web "$LIBRARY_WEB" --grace-period "$GRACE_PERIOD"
+ENTRYPOINT xferase --inbox "$INBOX" --library "$LIBRARY" --library-web "$LIBRARY_WEB"

Gemfile.lock

@@ -3,7 +3,7 @@ PATH
   specs:
     xferase (0.0.1)
       debouncer (~> 0.2)
-      photein (>= 0.0.9)
+      photein (~> 0.1, >= 0.1.2)
       rb-inotify (~> 0.10)
 
 GEM
@@ -27,7 +27,7 @@ GEM
     optipng (0.2.1)
       command-builder (>= 0.2.0)
       unix-whereis (>= 0.1.0)
-    photein (0.0.9)
+    photein (0.1.2)
       mediainfo (~> 1.5)
       mini_exiftool (~> 2.10)
       mini_magick (~> 4.11)

README.md

@@ -186,21 +186,14 @@ $ docker run -d \
     --env TZ=$(timedatectl show --property=Timezone --value) \
     --volume $HOME/Pictures:/data \
     --env INBOX=/data/.inbox \
-    --env STAGING=/data/.staging \
     --env LIBRARY=/data/master \
     rlue/xferase
 ```
 
 Any photos or videos placed in the **inbox**
-will be automatically moved to the **staging folder** for processing.
-From there, files are moved to the **library**,
+will be automatically moved to the **library**,
 with videos being compressed to save space on disk.
 
-> 🤔 **What’s with the staging folder?**
->
-> Using a temp folder makes it easier to resume from a crash,
-> especially when the `LIBRARY_WEB` env var is set.
-
 #### Option: `LIBRARY_WEB`
 
 ```sh
@@ -210,7 +203,6 @@ $ docker run -d \
     --env TZ=$(timedatectl show --property=Timezone --value) \
     --volume $HOME/Pictures:/data \
     --env INBOX=/data/.inbox \
-    --env STAGING=/data/.staging \
     --env LIBRARY=/data/master \
     --env LIBRARY_WEB=/data/web \
     rlue/xferase
@@ -229,30 +221,6 @@ it will be automatically deleted from the other.
 > if you shoot RAW+JPEG, deleting a .jpg will cause Xferase to delete the
 > corresponding raw image file (and vice versa).
 
-#### Option: `GRACE_PERIOD`
-
-```sh
-$ docker run -d \
-    --name xferase \
-    --user $(id -u) \
-    --env TZ=$(timedatectl show --property=Timezone --value) \
-    --volume $HOME/Pictures:/data \
-    --env INBOX=/data/.inbox \
-    --env STAGING=/data/.staging \
-    --env LIBRARY=/data/master \
-    --env LIBRARY_WEB=/data/web \
-    --env GRACE_PERIOD=60 \
-    rlue/xferase
-```
-
-Xferase will wait 60 seconds before initiating the import process.
-
-Why would you want this?
-Because not every picture you take belongs in your collection—maybe
-you just wanted to show a friend a weird bug you found on the sidewalk.
-With the grace period set, you can film it, send it off, and delete it
-before Xferase wastes CPU time transcoding it (twice).
-
 Guides
 ------
 

TODO.md

@@ -1,4 +1,3 @@
-* Replace staging directory with crash log (see ADR #1)
 * Rewrite inotify watcher to handle appearance of directories
 * double-check deletion-sync logic
   and make sure we don’t delete files with the same timestamp

bin/xferase

@@ -7,15 +7,15 @@ require 'photein'
 
 Xferase::Config.parse_opts!
 Xferase.logger.open
+
+Photein::Config.set(
+  'library-desktop': Xferase::Config.library,
+  'library-web': Xferase::Config.library_web,
+)
 Photein.logger = Xferase.logger
 
 ENV['MEDIAINFO_XML_PARSER'] ||= 'nokogiri'
 
-SRC_DEST_MAP = {
-  "#{Xferase::Config.staging}/desktop" => Xferase::Config.library,
-  "#{Xferase::Config.staging}/web" => Xferase::Config.library_web,
-}.compact
-
 FORMAT_GROUPS = [%w(.jpg .dng .heic), %w(.mov .mp4), %w(.png)]
 
 # Setup ------------------------------------------------------------------------
@@ -25,7 +25,7 @@ require 'pathname'
 require 'rb-inotify'
 require 'tmpdir'
 
-%i[inbox staging library library_web]
+%i[inbox library library_web]
   .map { |dir| Xferase::Config.send(dir) }
   .compact
   .reject(&File.method(:directory?))
@@ -36,102 +36,81 @@ require 'tmpdir'
 
 Thread.abort_on_exception = true
 
-mutex = Mutex.new
-debouncer = Debouncer.new(Xferase::Config.grace_period.to_i) do |*files|
-  files = files.map(&Pathname.method(:new))
-               .select(&:exist?)
-               .reject do |file|
-                 file.dirname.join(".syncthing.#{file.basename}.tmp").exist?
-               end
-
-  SRC_DEST_MAP.keys.each { |dir| FileUtils.ln(files, dir) }
-  FileUtils.rm(files + Dir["#{Xferase::Config.staging}/web/*.DNG"]) # FIXME (ugly hack)
-
-  break if debouncer.inspect_params[:threads] > 2 # don't let threads pile up
-
-  mutex.synchronize do
-    SRC_DEST_MAP.each do |src, dest|
-      Photein::Config.set(
-        source: src,
-        dest: dest,
-        'optimize-for': Pathname(src).basename.to_sym
-      )
-
-      Photein.run
-    end
-  rescue => e
-    warn e.message
-  end
-end.reducer(:+)
+# Helper methods ---------------------------------------------------------------
+def import(file)
+  file = case file
+         when INotify::Event
+           Pathname(file.absolute_name)
+         else
+           Pathname(file)
+         end.cleanpath
+
+  return unless file.exist?
+  return if file.basename.fnmatch?('.*')
+  return if file.dirname.join(".syncthing.#{file.basename}.tmp").exist?
+
+  Xferase.logger.debug("#{file.basename}: new file detected in watch directory; importing...")
+
+  Photein::MediaFile.for(file)&.import ||
+    Xferase.logger.debug("#{file.basename}: unrecognize media type")
+rescue => e
+  warn e.message
+end
 
-# Resume from interruption/failure ---------------------------------------------
-debouncer.call unless SRC_DEST_MAP.keys.all?(&Dir.method(:empty?))
+def sync_deletions(event)
+  Xferase.logger.info("#{event.name} has disappeared!")
 
-Dir["#{Xferase::Config.inbox}/**/*"]
-  .select(&File.method(:file?))
-  .each { |path| debouncer.call(path) }
+  deleted_file   = Pathname(event.absolute_name).expand_path
 
-# Start ------------------------------------------------------------------------
-Thread.new do
-  call_debouncer = ->(event) do
-    next if event.name.start_with?('.')
+  sister_file    = if deleted_file.to_s.start_with?(File.expand_path(Xferase::Config.library))
+                    deleted_file.sub(Xferase::Config.library, Xferase::Config.library_web)
+                  else
+                    deleted_file.sub(Xferase::Config.library_web, Xferase::Config.library)
+                  end
 
-    Xferase.logger.debug("#{event.name}: new file detected in watch directory")
-    debouncer.call(event.absolute_name)
-  rescue => e
-    warn e.message
-  end
+  sister_formats = FORMAT_GROUPS.find { |group| group.include?(deleted_file.extname) }
 
-  import_notifier = INotify::Notifier.new
-  import_notifier.watch(Xferase::Config.inbox, :close_write, &call_debouncer)
+  related_files  = [deleted_file, sister_file]
+    .product(sister_formats)
+    .map { |file, ext| file.sub_ext(ext) }
+    .select(&:file?)
 
-  # NOTE: inotify is not recursive,
-  # so subdirectories must be watched separately!
-  # (Why do Syncthing folders get special treatment?
-  # Because ST works by creating hidden tempfiles and moving them upon completion)
-  stfolders, simple_subdirs = Dir["#{Xferase::Config.inbox}/**/*"]
-    .select(&File.method(:directory?))
-    .partition { |dir| Dir.children(dir).include?('.stfolder') }
+  # (Why mv to tmpdir first? Why not rm straight away?
+  # Because rm would recursively trigger this inotify callback.)
+  related_files.each { |f| Xferase.logger.info("deleting #{f.realpath}") }
+  FileUtils.mv(related_files, Dir.tmpdir)
+  FileUtils.rm(related_files.map { |f| File.join(Dir.tmpdir, f.basename) })
+end
 
-  simple_subdirs.each { |dir| import_notifier.watch(dir, :close_write, &call_debouncer) }
-  stfolders.each { |dir| import_notifier.watch(dir, :moved_to, &call_debouncer) }
+# Resume from interruption/failure ---------------------------------------------
+Dir["#{Xferase::Config.inbox}/**/*"].sort
+  .select(&File.method(:file?))
+  .each(&method(:import))
 
-  import_notifier.run
+# Start ------------------------------------------------------------------------
+Thread.new do
+  INotify::Notifier.new.tap do |import_notifier|
+    import_notifier.watch(Xferase::Config.inbox, :close_write, &method(:import))
+
+    # NOTE: inotify is not recursive,
+    # so subdirectories must be watched separately!
+    # (Why do Syncthing folders get special treatment?
+    # Because ST works by creating hidden tempfiles and moving them upon completion)
+    stfolders, simple_subdirs = Dir["#{Xferase::Config.inbox}/**/*"]
+      .select(&File.method(:directory?))
+      .partition { |dir| Dir.children(dir).include?('.stfolder') }
+
+    simple_subdirs.each { |dir| import_notifier.watch(dir, :close_write, &method(:import)) }
+    stfolders.each { |dir| import_notifier.watch(dir, :moved_to, &method(:import)) }
+  end.run
 end
 
 Thread.new do
-  sync_deletions = ->(event) do
-    Xferase.logger.info("#{event.name} has disappeared!")
-
-    deleted_file   = Pathname(event.absolute_name).expand_path
-
-    sister_file    = if deleted_file.to_s.start_with?(File.expand_path(Xferase::Config.library))
-                      deleted_file.sub(Xferase::Config.library, Xferase::Config.library_web)
-                    else
-                      deleted_file.sub(Xferase::Config.library_web, Xferase::Config.library)
-                    end
-
-    sister_formats = FORMAT_GROUPS.find { |group| group.include?(deleted_file.extname) }
-
-    related_files  = [deleted_file, sister_file]
-      .product(sister_formats)
-      .map { |file, ext| file.sub_ext(ext) }
-      .select(&:file?)
-
-    # (Why mv to tmpdir first? Why not rm straight away?
-    # Because rm would recursively trigger this inotify callback.)
-    related_files.each { |f| Xferase.logger.info("deleting #{f.realpath}") }
-    FileUtils.mv(related_files, Dir.tmpdir)
-    FileUtils.rm(related_files.map { |f| File.join(Dir.tmpdir, f.basename) })
-  end
-
-  deletion_notifier = INotify::Notifier.new
-
-  # NOTE: inotify is not recursive,
-  # so subdirectories must be watched separately!
-  (Dir["#{Xferase::Config.library}/**/*"] + Dir["#{Xferase::Config.library_web}/**/*"])
-    .select(&File.method(:directory?))
-    .each { |dir| deletion_notifier.watch(dir, :delete, &sync_deletions) }
-
-  deletion_notifier.run
+  INotify::Notifier.new.tap do |deletion_notifier|
+    # NOTE: inotify is not recursive,
+    # so subdirectories must be watched separately!
+    (Dir["#{Xferase::Config.library}/**/*"] + Dir["#{Xferase::Config.library_web}/**/*"])
+      .select(&File.method(:directory?))
+      .each { |dir| deletion_notifier.watch(dir, :delete, &method(:sync_deletions)) }
+  end.run
 end.join

doc/adr/0001-replace-staging-directory-with-crash-log.md

@@ -4,7 +4,7 @@ Date: 2021-07-23
 
 ## Status
 
-Accepted
+Superseded
 
 ## Context
 

doc/adr/0002-remove-staging-directory.md

@@ -0,0 +1,44 @@
+# 2. Remove Staging Directory
+
+Date: 2021-12-09
+
+## Status
+
+Completed
+
+## Context
+
+> The objective of the change described herein
+> is to simplify/refine the approach described in ADR #1.
+
+ADR #1 laid out a plan to replace the staging directory with a crash log
+under an assumption that photein could only import media files sequentially
+(_i.e.,_ one after another):
+if the program crashes after a file has been imported to the master library
+but before it has been imported to the web library,
+Xferase needs some record of its progress to recover from that crash
+and know which files still need to be imported to which libraries.
+
+## Decision
+
+Upon further consideration, the author opted to improve Photein
+to enable parallel processing/import of individual files
+to minimize this potential window of failure
+and dispense with the implementation of a crash log entirely.
+
+## Consequences
+
+Previously, the design of the program was to batch-import a set of new files
+first to the desktop library, and then to the web library.
+The new design imports each individual file as it is detected,
+and imports to both libraries simultaneously in separate threads.
+
+Thus, there is still a potential window of failure
+that could result in a state requiring manual intervention to recover from.
+However, this window is significantly reduced over the previous case,
+and no record exists of a program crash occurring _during_ import/encoding.
+(Typically, in the past, program crashes would occur
+when invalid media files appeared/were transferred to the staging directory.)
+
+This significantly simplifies the design of the program
+over the previous design plan.

guides/ingest.md

@@ -36,7 +36,6 @@ $ docker run -d \
     --env TZ=$(timedatectl show --property=Timezone --value) \
     --volume $HOME/Pictures:/data \
     --env INBOX=/data/.inbox \
-    --env STAGING=/data/.staging \
     --env LIBRARY=/data/master \
     --env LIBRARY_WEB=/data/web \
     rlue/xferase
@@ -57,7 +56,6 @@ services:
   environment:
     TZ: America/Los_Angeles  # or whatever the value of `timedatectl show --property=Timezone --value` is
     INBOX: /data/.inbox
-    STAGING: /data/.staging
     LIBRARY: /data/master
     LIBRARY_WEB: /data/web
   restart: unless-stopped
@@ -95,7 +93,6 @@ Option 2: RubyGems + systemd
        [Service]
        ExecStart=xferase \  # Using rbenv or rvm? Use `rbenv exec xferase` or `rvm-exec xferase` instead
          --inbox=$HOME/Pictures/.inbox \
-         --staging=$HOME/Pictures/.staging \
          --library=$HOME/Pictures/master \
          --library-web=$HOME/Pictures/web
        Restart=on-failure

lib/xferase/config.rb

@@ -10,10 +10,8 @@ class Config
     OPTIONS = [
       ['-v',              '--verbose',                 'print verbose output'],
       ['-i INBOX',        '--inbox=INBOX',             'path to the inbox (required)'],
-      ['-s STAGING',      '--staging=STAGING',         'path to the staging directory (required)'],
       ['-l LIBRARY',      '--library=LIBRARY',         'path to the master library (required)'],
       ['-w LIBRARY_WEB',  '--library-web=LIBRARY_WEB', 'path to the web-optimized library'],
-      ['-g INTERVAL',     '--grace-period=INTERVAL',   'wait n seconds for additional files before import'],
     ].freeze
 
     OPTION_NAMES = OPTIONS
@@ -38,8 +36,8 @@ def parse_opts!
         @params.freeze
 
         raise "no inbox directory given" if !@params.key?(:inbox)
-        raise "no staging directory given" if !@params.key?(:staging)
-        raise "no master library given" if !@params.key?(:library)
+        (%i[library library-web] & @params.keys)
+          .then { |dest_dirs| raise "no destination library given" if dest_dirs.empty? }
       rescue => e
         warn("#{parser.program_name}: #{e.message}")
         warn(parser.help) if e.is_a?(OptionParser::ParseError)

share/systemd/user/xferase.service

@@ -2,7 +2,7 @@
 Description=Photo/video import daemon
 
 [Service]
-ExecStart=$HOME/.rbenv/bin/rbenv exec xferase --inbox="$HOME/Pictures/.inbox" --staging="$HOME/Pictures/.staging" --library="$HOME/Pictures/master" --library-web="$HOME/Pictures/web"
+ExecStart=$HOME/.rbenv/bin/rbenv exec xferase --inbox="$HOME/Pictures/.inbox" --library="$HOME/Pictures/master" --library-web="$HOME/Pictures/web"
 Restart=on-failure
 
 [Install]