diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 81ab37b7..5db21dba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,10 +16,10 @@ jobs: steps: # dependencies - name: goreleaser - uses: goreleaser/goreleaser-action@v3 + uses: goreleaser/goreleaser-action@v4 with: install-only: true - version: 1.7.0 + version: latest - name: goreleaser info run: goreleaser -v @@ -44,9 +44,9 @@ jobs: # setup go - name: go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.21 - name: go info run: | @@ -106,15 +106,9 @@ jobs: name: build_linux path: dist/*linux* - - name: artifact_darwin - uses: actions/upload-artifact@v3 - with: - name: build_darwin - path: dist/*darwin* - # docker login - name: docker login - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} diff --git a/.goreleaser.yml b/.goreleaser.yml index 7d097349..dfffd71e 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -11,6 +11,8 @@ builds: main: ./cmd/autoscan goarch: - amd64 + goamd64: + - v2 ldflags: - -s -w - -X "main.Version={{ .Version }}" @@ -18,11 +20,6 @@ builds: - -X "main.Timestamp={{ .Timestamp }}" flags: - -trimpath - ignore: - - goos: freebsd - goarch: arm64 - - goos: freebsd - goarch: arm # MacOS Universal Binaries universal_binaries: diff --git a/Taskfile.yml b/Taskfile.yml index d239499c..2d3c5452 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -53,14 +53,14 @@ tasks: release: desc: Generate a release, but don't publish cmds: - - goreleaser --skip-validate --skip-publish --rm-dist + - goreleaser --skip-validate --skip-publish --clean snapshot: desc: Generate a snapshot release cmds: - - goreleaser --snapshot --skip-publish --rm-dist + - goreleaser --snapshot --skip-publish --clean publish: desc: Generate a release, and publish cmds: - - goreleaser --rm-dist \ No newline at end of file + - goreleaser --clean diff --git a/autoscan.go b/autoscan.go index ed045ae3..68e4ef96 100644 --- a/autoscan.go +++ b/autoscan.go @@ -13,9 +13,10 @@ import ( // // The Scan is used across Triggers, Targets and the Processor. type Scan struct { - Folder string - Priority int - Time time.Time + Folder string + RelativePath string + Priority int + Time time.Time } type ProcessorFunc func(...Scan) error diff --git a/cmd/autoscan/main.go b/cmd/autoscan/main.go index eb5e235b..510b7316 100644 --- a/cmd/autoscan/main.go +++ b/cmd/autoscan/main.go @@ -168,7 +168,9 @@ func main() { Err(err). Msg("Failed opening config") } - defer file.Close() + defer func() { + _ = file.Close() + }() // set default values c := config{ @@ -353,10 +355,9 @@ func main() { targetsAvailable := false targetsSize := len(targets) for { - // sleep indefinitely when no targets setup + // exit when no targets setup if targetsSize == 0 { - log.Warn().Msg("No targets initialised, processor stopped, triggers will continue...") - select {} + log.Fatal().Msg("No targets initialised, exiting...") } // target availability checker @@ -365,18 +366,12 @@ func main() { switch { case err == nil: targetsAvailable = true + case errors.Is(err, autoscan.ErrFatal): - log.Error(). - Err(err). - Msg("Fatal error occurred while checking target availability, processor stopped, triggers will continue...") + log.Fatal().Err(err).Msg("Fatal error occurred while checking target availability, exiting...") - // sleep indefinitely - select {} default: - log.Error(). - Err(err). - Msg("Not all targets are available, retrying in 15 seconds...") - + log.Error().Err(err).Msg("Not all targets are available, retrying in 15 seconds...") time.Sleep(15 * time.Second) continue } @@ -391,40 +386,24 @@ func main() { case errors.Is(err, autoscan.ErrNoScans): // No scans currently available, let's wait a couple of seconds - log.Trace(). - Msg("No scans are available, retrying in 15 seconds...") - + log.Trace().Msg("No scans are available, retrying in 15 seconds...") time.Sleep(15 * time.Second) case errors.Is(err, autoscan.ErrAnchorUnavailable): - log.Error(). - Err(err). - Msg("Not all anchor files are available, retrying in 15 seconds...") - + log.Error().Err(err).Msg("Not all anchor files are available, retrying in 15 seconds...") time.Sleep(15 * time.Second) case errors.Is(err, autoscan.ErrTargetUnavailable): targetsAvailable = false - log.Error(). - Err(err). - Msg("Not all targets are available, retrying in 15 seconds...") - + log.Error().Err(err).Msg("Not all targets are available, retrying in 15 seconds...") time.Sleep(15 * time.Second) case errors.Is(err, autoscan.ErrFatal): - // fatal error occurred, processor must stop (however, triggers must not) - log.Error(). - Err(err). - Msg("Fatal error occurred while processing targets, processor stopped, triggers will continue...") - - // sleep indefinitely - select {} + log.Error().Err(err).Msg("Fatal error occurred while processing targets, exiting...") default: // unexpected error - log.Fatal(). - Err(err). - Msg("Failed processing targets") + log.Fatal().Err(err).Msg("Failed processing targets, exiting...") } } } diff --git a/docker/Dockerfile b/docker/Dockerfile index 6993e339..285e6845 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,7 @@ ENV \ AUTOSCAN_VERBOSITY="0" # Binary -COPY ["./dist/autoscan_${TARGETOS}_${TARGETARCH}${TARGETVARIANT:+_7}/autoscan", "/app/autoscan/autoscan"] +COPY ["./dist/autoscan_${TARGETOS}_${TARGETARCH}${TARGETVARIANT:+_7}_v2/autoscan", "/app/autoscan/autoscan"] # Add root files COPY ["./docker/run", "/etc/services.d/autoscan/run"] diff --git a/go.mod b/go.mod index 184348b8..bd9a8ac3 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,43 @@ module github.com/cloudbox/autoscan -go 1.19 +go 1.21 require ( - github.com/BurntSushi/toml v0.3.1 // indirect - github.com/alecthomas/kong v0.6.1 - github.com/fsnotify/fsnotify v1.5.4 - github.com/go-chi/chi/v5 v5.0.7 + github.com/alecthomas/kong v0.8.0 + github.com/fsnotify/fsnotify v1.6.0 + github.com/go-chi/chi/v5 v5.0.10 github.com/l3uddz/bernard v0.5.1 github.com/m-rots/stubbs v1.1.0 - github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/oriser/regroup v0.0.0-20210730155327-fca8d7531263 + github.com/oriser/regroup v0.0.0-20201024192559-010c434ff8f3 github.com/robfig/cron/v3 v3.0.1 - github.com/rs/zerolog v1.28.0 - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sync v0.0.0-20220907140024-f12130a52804 - golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 // indirect - golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 - golang.org/x/tools v0.1.12 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + github.com/rs/zerolog v1.30.0 + golang.org/x/sync v0.3.0 + golang.org/x/time v0.3.0 gopkg.in/yaml.v2 v2.4.0 - modernc.org/cc/v3 v3.38.1 // indirect - modernc.org/sqlite v1.18.2 - modernc.org/strutil v1.1.3 // indirect + modernc.org/sqlite v1.25.0 ) require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect - github.com/rs/xid v1.4.0 // indirect - lukechampine.com/uint128 v1.2.0 // indirect - modernc.org/ccgo/v3 v3.16.9 // indirect - modernc.org/libc v1.19.0 // indirect - modernc.org/mathutil v1.5.0 // indirect - modernc.org/memory v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rs/xid v1.5.0 // indirect + golang.org/x/mod v0.11.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/tools v0.9.3 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + lukechampine.com/uint128 v1.3.0 // indirect + modernc.org/cc/v3 v3.41.0 // indirect + modernc.org/ccgo/v3 v3.16.15 // indirect + modernc.org/libc v1.24.1 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.0 // indirect modernc.org/opt v0.1.3 // indirect - modernc.org/token v1.0.1 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 44f855f9..33485892 100644 --- a/go.sum +++ b/go.sum @@ -1,26 +1,34 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= +github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA= github.com/alecthomas/kong v0.2.9/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= github.com/alecthomas/kong v0.2.11/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= -github.com/alecthomas/kong v0.6.1 h1:1kNhcFepkR+HmasQpbiKDLylIL8yh5B5y1zPp5bJimA= -github.com/alecthomas/kong v0.6.1/go.mod h1:JfHWDzLmbh/puW6I3V7uWenoh56YNVONW+w8eKeUr9I= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= -github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/alecthomas/kong v0.8.0 h1:ryDCzutfIqJPnNn0omnrgHLbAggDQM2VWHikE1xqK7s= +github.com/alecthomas/kong v0.8.0/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= +github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= +github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= -github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= +github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/l3uddz/bernard v0.5.1 h1:PdkmJn44q4dmix1riBkrvnpb2LVvhyRLwIRhWoc05bo= @@ -33,47 +41,47 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/oriser/regroup v0.0.0-20201024192559-010c434ff8f3 h1:SIHa1eb8CJDqFu8potgn0n7grPG2cf2WEKekk/nSz5g= github.com/oriser/regroup v0.0.0-20201024192559-010c434ff8f3/go.mod h1:odkMeLkWS8G6+WP2z3Pn2vkzhPSvBtFhAUYTKXAtZMQ= -github.com/oriser/regroup v0.0.0-20210730155327-fca8d7531263 h1:Qd1Ml+uEhpesT8Og0ysEhu5+DGhbhW+qxjapH8t1Kvs= -github.com/oriser/regroup v0.0.0-20210730155327-fca8d7531263/go.mod h1:odkMeLkWS8G6+WP2z3Pn2vkzhPSvBtFhAUYTKXAtZMQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= -github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= +github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220907140024-f12130a52804 h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A= -golang.org/x/sync v0.0.0-20220907140024-f12130a52804/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -82,76 +90,71 @@ golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 h1:ohgcoMbSofXygzo6AD2I1kz3BFmW1QArPYTtwEM3UXc= -golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= +lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.31.5-0.20210308123301-7a3e9dab9009/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878= -modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v3 v3.38.1 h1:Yu2IiiRpustRFUgMDZKwVn2RvyJzpfYSOw7zHeKtSi4= -modernc.org/cc/v3 v3.38.1/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= +modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= +modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y= modernc.org/ccgo/v3 v3.9.0/go.mod h1:nQbgkn8mwzPdp4mm6BT6+p85ugQ7FrGgIcYaE7nSrpY= -modernc.org/ccgo/v3 v3.16.9 h1:AXquSwg7GuMk11pIdw7fmO1Y/ybgazVkMhsZWCV0mHM= -modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccgo/v3 v3.16.15 h1:KbDR3ZAVU+wiLyMESPtbtE/Add4elztFyfsWoNTgxS0= +modernc.org/ccgo/v3 v3.16.15/go.mod h1:yT7B+/E2m43tmMOT51GMoM98/MtHIcQQSleGnddkUNI= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/libc v1.7.13-0.20210308123627-12f642a52bb8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= modernc.org/libc v1.8.0/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= -modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= -modernc.org/libc v1.19.0 h1:bXyVhGQg6KIClTr8FMVIDPl7jtbcs7aS5WP7vLDaxPs= -modernc.org/libc v1.19.0/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= +modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= +modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc= -modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= -modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk= -modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.7.0 h1:2pXdbgdP5hIyDp2JqIwkHNZ1sAjEbh8GnRpcqFWBf7E= +modernc.org/memory v1.7.0/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sqlite v1.10.0/go.mod h1:PGzq6qlhyYjL6uVbSgS6WoF7ZopTW/sI7+7p+mb4ZVU= -modernc.org/sqlite v1.18.2 h1:S2uFiaNPd/vTAP/4EmyY8Qe2Quzu26A2L1e25xRNTio= -modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= +modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA= +modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/tcl v1.5.0/go.mod h1:gb57hj4pO8fRrK54zveIfFXBaMHK3SKJNWcmRw1cRzc= -modernc.org/tcl v1.13.2 h1:5PQgL/29XkQ9wsEmmNPjzKs+7iPCaYqUJAhzPvQbjDA= +modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= +modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= -modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= -modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= +modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= +modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE= diff --git a/processor/datastore.go b/processor/datastore.go index 7f8feaac..d0bc8af4 100644 --- a/processor/datastore.go +++ b/processor/datastore.go @@ -33,15 +33,16 @@ func newDatastore(db *sql.DB, mg *migrate.Migrator) (*datastore, error) { } const sqlUpsert = ` -INSERT INTO scan (folder, priority, time) -VALUES (?, ?, ?) +INSERT INTO scan (folder, relative_path, priority, time) +VALUES (?, ?, ?, ?) ON CONFLICT (folder) DO UPDATE SET priority = MAX(excluded.priority, scan.priority), + relative_path = excluded.relative_path, time = excluded.time ` func (store *datastore) upsert(tx *sql.Tx, scan autoscan.Scan) error { - _, err := tx.Exec(sqlUpsert, scan.Folder, scan.Priority, scan.Time) + _, err := tx.Exec(sqlUpsert, scan.Folder, scan.RelativePath, scan.Priority, scan.Time) return err } @@ -82,7 +83,7 @@ func (store *datastore) GetScansRemaining() (int, error) { } const sqlGetAvailableScan = ` -SELECT folder, priority, time FROM scan +SELECT folder, relative_path, priority, time FROM scan WHERE time < ? ORDER BY priority DESC, time ASC LIMIT 1 @@ -92,7 +93,7 @@ func (store *datastore) GetAvailableScan(minAge time.Duration) (autoscan.Scan, e row := store.QueryRow(sqlGetAvailableScan, now().Add(-1*minAge)) scan := autoscan.Scan{} - err := row.Scan(&scan.Folder, &scan.Priority, &scan.Time) + err := row.Scan(&scan.Folder, &scan.RelativePath, &scan.Priority, &scan.Time) switch { case errors.Is(err, sql.ErrNoRows): return scan, autoscan.ErrNoScans @@ -104,7 +105,7 @@ func (store *datastore) GetAvailableScan(minAge time.Duration) (autoscan.Scan, e } const sqlGetAll = ` -SELECT folder, priority, time FROM scan +SELECT folder, relative_path, priority, time FROM scan ` func (store *datastore) GetAll() (scans []autoscan.Scan, err error) { @@ -112,11 +113,13 @@ func (store *datastore) GetAll() (scans []autoscan.Scan, err error) { if err != nil { return scans, err } + defer func() { + _ = rows.Close() + }() - defer rows.Close() for rows.Next() { scan := autoscan.Scan{} - err = rows.Scan(&scan.Folder, &scan.Priority, &scan.Time) + err = rows.Scan(&scan.Folder, &scan.RelativePath, &scan.Priority, &scan.Time) if err != nil { return scans, err } diff --git a/processor/migrations/1_init.sql b/processor/migrations/1_init.sql index d46a0c10..749d903e 100644 --- a/processor/migrations/1_init.sql +++ b/processor/migrations/1_init.sql @@ -3,4 +3,4 @@ CREATE TABLE IF NOT EXISTS scan ( "priority" INTEGER NOT NULL, "time" DATETIME NOT NULL, PRIMARY KEY(folder) -) \ No newline at end of file +); diff --git a/processor/migrations/2_add_relative_path.sql b/processor/migrations/2_add_relative_path.sql new file mode 100644 index 00000000..b69e60cf --- /dev/null +++ b/processor/migrations/2_add_relative_path.sql @@ -0,0 +1 @@ +ALTER TABLE scan ADD COLUMN "relative_path" TEXT NOT NULL DEFAULT ''; diff --git a/targets/autoscan/api.go b/targets/autoscan/api.go index 82cdd557..b8bf8b80 100644 --- a/targets/autoscan/api.go +++ b/targets/autoscan/api.go @@ -77,7 +77,7 @@ func (c apiClient) Available() error { return nil } -func (c apiClient) Scan(path string) error { +func (c apiClient) Scan(folder string, path string) error { // create request req, err := http.NewRequest("POST", autoscan.JoinURL(c.baseURL, "triggers", "manual"), nil) if err != nil { @@ -89,7 +89,13 @@ func (c apiClient) Scan(path string) error { } q := url.Values{} - q.Add("dir", path) + + if path == "" { + q.Add("dir", folder) + } else { + q.Add("path", path) + } + req.URL.RawQuery = q.Encode() // send request diff --git a/targets/autoscan/autoscan.go b/targets/autoscan/autoscan.go index 5d4efcc5..fc3d136f 100644 --- a/targets/autoscan/autoscan.go +++ b/targets/autoscan/autoscan.go @@ -1,6 +1,8 @@ package autoscan import ( + "path" + "github.com/rs/zerolog" "github.com/cloudbox/autoscan" @@ -48,14 +50,20 @@ func New(c Config) (autoscan.Target, error) { func (t target) Scan(scan autoscan.Scan) error { scanFolder := t.rewrite(scan.Folder) + scanPath := "" + if scan.RelativePath != "" { + scanPath = path.Join(scan.Folder, scan.RelativePath) + } + // send scan request l := t.log.With(). - Str("path", scanFolder). + Str("folder", scanFolder). + Str("path", scanPath). Logger() l.Trace().Msg("Sending scan request") - if err := t.api.Scan(scanFolder); err != nil { + if err := t.api.Scan(scanFolder, scanPath); err != nil { return err } diff --git a/targets/emby/emby.go b/targets/emby/emby.go index bed29bb2..c5b4dd28 100644 --- a/targets/emby/emby.go +++ b/targets/emby/emby.go @@ -2,6 +2,7 @@ package emby import ( "fmt" + "path" "strings" "github.com/rs/zerolog" @@ -76,15 +77,20 @@ func (t target) Scan(scan autoscan.Scan) error { return nil } + scanPath := scanFolder + if scan.RelativePath != "" { + scanPath = path.Join(scanFolder, scan.RelativePath) + } + l := t.log.With(). - Str("path", scanFolder). + Str("path", scanPath). Str("library", lib.Name). Logger() // send scan request l.Trace().Msg("Sending scan request") - if err := t.api.Scan(scanFolder); err != nil { + if err := t.api.Scan(scanPath); err != nil { return err } diff --git a/targets/jellyfin/jellyfin.go b/targets/jellyfin/jellyfin.go index fff54b6c..f0ab2c93 100644 --- a/targets/jellyfin/jellyfin.go +++ b/targets/jellyfin/jellyfin.go @@ -2,6 +2,7 @@ package jellyfin import ( "fmt" + "path" "strings" "github.com/rs/zerolog" @@ -76,15 +77,20 @@ func (t target) Scan(scan autoscan.Scan) error { return nil } + scanPath := scanFolder + if scan.RelativePath != "" { + scanPath = path.Join(scanFolder, scan.RelativePath) + } + l := t.log.With(). - Str("path", scanFolder). + Str("path", scanPath). Str("library", lib.Name). Logger() // send scan request l.Trace().Msg("Sending scan request") - if err := t.api.Scan(scanFolder); err != nil { + if err := t.api.Scan(scanPath); err != nil { return err } diff --git a/triggers/manual/manual.go b/triggers/manual/manual.go index eea260ab..8e0aa355 100644 --- a/triggers/manual/manual.go +++ b/triggers/manual/manual.go @@ -47,11 +47,9 @@ type handler struct { } func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - var err error rlog := hlog.FromRequest(r) query := r.URL.Query() - directories := query["dir"] switch r.Method { case "GET": @@ -63,13 +61,15 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } - if len(directories) == 0 { - rlog.Error().Msg("Manual webhook should receive at least one directory") + directories := query["dir"] + paths := query["path"] + if len(directories) == 0 && len(paths) == 0 { + rlog.Error().Msg("Manual webhook should receive at least one directory or path") rw.WriteHeader(http.StatusBadRequest) return } - rlog.Trace().Interface("dirs", directories).Msg("Received directories") + rlog.Trace().Strs("dirs", directories).Strs("paths", paths).Msg("Received directories & paths") scans := make([]autoscan.Scan, 0) @@ -84,7 +84,22 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { }) } - err = h.callback(scans...) + for _, p := range paths { + folder, relativePath := path.Split(path.Clean(p)) + + // Rewrite the path based on the provided rewriter. + folderPath := h.rewrite(path.Clean(folder)) + + scans = append(scans, autoscan.Scan{ + Folder: folderPath, + RelativePath: relativePath, + Priority: h.priority, + Time: now(), + }) + + } + + err := h.callback(scans...) if err != nil { rlog.Error().Err(err).Msg("Processor could not process scans") rw.WriteHeader(http.StatusInternalServerError) @@ -95,6 +110,7 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { for _, scan := range scans { rlog.Info(). Str("path", scan.Folder). + Str("relative_path", scan.RelativePath). Msg("Scan moved to processor") } } diff --git a/triggers/manual/manual_test.go b/triggers/manual/manual_test.go index 4ea9e6c3..f0f11bc8 100644 --- a/triggers/manual/manual_test.go +++ b/triggers/manual/manual_test.go @@ -34,7 +34,11 @@ func TestHandler(t *testing.T) { Rewrite: []autoscan.Rewrite{{ From: "/Movies/*", To: "/mnt/unionfs/Media/Movies/$1", - }}, + }, { + From: "/TV/*", + To: "/mnt/unionfs/Media/TV/$1", + }, + }, } currentTime := time.Now() @@ -70,14 +74,97 @@ func TestHandler(t *testing.T) { StatusCode: 200, Scans: []autoscan.Scan{ { - Folder: "/mnt/unionfs/Media/Movies/Interstellar (2014)", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/Movies/Interstellar (2014)", + RelativePath: "", + Priority: 5, + Time: currentTime, + }, + { + Folder: "/mnt/unionfs/Media/Movies/Parasite (2019)", + RelativePath: "", + Priority: 5, + Time: currentTime, + }, + }, + }, + }, + { + "Returns 200 when given multiple paths", + Given{ + Config: standardConfig, + Query: url.Values{ + "path": []string{ + "/Movies/Interstellar (2014)/Interstellar.mkv", + "/Movies/Parasite (2019)/Parasite.mkv", + "/TV/Chernobyl (2019)/Season 1/Chernobyl S01E01.mkv", + }, + }, + }, + Expected{ + StatusCode: 200, + Scans: []autoscan.Scan{ + { + Folder: "/mnt/unionfs/Media/Movies/Interstellar (2014)", + RelativePath: "Interstellar.mkv", + Priority: 5, + Time: currentTime, + }, + { + Folder: "/mnt/unionfs/Media/Movies/Parasite (2019)", + RelativePath: "Parasite.mkv", + Priority: 5, + Time: currentTime, + }, + { + Folder: "/mnt/unionfs/Media/TV/Chernobyl (2019)/Season 1", + RelativePath: "Chernobyl S01E01.mkv", + Priority: 5, + Time: currentTime, + }, + }, + }, + }, + { + "Returns 200 when given multiple paths & directories", + Given{ + Config: standardConfig, + Query: url.Values{ + "dir": []string{ + "/Movies/Interstellar (2014)", + "/TV/Westworld [imdb:tt0475784]/Season 2/Westworld.S02E01.mkv", + }, + "path": []string{ + "/Movies/Parasite (2019)/Parasite.mkv", + "/TV/Chernobyl (2019)/Season 1/Chernobyl S01E01.mkv", + }, + }, + }, + Expected{ + StatusCode: 200, + Scans: []autoscan.Scan{ + { + Folder: "/mnt/unionfs/Media/Movies/Interstellar (2014)", + RelativePath: "", + Priority: 5, + Time: currentTime, + }, + { + Folder: "/mnt/unionfs/Media/TV/Westworld [imdb:tt0475784]/Season 2/Westworld.S02E01.mkv", + RelativePath: "", + Priority: 5, + Time: currentTime, + }, + { + Folder: "/mnt/unionfs/Media/Movies/Parasite (2019)", + RelativePath: "Parasite.mkv", + Priority: 5, + Time: currentTime, }, { - Folder: "/mnt/unionfs/Media/Movies/Parasite (2019)", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/TV/Chernobyl (2019)/Season 1", + RelativePath: "Chernobyl S01E01.mkv", + Priority: 5, + Time: currentTime, }, }, }, diff --git a/triggers/radarr/radarr.go b/triggers/radarr/radarr.go index 5d1f9365..dba41ce5 100644 --- a/triggers/radarr/radarr.go +++ b/triggers/radarr/radarr.go @@ -75,7 +75,10 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } - var folderPath string + var ( + folderPath string + filePath string + ) if strings.EqualFold(event.Type, "Download") || strings.EqualFold(event.Type, "MovieFileDelete") { if event.File.RelativePath == "" || event.Movie.FolderPath == "" { @@ -85,6 +88,7 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } folderPath = path.Dir(path.Join(event.Movie.FolderPath, event.File.RelativePath)) + filePath = path.Base(path.Join(event.Movie.FolderPath, event.File.RelativePath)) } if strings.EqualFold(event.Type, "MovieDelete") || strings.EqualFold(event.Type, "Rename") { @@ -98,9 +102,10 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } scan := autoscan.Scan{ - Folder: h.rewrite(folderPath), - Priority: h.priority, - Time: now(), + Folder: h.rewrite(folderPath), + RelativePath: filePath, + Priority: h.priority, + Time: now(), } err = h.callback(scan) diff --git a/triggers/radarr/radarr_test.go b/triggers/radarr/radarr_test.go index 2958d9ea..8c278e67 100644 --- a/triggers/radarr/radarr_test.go +++ b/triggers/radarr/radarr_test.go @@ -54,9 +54,10 @@ func TestHandler(t *testing.T) { StatusCode: 200, Scans: []autoscan.Scan{ { - Folder: "/mnt/unionfs/Media/Movies/Interstellar (2014)", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/Movies/Interstellar (2014)", + RelativePath: "Interstellar.2014.UHD.BluRay.2160p.REMUX.mkv", + Priority: 5, + Time: currentTime, }, }, }, @@ -71,9 +72,10 @@ func TestHandler(t *testing.T) { StatusCode: 200, Scans: []autoscan.Scan{ { - Folder: "/mnt/unionfs/Media/Movies/Tenet (2020)", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/Movies/Tenet (2020)", + RelativePath: "Tenet.2020.mkv", + Priority: 5, + Time: currentTime, }, }, }, diff --git a/triggers/sonarr/sonarr.go b/triggers/sonarr/sonarr.go index 4fc9a769..c79de23e 100644 --- a/triggers/sonarr/sonarr.go +++ b/triggers/sonarr/sonarr.go @@ -81,7 +81,7 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } - var paths []string + paths := make(map[string]string) // a Download event is either an upgrade or a new file. // the EpisodeFileDelete event shares the same request format as Download. @@ -94,7 +94,9 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { // Use path.Dir to get the directory in which the file is located folderPath := path.Dir(path.Join(event.Series.Path, event.File.RelativePath)) - paths = append(paths, folderPath) + // Use path.Base to get the filename + filePath := path.Base(path.Join(event.Series.Path, event.File.RelativePath)) + paths[folderPath] = filePath } // An entire show has been deleted @@ -106,7 +108,7 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } // Scan the folder of the show - paths = append(paths, event.Series.Path) + paths[event.Series.Path] = "" } if strings.EqualFold(event.Type, "Rename") { @@ -121,31 +123,34 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { for _, renamedFile := range event.RenamedFiles { previousPath := path.Dir(renamedFile.PreviousPath) + previousFile := path.Base(renamedFile.PreviousPath) currentPath := path.Dir(path.Join(event.Series.Path, renamedFile.RelativePath)) + currentFile := path.Base(path.Join(event.Series.Path, renamedFile.RelativePath)) // if previousPath not in paths, then add it. if _, ok := encountered[previousPath]; !ok { encountered[previousPath] = true - paths = append(paths, previousPath) + paths[previousPath] = previousFile } // if currentPath not in paths, then add it. if _, ok := encountered[currentPath]; !ok { encountered[currentPath] = true - paths = append(paths, currentPath) + paths[currentPath] = currentFile } } } var scans []autoscan.Scan - for _, folderPath := range paths { + for folderPath, filePath := range paths { folderPath := h.rewrite(folderPath) scan := autoscan.Scan{ - Folder: folderPath, - Priority: h.priority, - Time: now(), + Folder: folderPath, + RelativePath: filePath, + Priority: h.priority, + Time: now(), } scans = append(scans, scan) diff --git a/triggers/sonarr/sonarr_test.go b/triggers/sonarr/sonarr_test.go index ca68fd66..c37005f6 100644 --- a/triggers/sonarr/sonarr_test.go +++ b/triggers/sonarr/sonarr_test.go @@ -54,9 +54,10 @@ func TestHandler(t *testing.T) { StatusCode: 200, Scans: []autoscan.Scan{ { - Folder: "/mnt/unionfs/Media/TV/Westworld/Season 1", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/TV/Westworld/Season 1", + RelativePath: "Westworld.S01E01.mkv", + Priority: 5, + Time: currentTime, }, }, }, @@ -71,9 +72,10 @@ func TestHandler(t *testing.T) { StatusCode: 200, Scans: []autoscan.Scan{ { - Folder: "/mnt/unionfs/Media/TV/Westworld/Season 2", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/TV/Westworld/Season 2", + RelativePath: "Westworld.S02E01.mkv", + Priority: 5, + Time: currentTime, }, }, }, @@ -88,24 +90,28 @@ func TestHandler(t *testing.T) { StatusCode: 200, Scans: []autoscan.Scan{ { - Folder: "/mnt/unionfs/Media/TV/Westworld/Season 1", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/TV/Westworld/Season 1", + RelativePath: "Westworld.S01E01.mkv", + Priority: 5, + Time: currentTime, }, { - Folder: "/mnt/unionfs/Media/TV/Westworld [imdb:tt0475784]/Season 1", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/TV/Westworld [imdb:tt0475784]/Season 1", + RelativePath: "Westworld.S01E01.mkv", + Priority: 5, + Time: currentTime, }, { - Folder: "/mnt/unionfs/Media/TV/Westworld/Season 2", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/TV/Westworld/Season 2", + RelativePath: "Westworld.S01E02.mkv", + Priority: 5, + Time: currentTime, }, { - Folder: "/mnt/unionfs/Media/TV/Westworld [imdb:tt0475784]/Season 2", - Priority: 5, - Time: currentTime, + Folder: "/mnt/unionfs/Media/TV/Westworld [imdb:tt0475784]/Season 2", + RelativePath: "Westworld.S02E01.mkv", + Priority: 5, + Time: currentTime, }, }, }, @@ -155,8 +161,8 @@ func TestHandler(t *testing.T) { if !reflect.DeepEqual(tc.Expected.Scans, scans) { t.Log(scans) t.Log(tc.Expected.Scans) - t.Errorf("Scans do not equal") - return errors.New("Scans do not equal") + t.Errorf("Scans are not equal") + return errors.New("scans are not equal") } return nil