diff --git a/build.gradle.kts b/build.gradle.kts index 0a6d253..b228ad2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,7 +15,7 @@ description = "RADAR Push API Gateway to handle secured data flow to backend." allprojects { group = "org.radarbase" - version = "0.2.2" + version = "0.2.3" repositories { mavenCentral() diff --git a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/GarminRequestGenerator.kt b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/GarminRequestGenerator.kt index 723189d..4634db9 100644 --- a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/GarminRequestGenerator.kt +++ b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/GarminRequestGenerator.kt @@ -70,31 +70,46 @@ class GarminRequestGenerator( ), ) + private val userNextRequest: MutableMap = mutableMapOf() + private var nextRequestTime: Instant = Instant.MIN private val shouldBackoff: Boolean get() = Instant.now() < nextRequestTime override fun requests(user: User, max: Int): Sequence { - return routes.asSequence() - .flatMap { route -> - val offsets: Offsets? = offsetPersistenceFactory.read(user.versionedId) - val startDate = userRepository.getBackfillStartDate(user) - val startOffset: Instant = if (offsets == null) { - logger.debug("No offsets found for $user, using the start date.") - startDate - } else { - logger.debug("Offsets found in persistence.") - offsets.offsetsMap.getOrDefault( - UserRoute(user.versionedId, route.toString()), startDate - ).coerceAtLeast(startDate) + return if (user.ready()) { + routes.asSequence() + .flatMap { route -> + val offsets: Offsets? = offsetPersistenceFactory.read(user.versionedId) + val backfillLimit = Instant.now().minus(route.maxBackfillPeriod()) + val startDate = userRepository.getBackfillStartDate(user) + var startOffset: Instant = if (offsets == null) { + logger.debug("No offsets found for $user, using the start date.") + startDate + } else { + logger.debug("Offsets found in persistence.") + offsets.offsetsMap.getOrDefault( + UserRoute(user.versionedId, route.toString()), startDate + ).coerceAtLeast(startDate) + } + + if (startOffset <= backfillLimit) { + // the start date is before the backfill limits + logger.warn( + "Backfill limit exceeded for $user and $route. " + + "Resetting to earliest allowed start offset." + ) + startOffset = backfillLimit.plus(Duration.ofDays(2)) + } + + val endDate = userRepository.getBackfillEndDate(user) + if (endDate <= startOffset) return@flatMap emptySequence() + val endTime = (startOffset + defaultQueryRange).coerceAtMost(endDate) + route.generateRequests(user, startOffset, endTime, max / routes.size) } - val endDate = userRepository.getBackfillEndDate(user) - if (endDate <= startOffset) return@flatMap emptySequence() - val endTime = (startOffset + defaultQueryRange).coerceAtMost(endDate) - route.generateRequests(user, startOffset, endTime, max / routes.size) - } - .takeWhile { !shouldBackoff } + .takeWhile { !shouldBackoff } + } else emptySequence() } override fun requestSuccessful(request: RestRequest, response: Response) { @@ -119,12 +134,28 @@ class GarminRequestGenerator( logger.info("A duplicate request was made. Marking successful...") requestSuccessful(request, response) } + 412 -> { + logger.warn( + "User ${request.user} does not have correct permissions/scopes enabled. " + + "Please enable in garmin connect. User backing off for $USER_BACK_OFF_TIME..." + ) + userNextRequest[request.user.versionedId] = Instant.now().plus(USER_BACK_OFF_TIME) + } else -> logger.warn("Request Failed: {}, {}", request, response) } } + private fun User.ready(): Boolean { + return if (versionedId in userNextRequest) { + Instant.now() > userNextRequest[versionedId] + } else { + true + } + } + companion object { private val logger = LoggerFactory.getLogger(GarminRequestGenerator::class.java) private val BACK_OFF_TIME = Duration.ofMinutes(1L) + private val USER_BACK_OFF_TIME = Duration.ofDays(1L) } } diff --git a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/Route.kt b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/Route.kt index c664b32..75d18f2 100644 --- a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/Route.kt +++ b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/Route.kt @@ -17,4 +17,6 @@ interface Route { * This is how it would appear in the offsets */ override fun toString(): String + + fun maxBackfillPeriod(): Duration } diff --git a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminActivitiesRoute.kt b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminActivitiesRoute.kt index fb9cae2..4309d81 100644 --- a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminActivitiesRoute.kt +++ b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminActivitiesRoute.kt @@ -1,6 +1,7 @@ package org.radarbase.push.integration.garmin.backfill.route import org.radarbase.push.integration.garmin.user.GarminUserRepository +import java.time.Duration class GarminActivitiesRoute( consumerKey: String, @@ -10,4 +11,9 @@ class GarminActivitiesRoute( override fun subPath(): String = "activities" override fun toString(): String = "garmin_activities" + + override fun maxBackfillPeriod(): Duration { + // 2 years default. Activity API routes will override this with 5 years + return Duration.ofDays(365 * 5) + } } diff --git a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminActivityDetailsRoute.kt b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminActivityDetailsRoute.kt index 5f5e8db..33a3511 100644 --- a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminActivityDetailsRoute.kt +++ b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminActivityDetailsRoute.kt @@ -1,6 +1,7 @@ package org.radarbase.push.integration.garmin.backfill.route import org.radarbase.push.integration.garmin.user.GarminUserRepository +import java.time.Duration class GarminActivityDetailsRoute( consumerKey: String, @@ -10,4 +11,9 @@ class GarminActivityDetailsRoute( override fun subPath(): String = "activityDetails" override fun toString(): String = "garmin_activity_details" + + override fun maxBackfillPeriod(): Duration { + // 2 years default. Activity API routes will override this with 5 years + return Duration.ofDays(365 * 5) + } } diff --git a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminMoveIQRoute.kt b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminMoveIQRoute.kt index 2dd89cb..a41b90b 100644 --- a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminMoveIQRoute.kt +++ b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminMoveIQRoute.kt @@ -1,6 +1,7 @@ package org.radarbase.push.integration.garmin.backfill.route import org.radarbase.push.integration.garmin.user.GarminUserRepository +import java.time.Duration class GarminMoveIQRoute( consumerKey: String, @@ -10,4 +11,9 @@ class GarminMoveIQRoute( override fun subPath(): String = "moveiq" override fun toString(): String = "garmin_move_iq" + + override fun maxBackfillPeriod(): Duration { + // 2 years default. Activity API routes will override this with 5 years + return Duration.ofDays(365 * 2) + } } diff --git a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminRoute.kt b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminRoute.kt index 9355e74..0b71fdd 100644 --- a/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminRoute.kt +++ b/src/main/kotlin/org/radarbase/push/integration/garmin/backfill/route/GarminRoute.kt @@ -69,6 +69,11 @@ abstract class GarminRoute( } } + override fun maxBackfillPeriod(): Duration { + // 2 years default. Activity API routes will override this with 5 years + return Duration.ofDays(365 * 2) + } + abstract fun subPath(): String companion object { diff --git a/src/main/kotlin/org/radarbase/push/integration/garmin/user/GarminServiceUserRepository.kt b/src/main/kotlin/org/radarbase/push/integration/garmin/user/GarminServiceUserRepository.kt index 8d4fac9..1a9eb5c 100644 --- a/src/main/kotlin/org/radarbase/push/integration/garmin/user/GarminServiceUserRepository.kt +++ b/src/main/kotlin/org/radarbase/push/integration/garmin/user/GarminServiceUserRepository.kt @@ -127,7 +127,7 @@ class GarminServiceUserRepository( @Throws(IOException::class) override fun applyPendingUpdates() { logger.info("Requesting user information from webservice") - val request = requestFor("users?source-type=$GARMIN_SOURCE").build() + val request = requestFor("users?source-type=$GARMIN_SOURCE&authorized=true").build() timedCachedUsers = makeRequest(request, userListReader).users nextFetch = Instant.now().plus(FETCH_THRESHOLD)