Fix

Netflix · Oct 18, 2023 · 593fc90 · 593fc90
1 parent f1cb526
commit 593fc90
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 2 deletions.
diff --git a/...-plane-server/src/main/java/io/mantisrx/server/master/scheduler/BatchScheduleRequest.java b/...-plane-server/src/main/java/io/mantisrx/server/master/scheduler/BatchScheduleRequest.java
@@ -23,4 +23,6 @@
 public class BatchScheduleRequest {
 
     List<ScheduleRequest> scheduleRequests;
+
+    // TODO: not sure if we need readyAt;
 }
diff --git a/...src/main/java/io/mantisrx/server/master/scheduler/ResourceClusterAwareSchedulerActor.java b/...src/main/java/io/mantisrx/server/master/scheduler/ResourceClusterAwareSchedulerActor.java
@@ -176,8 +176,21 @@ private void onAssignedBatchScheduleRequestEvent(AssignedBatchScheduleRequestEve
     }
 
     private void onFailedToBatchScheduleRequestEvent(FailedToBatchScheduleRequestEvent event) {
-        // TODO: fill this...
-        log.warn("onFailedToSubmitBatchScheduleRequestEvent called for event: {}", event.getScheduleRequestEvent());
+        schedulingFailures.increment();
+        if (event.getAttempt() >= this.maxScheduleRetries) {
+            log.error("Failed to submit the request {} because of ", event.getScheduleRequestEvent(), event.getThrowable());
+        } else {
+            // TODO: honor the readyAt attribute from schedule request's rate limiter.
+            Duration timeout = Duration.ofMillis(intervalBetweenRetries.toMillis());
+            log.error("Failed to submit the request {}; Retrying in {} because of ",
+                event.getScheduleRequestEvent(), timeout, event.getThrowable());
+
+            String jobId = event.getScheduleRequestEvent().getRequest().getScheduleRequests().get(0).getJobMetadata().getJobId();
+            getTimers().startSingleTimer(
+                getBatchSchedulingQueueKeyFor(jobId),
+                event.onRetry(),
+                timeout);
+        }
     }
 
     /**
@@ -585,4 +598,8 @@ private static class Noop {
     private String getSchedulingQueueKeyFor(WorkerId workerId) {
         return "Retry-Schedule-Request-For" + workerId.toString();
     }
+
+    private String getBatchSchedulingQueueKeyFor(String jobId) {
+        return "Retry-Batch-Schedule-Request-For" + jobId;
+    }
 }