Retry upload requests under certain conditions

javascript-client/src/client.ts

@@ -1,4 +1,9 @@
-import axios, { AxiosInstance, AxiosRequestConfig } from 'axios';
+import axios, {
+  AxiosError,
+  AxiosInstance,
+  AxiosRequestConfig,
+  AxiosResponse,
+} from 'axios';
 
 // Description of a part from initializeUpload()
 interface PartInfo {
@@ -36,6 +41,7 @@ export enum S3FileFieldProgressState {
   Sending,
   Finalizing,
   Done,
+  Retrying,
 }
 
 export interface S3FileFieldProgress {
@@ -49,7 +55,41 @@ export type S3FileFieldProgressCallback = (progress: S3FileFieldProgress) => voi
 export interface S3FileFieldClientOptions {
   readonly baseUrl: string;
   readonly onProgress?: S3FileFieldProgressCallback;
-  readonly apiConfig?: AxiosRequestConfig
+  readonly apiConfig?: AxiosRequestConfig;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    window.setTimeout(() => resolve(), ms);
+  });
+}
+
+function shouldRetry(error: Error): boolean {
+  // We only retry requests under certain failure modes. Namely, either
+  // network errors, or a subset of HTTP error codes.
+  const axiosErr = (error as AxiosError);
+  return axiosErr.isAxiosError && (
+    !axiosErr.response
+    || [429, 500, 502, 503, 504].includes(axiosErr.response.status)
+  );
+}
+
+async function retry<T>(
+  fn: () => Promise<T>, onRetry: () => void, condition: (error: Error) => boolean = shouldRetry,
+  interval = 5000,
+): Promise<T> {
+  while (true) { // eslint-disable-line no-constant-condition
+    try {
+      return await fn(); // eslint-disable-line no-await-in-loop
+    } catch (error) {
+      if (condition(error)) {
+        onRetry();
+        await sleep(interval); // eslint-disable-line no-await-in-loop
+      } else {
+        throw error;
+      }
+    }
+  }
 }
 
 export default class S3FileFieldClient {
@@ -108,17 +148,23 @@ export default class S3FileFieldClient {
     let fileOffset = 0;
     for (const part of parts) {
       const chunk = file.slice(fileOffset, fileOffset + part.size);
-      // eslint-disable-next-line no-await-in-loop
-      const response = await axios.put(part.upload_url, chunk, {
-        // eslint-disable-next-line @typescript-eslint/no-loop-func
+      // eslint-disable-next-line @typescript-eslint/no-loop-func, no-await-in-loop
+      const response = await retry<AxiosResponse>(() => axios.put(part.upload_url, chunk, {
         onUploadProgress: (e) => {
           this.onProgress({
             uploaded: fileOffset + e.loaded,
             total: file.size,
             state: S3FileFieldProgressState.Sending,
           });
         },
+      }), () => { // eslint-disable-line @typescript-eslint/no-loop-func
+        this.onProgress({
+          uploaded: fileOffset,
+          total: file.size,
+          state: S3FileFieldProgressState.Retrying,
+        });
       });
+
       uploadedParts.push({
         part_number: part.part_number,
         size: part.size,
@@ -140,15 +186,18 @@ export default class S3FileFieldClient {
   protected async completeUpload(
     multipartInfo: MultipartInfo, parts: UploadedPart[],
   ): Promise<void> {
-    const response = await this.api.post('upload-complete/', {
+    const response = await retry<AxiosResponse>(() => this.api.post('upload-complete/', {
       upload_signature: multipartInfo.upload_signature,
       upload_id: multipartInfo.upload_id,
       parts,
+    }), () => {
+      this.onProgress({ state: S3FileFieldProgressState.Retrying });
     });
     const { complete_url: completeUrl, body } = response.data;
 
+    // TODO support HTTP 200 error: https://github.com/girder/django-s3-file-field/issues/209
     // Send the CompleteMultipartUpload operation to S3
-    await axios.post(completeUrl, body, {
+    await retry<AxiosResponse>(() => axios.post(completeUrl, body, {
       headers: {
         // By default, Axios sets "Content-Type: application/x-www-form-urlencoded" on POST
         // requests. This causes AWS's API to interpret the request body as additional parameters
@@ -157,6 +206,8 @@ export default class S3FileFieldClient {
         // CompleteMultipartUpload docs.
         'Content-Type': null,
       },
+    }), () => {
+      this.onProgress({ state: S3FileFieldProgressState.Retrying });
     });
   }
 
@@ -168,8 +219,10 @@ export default class S3FileFieldClient {
    * @param multipartInfo Signed information returned from /upload-complete/.
    */
   protected async finalize(multipartInfo: MultipartInfo): Promise<string> {
-    const response = await this.api.post('finalize/', {
+    const response = await retry<AxiosResponse>(() => this.api.post('finalize/', {
       upload_signature: multipartInfo.upload_signature,
+    }), () => {
+      this.onProgress({ state: S3FileFieldProgressState.Retrying });
     });
     return response.data.field_value;
   }