diff --git a/javascript-client/src/client.ts b/javascript-client/src/client.ts index eb473cd..58011e5 100644 --- a/javascript-client/src/client.ts +++ b/javascript-client/src/client.ts @@ -1,4 +1,9 @@ -import axios, { AxiosInstance, AxiosRequestConfig } from 'axios'; +import axios, { + AxiosError, + AxiosInstance, + AxiosRequestConfig, + AxiosResponse, +} from 'axios'; // Description of a part from initializeUpload() interface PartInfo { @@ -36,6 +41,7 @@ export enum S3FileFieldProgressState { Sending, Finalizing, Done, + Retrying, } export interface S3FileFieldProgress { @@ -49,7 +55,42 @@ export type S3FileFieldProgressCallback = (progress: S3FileFieldProgress) => voi export interface S3FileFieldClientOptions { readonly baseUrl: string; readonly onProgress?: S3FileFieldProgressCallback; - readonly apiConfig?: AxiosRequestConfig + readonly apiConfig?: AxiosRequestConfig; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => { + window.setTimeout(() => resolve(), ms); + }); +} + +function shouldRetry(error: Error): boolean { + // We only retry requests under certain failure modes. Namely, either + // network errors, or a subset of HTTP error codes. + const axiosErr = (error as AxiosError); + return axiosErr.isAxiosError && ( + !axiosErr.response + || [429, 500, 502, 503, 504].includes(axiosErr.response.status) + ); +} + +async function retry( + fn: () => Promise, onRetry: () => void, condition: (error: Error) => boolean = shouldRetry, + interval = 5000, +): Promise { + while (true) { // eslint-disable-line no-constant-condition + try { + return fn(); + } catch (error) { + if (condition(error)) { + onRetry(); + // eslint-disable-next-line no-await-in-loop + await sleep(interval); + } else { + throw error; + } + } + } } export default class S3FileFieldClient { @@ -108,9 +149,8 @@ export default class S3FileFieldClient { let fileOffset = 0; for (const part of parts) { const chunk = file.slice(fileOffset, fileOffset + part.size); - // eslint-disable-next-line no-await-in-loop - const response = await axios.put(part.upload_url, chunk, { - // eslint-disable-next-line @typescript-eslint/no-loop-func + // eslint-disable-next-line @typescript-eslint/no-loop-func, no-await-in-loop + const response = await retry(() => axios.put(part.upload_url, chunk, { onUploadProgress: (e) => { this.onProgress({ uploaded: fileOffset + e.loaded, @@ -118,7 +158,14 @@ export default class S3FileFieldClient { state: S3FileFieldProgressState.Sending, }); }, + }), () => { // eslint-disable-line @typescript-eslint/no-loop-func + this.onProgress({ + uploaded: fileOffset, + total: file.size, + state: S3FileFieldProgressState.Retrying, + }); }); + uploadedParts.push({ part_number: part.part_number, size: part.size, @@ -140,15 +187,18 @@ export default class S3FileFieldClient { protected async completeUpload( multipartInfo: MultipartInfo, parts: UploadedPart[], ): Promise { - const response = await this.api.post('upload-complete/', { + const response = await retry(() => this.api.post('upload-complete/', { upload_signature: multipartInfo.upload_signature, upload_id: multipartInfo.upload_id, parts, + }), () => { + this.onProgress({ state: S3FileFieldProgressState.Retrying }); }); const { complete_url: completeUrl, body } = response.data; + // TODO support HTTP 200 error: https://github.com/girder/django-s3-file-field/issues/209 // Send the CompleteMultipartUpload operation to S3 - await axios.post(completeUrl, body, { + await retry(() => axios.post(completeUrl, body, { headers: { // By default, Axios sets "Content-Type: application/x-www-form-urlencoded" on POST // requests. This causes AWS's API to interpret the request body as additional parameters @@ -157,6 +207,8 @@ export default class S3FileFieldClient { // CompleteMultipartUpload docs. 'Content-Type': null, }, + }), () => { + this.onProgress({ state: S3FileFieldProgressState.Retrying }); }); } @@ -168,8 +220,10 @@ export default class S3FileFieldClient { * @param multipartInfo Signed information returned from /upload-complete/. */ protected async finalize(multipartInfo: MultipartInfo): Promise { - const response = await this.api.post('finalize/', { + const response = await retry(() => this.api.post('finalize/', { upload_signature: multipartInfo.upload_signature, + }), () => { + this.onProgress({ state: S3FileFieldProgressState.Retrying }); }); return response.data.field_value; }