diff --git a/libs/rest/src/fetcher/BaseBucket.ts b/libs/rest/src/fetcher/BaseBucket.ts new file mode 100644 index 0000000..3e402f9 --- /dev/null +++ b/libs/rest/src/fetcher/BaseBucket.ts @@ -0,0 +1,69 @@ +import { DiscordFetchOptions } from './Fetch'; +import type { Rest } from '../struct'; + +export interface BucketConstructor { + makeRoute(method: string, url: string): string; + new (rest: Rest, route: string): BaseBucket; +} + +/** + * Base bucket class - used to deal with all of the HTTP semantics + */ +export abstract class BaseBucket { + /** + * Time after a Bucket is destroyed if unused + */ + public static readonly BUCKET_TTL = 1e4; + + /** + * Creates a simple API route representation (e.g. /users/:id), used as an identifier for each bucket. + * + * Credit to https://github.com/abalabahaha/eris + */ + public static makeRoute(method: string, url: string) { + let route = url + .replace(/\/([a-z-]+)\/(?:[0-9]{17,19})/g, (match, p) => (['channels', 'guilds', 'webhook'].includes(p) ? match : `/${p}/:id`)) + .replace(/\/invites\/[\w\d-]{2,}/g, '/invites/:code') + .replace(/\/reactions\/[^/]+/g, '/reactions/:id') + .replace(/^\/webhooks\/(\d+)\/[A-Za-z0-9-_]{64,}/, '/webhooks/$1/:token') + .replace(/\?.*$/, ''); + + // Message deletes have their own rate limit + if (method === 'delete' && route.endsWith('/messages/:id')) { + route = method + route; + } + + // In this case, /channels/[idHere]/messages is correct, + // however /channels/[idHere] is not. we need "/channels/:id" + if (/^\/channels\/[0-9]{17,19}$/.test(route)) { + route = route.replace(/[0-9]{17,19}/, ':id'); + } + + return route; + } + + protected readonly _destroyTimeout: NodeJS.Timeout; + + public ['constructor']!: typeof BaseBucket; + + public constructor( + public readonly rest: Rest, + public readonly route: string + ) { + // This is in the base constructor for backwards compatibility - in the future it'll be only in the Bucket class + this._destroyTimeout = setTimeout(() => this.rest.buckets.delete(this.route), this.constructor.BUCKET_TTL).unref(); + } + + /** + * Shortcut for the manager mutex + */ + public get mutex() { + return this.rest.mutex; + } + + /** + * Makes a request to Discord + * @param req Request options + */ + public abstract make(req: DiscordFetchOptions): Promise; +} diff --git a/libs/rest/src/struct/Bucket.ts b/libs/rest/src/fetcher/Bucket.ts similarity index 58% rename from libs/rest/src/struct/Bucket.ts rename to libs/rest/src/fetcher/Bucket.ts index f5e3a9a..330ee09 100644 --- a/libs/rest/src/struct/Bucket.ts +++ b/libs/rest/src/fetcher/Bucket.ts @@ -1,6 +1,6 @@ -import { discordFetch, DiscordFetchOptions } from '../Fetch'; +import { discordFetch, DiscordFetchOptions } from './Fetch'; import { CordisRestError, HTTPError } from '../Error'; -import type { Rest } from './Rest'; +import { BaseBucket } from './BaseBucket'; /** * Data held to represent ratelimit state for a Bucket @@ -13,62 +13,9 @@ export interface RatelimitData { } /** - * Represents a rate limiting bucket for Discord's API + * Simple, default sequential bucket */ -export class Bucket { - public static readonly BUCKET_TTL = 1e4; - - /** - * Creates a simple API route representation (e.g. /users/:id), used as an identifier for each bucket. - * - * Credit to https://github.com/abalabahaha/eris - */ - public static makeRoute(method: string, url: string) { - let route = url - .replace(/\/([a-z-]+)\/(?:[0-9]{17,19})/g, (match, p) => (['channels', 'guilds', 'webhook'].includes(p) ? match : `/${p}/:id`)) - .replace(/\/invites\/[\w\d-]{2,}/g, '/invites/:code') - .replace(/\/reactions\/[^/]+/g, '/reactions/:id') - .replace(/^\/webhooks\/(\d+)\/[A-Za-z0-9-_]{64,}/, '/webhooks/$1/:token') - .replace(/\?.*$/, ''); - - // Message deletes have their own rate limit - if (method === 'delete' && route.endsWith('/messages/:id')) { - route = method + route; - } - - // In this case, /channels/[idHere]/messages is correct, - // however /channels/[idHere] is not. we need "/channels/:id" - if (/^\/channels\/[0-9]{17,19}$/.test(route)) { - route = route.replace(/[0-9]{17,19}/, ':id'); - } - - return route; - } - - private readonly _destroyTimeout: NodeJS.Timeout; - - /** - * @param rest The rest manager using this bucket instance - * @param route The identifier of this bucket - */ - public constructor( - public readonly rest: Rest, - public readonly route: string - ) { - this._destroyTimeout = setTimeout(() => this.rest.buckets.delete(this.route), Bucket.BUCKET_TTL).unref(); - } - - /** - * Shortcut for the manager mutex - */ - public get mutex() { - return this.rest.mutex; - } - - /** - * Makes a request to Discord - * @param req Request options - */ +export class Bucket extends BaseBucket { public async make(req: DiscordFetchOptions): Promise { this._destroyTimeout.refresh(); diff --git a/libs/rest/src/Fetch.ts b/libs/rest/src/fetcher/Fetch.ts similarity index 89% rename from libs/rest/src/Fetch.ts rename to libs/rest/src/fetcher/Fetch.ts index 72d13c3..50780f0 100644 --- a/libs/rest/src/Fetch.ts +++ b/libs/rest/src/fetcher/Fetch.ts @@ -3,6 +3,7 @@ import FormData from 'form-data'; import { URLSearchParams } from 'url'; import AbortController from 'abort-controller'; import { RouteBases } from 'discord-api-types/v9'; +import type { Readable } from 'stream'; /** * Represents a file that can be sent to Discord @@ -37,7 +38,8 @@ export interface DiscordFetchOptions { isRetryAfterRatelimit: boolean; query?: Q | string; files?: File[]; - data?: D; + data?: D | Readable; + domain?: string; } /** @@ -45,7 +47,7 @@ export interface DiscordFetchOptions { * @param options Options for the request */ export const discordFetch = async (options: DiscordFetchOptions) => { - let { path, method, headers, controller, query, files, data } = options; + let { path, method, headers, controller, query, files, data, domain = RouteBases.api } = options; let queryString: string | null = null; if (query) { @@ -59,7 +61,7 @@ export const discordFetch = async (options: DiscordFetchOptions) => ).toString(); } - const url = `${RouteBases.api}${path}${queryString ? `?${queryString}` : ''}`; + const url = `${domain}${path}${queryString ? `?${queryString}` : ''}`; let body: string | FormData; if (files?.length) { diff --git a/libs/rest/src/fetcher/ProxyBucket.ts b/libs/rest/src/fetcher/ProxyBucket.ts new file mode 100644 index 0000000..1f2fb05 --- /dev/null +++ b/libs/rest/src/fetcher/ProxyBucket.ts @@ -0,0 +1,47 @@ +import { discordFetch, DiscordFetchOptions } from './Fetch'; +import { CordisRestError, HTTPError } from '../Error'; +import { BaseBucket } from './BaseBucket'; +import type { Rest } from '../struct'; + +/** + * Unconventional Bucket implementation that will hijack all requests (i.e. there is no seperate bucket depending on the route) + * + * This is meant for proxying requests, but will not handle any ratelimiting and will entirely ignore mutexes + */ +export class ProxyBucket extends BaseBucket { + public static override makeRoute() { + return 'proxy'; + } + + public constructor( + rest: Rest, + route: string + ) { + super(rest, route); + // This shouldn't be needed - but for backwards compatibility BaseBucket sets this timeout still + clearTimeout(this._destroyTimeout); + } + + public async make(req: DiscordFetchOptions): Promise { + let timeout: NodeJS.Timeout; + if (req.implicitAbortBehavior) { + timeout = setTimeout(() => req.controller.abort(), this.rest.abortAfter); + } + + const res = await discordFetch(req).finally(() => clearTimeout(timeout)); + + if (res.status === 429) { + return Promise.reject(new CordisRestError('rateLimited', `${req.method.toUpperCase()} ${req.path}`)); + } else if (res.status >= 500 && res.status < 600) { + return Promise.reject(new CordisRestError('internal', `${req.method.toUpperCase()} ${req.path}`)); + } else if (!res.ok) { + return Promise.reject(new HTTPError(res.clone(), await res.text())); + } + + if (res.headers.get('content-type')?.startsWith('application/json')) { + return res.json() as Promise; + } + + return res.blob() as Promise as Promise; + } +} diff --git a/libs/rest/src/fetcher/index.ts b/libs/rest/src/fetcher/index.ts new file mode 100644 index 0000000..8b4a276 --- /dev/null +++ b/libs/rest/src/fetcher/index.ts @@ -0,0 +1,4 @@ +export * from './BaseBucket'; +export * from './Bucket'; +export * from './Fetch'; +export * from './ProxyBucket'; diff --git a/libs/rest/src/index.ts b/libs/rest/src/index.ts index 98eefde..d20ec94 100644 --- a/libs/rest/src/index.ts +++ b/libs/rest/src/index.ts @@ -1,5 +1,5 @@ +export * from './fetcher'; export * from './mutex'; export * from './struct'; export * from './Constants'; export * from './Error'; -export * from './Fetch'; diff --git a/libs/rest/src/mutex/MemoryMutex.ts b/libs/rest/src/mutex/MemoryMutex.ts index ba3bede..1c51e93 100644 --- a/libs/rest/src/mutex/MemoryMutex.ts +++ b/libs/rest/src/mutex/MemoryMutex.ts @@ -1,7 +1,7 @@ // ? Anything ignored from coverage in this file are just weird edge cases - nothing to really cover. import { Mutex } from './Mutex'; -import type { RatelimitData } from '../struct'; +import type { RatelimitData } from '../fetcher'; export interface MemoryRatelimitData extends RatelimitData { expiresAt: Date; diff --git a/libs/rest/src/mutex/Mutex.ts b/libs/rest/src/mutex/Mutex.ts index 0a7255d..34c645d 100644 --- a/libs/rest/src/mutex/Mutex.ts +++ b/libs/rest/src/mutex/Mutex.ts @@ -1,6 +1,6 @@ import { halt } from '@cordis/common'; import { CordisRestError } from '../Error'; -import type { RatelimitData } from '../struct'; +import type { RatelimitData } from '../fetcher'; /** * "Mutex" used to ensure requests don't go through when a ratelimit is about to happen diff --git a/libs/rest/src/struct/Rest.ts b/libs/rest/src/struct/Rest.ts index 25beb5b..92f50b2 100644 --- a/libs/rest/src/struct/Rest.ts +++ b/libs/rest/src/struct/Rest.ts @@ -1,4 +1,13 @@ -import { Bucket, RatelimitData } from './Bucket'; +import { + BaseBucket, + BucketConstructor, + Bucket, + RatelimitData, + DiscordFetchOptions, + File, + RequestBodyData, + StringRecord +} from '../fetcher'; import { USER_AGENT } from '../Constants'; import { EventEmitter } from 'events'; import { Headers, Response } from 'node-fetch'; @@ -6,7 +15,8 @@ import { Mutex, MemoryMutex } from '../mutex'; import AbortController from 'abort-controller'; import { CordisRestError, HTTPError } from '../Error'; import { halt } from '@cordis/common'; -import type { DiscordFetchOptions, File, RequestBodyData, StringRecord } from '../Fetch'; +import type { Readable } from 'stream'; +import { RouteBases } from 'discord-api-types/v9'; /** * Options for constructing a rest manager @@ -14,6 +24,8 @@ import type { DiscordFetchOptions, File, RequestBodyData, StringRecord } from '. export interface RestOptions { /** * How many times to retry making a request before giving up + * + * Tip: If using ProxyBucket you should probably set this to 1 depending on your proxy server's implementation */ retries?: number; /** @@ -34,6 +46,14 @@ export interface RestOptions { * Overwrites the default for `{@link RequestOptions.cacheTime}` */ cacheTime?: number; + /** + * Bucket constructor to use + */ + bucket?: BucketConstructor; + /** + * Overwrites the default domain used for every request + */ + domain?: string; } export interface Rest { @@ -114,7 +134,7 @@ export interface RequestOptions { /** * Body to send, if any */ - data?: D; + data?: D | Readable; /** * Wether or not this request should be re-attempted after a ratelimit is waited out */ @@ -132,6 +152,10 @@ export interface RequestOptions { * @default 10000 */ cacheTime?: number; + /** + * Overwrites the domain used for this request - not taking into account the option passed into {@link RestOptions} + */ + domain?: string; } /** @@ -151,13 +175,15 @@ export class Rest extends EventEmitter { /** * Current active rate limiting Buckets */ - public readonly buckets = new Map(); + public readonly buckets = new Map(); public readonly retries: number; public readonly abortAfter: number; public readonly mutex: Mutex; public readonly retryAfterRatelimit: boolean; public readonly cacheTime: number; + public readonly bucket: BucketConstructor; + public readonly domain: string; /** * @param auth Your bot's Discord token @@ -174,6 +200,8 @@ export class Rest extends EventEmitter { mutex = new MemoryMutex(), retryAfterRatelimit = true, cacheTime = 10000, + bucket = Bucket, + domain = RouteBases.api } = options; this.retries = retries; @@ -181,6 +209,8 @@ export class Rest extends EventEmitter { this.mutex = mutex; this.retryAfterRatelimit = retryAfterRatelimit; this.cacheTime = cacheTime; + this.bucket = bucket; + this.domain = domain; } /** @@ -188,12 +218,12 @@ export class Rest extends EventEmitter { * @param options Options needed for making a request; only the path is required */ public async make(options: RequestOptions): Promise { - const route = Bucket.makeRoute(options.method, options.path); + const route = this.bucket.makeRoute(options.method, options.path); let bucket = this.buckets.get(route); if (!bucket) { - bucket = new Bucket(this, route); + bucket = new this.bucket(this, route); this.buckets.set(route, bucket); } @@ -209,6 +239,7 @@ export class Rest extends EventEmitter { } options.cacheTime ??= this.cacheTime; + options.domain ??= this.domain; let isRetryAfterRatelimit = false; diff --git a/libs/rest/src/struct/index.ts b/libs/rest/src/struct/index.ts index aa5b479..45f2359 100644 --- a/libs/rest/src/struct/index.ts +++ b/libs/rest/src/struct/index.ts @@ -1,2 +1 @@ -export * from './Bucket'; export * from './Rest'; diff --git a/libs/rest/src/struct/rest.test.ts b/libs/rest/src/struct/rest.test.ts index f953fc2..cc6b5cb 100644 --- a/libs/rest/src/struct/rest.test.ts +++ b/libs/rest/src/struct/rest.test.ts @@ -1,6 +1,6 @@ import fetch, { Response, Headers } from 'node-fetch'; import Blob from 'fetch-blob'; -import { Bucket } from './Bucket'; +import { Bucket } from '../fetcher'; import { CordisRestError, HTTPError } from '../Error'; import { RequestOptions, Rest } from './Rest'; import AbortController, { AbortSignal } from 'abort-controller';