From fa7b1ebc49c76f4aebc0aad95d1ebc0e9625ecde Mon Sep 17 00:00:00 2001 From: Eric Carraway Date: Sat, 11 May 2024 11:53:50 -0400 Subject: [PATCH 1/3] add `FindManyOpts` --- src/index.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/index.ts b/src/index.ts index 33ce98c..251372a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,6 +29,41 @@ export type CreateRecordOpts = { tableIdOrName: string; }; +export type FindManyOpts = { + /** + * If you don't need every field, you can use this parameter + * to reduce the amount of data transferred. + */ + fields?: string[]; + + /** + * @see https://support.airtable.com/docs/formula-field-reference + */ + filterByFormula?: string; + + /** + * When true, we'll attach the Airtable record ID to each record as `_airtableId`. + * Otherwise, each record will only include its fields. + */ + includeAirtableId?: boolean; + + /** + * The number of records to return in each paginated request. + * Can be used as an optimization in "find one" scenarios. + * Defaults to 100. The maximum is 100. + */ + maxRecords?: number | null; + + /** + * Instructs Airtable to limit the records returned to those that + * have been modified since the specified number of hours ago. + * Cannot be used in combination with `filterByFormula`. + */ + modifiedSinceHours?: number | null; + + tableIdOrName: string; +}; + export type GetRecordOpts = { /** A string of at least 10 characters beginning with "rec". */ recordId: string; From df75fd69b0653cf86157e96a8bcef80130f89c13 Mon Sep 17 00:00:00 2001 From: Eric Carraway Date: Sat, 11 May 2024 13:36:48 -0400 Subject: [PATCH 2/3] implement `findMany` --- dist/index.d.ts | 48 +++++++++- dist/index.js | 116 ++++++++++++++++++++++++ src/index.ts | 234 +++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 383 insertions(+), 15 deletions(-) diff --git a/dist/index.d.ts b/dist/index.d.ts index adb9b5e..ea5b38a 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -9,17 +9,52 @@ export type AirtableClientOpts = { */ baseUrl?: string; }; +export type FieldsObj = Record | string | string[] | undefined>; +export type AirtableRecord = { + /** A date timestamp in the ISO format. */ + createdTime: string; + fields: FieldsObj; + /** Airtable record ID (begins with 'rec'). */ + id: string; +}; export type AirtableResponse = { - data: unknown; + data: AirtableRecord; ok: boolean; status: number; statusText: string; }; -export type FieldsObj = Record; export type CreateRecordOpts = { fields: FieldsObj; tableIdOrName: string; }; +export type FindManyOpts = { + /** + * If you don't need every field, you can use this parameter + * to reduce the amount of data transferred. + */ + fields?: string[]; + /** + * @see https://support.airtable.com/docs/formula-field-reference + */ + filterByFormula?: string; + /** + * When true, we'll attach the Airtable record ID to each record as `_airtableId`. + * Otherwise, each record will only include its fields. + */ + includeAirtableId?: boolean; + /** + * The maximum total number of records to return across all (paginated) requests. + * Can be used as an optimization in "find one" scenarios. + */ + maxRecords?: number | null; + /** + * Instructs Airtable to limit the records returned to those that + * have been modified since the specified number of hours ago. + * Cannot be used in combination with `filterByFormula`. + */ + modifiedSinceHours?: number | null; + tableIdOrName: string; +}; export type GetRecordOpts = { /** A string of at least 10 characters beginning with "rec". */ recordId: string; @@ -77,6 +112,15 @@ export declare class AirtableClient { * @returns {Promise} A promise that resolves with the result of the API call. */ createRecord({ fields, tableIdOrName, }: CreateRecordOpts): Promise; + /** + * Retrieve many (or all) records from a table. + * This method makes paginated requests as necessary. + * Returns an array of records. + * @see https://airtable.com/developers/web/api/list-records + */ + findMany({ fields, filterByFormula, includeAirtableId, maxRecords, modifiedSinceHours, tableIdOrName, }: FindManyOpts): Promise<(FieldsObj | (FieldsObj & { + _airtableId: string; + }))[]>; /** * Retrieve a single record using an Airtable `recordId`. * Any "empty" fields (e.g. "", [], or false) in the record will not be returned. diff --git a/dist/index.js b/dist/index.js index 732498c..c909bc4 100644 --- a/dist/index.js +++ b/dist/index.js @@ -89,6 +89,122 @@ export class AirtableClient { const data = await res.json(); return { data, ok: res.ok, status: res.status, statusText: res.statusText }; } + /** + * Retrieve many (or all) records from a table. + * This method makes paginated requests as necessary. + * Returns an array of records. + * @see https://airtable.com/developers/web/api/list-records + */ + async findMany({ fields, filterByFormula, includeAirtableId, maxRecords, modifiedSinceHours, tableIdOrName, }) { + if (fields) { + const fieldsArrIsValid = Array.isArray(fields) && + fields.length > 0 && + fields.every((field) => !!field && typeof field === `string`); + if (!fieldsArrIsValid) { + throw new TypeError(`Airtable findMany expected 'fields' to be a lengthy array of strings`); + } + } + // Else, `fields` wasn't provided. We'll retrieve all fields. + if (filterByFormula && typeof filterByFormula !== `string`) { + throw new TypeError(`Airtable findMany expected 'filterByFormula' to be a string`); + } + if ((maxRecords && (!Number.isInteger(maxRecords) || maxRecords < 1)) || + maxRecords === 0) { + throw new TypeError(`Airtable findMany expected 'maxRecords' to be a positive integer`); + } + // Else, `maxRecords` wasn't provided. We'll retrieve all records. + if ((modifiedSinceHours && + (!Number.isInteger(modifiedSinceHours) || modifiedSinceHours < 1)) || + modifiedSinceHours === 0) { + throw new TypeError(`Airtable findMany expected 'modifiedSinceHours' to be a positive integer`); + } + // Else, `modifiedSinceHours` wasn't provided or is `null`. We'll retrieve all records. + if (filterByFormula && modifiedSinceHours) { + throw new Error(`Airtable findMany cannot use both 'filterByFormula' and 'modifiedSinceHours'`); + } + if (!tableIdOrName || typeof tableIdOrName !== `string`) { + throw new TypeError(`Airtable findMany expected 'tableIdOrName' to be a non-empty string`); + } + const basePayload = {}; + if (fields) { + basePayload.fields = fields; + } + if (filterByFormula) { + basePayload.filterByFormula = filterByFormula; + } + else if (modifiedSinceHours) { + basePayload.filterByFormula = `{lastModifiedTime}>=DATETIME_FORMAT(DATEADD(NOW(),-${modifiedSinceHours},'hours'))`; + } + if (maxRecords) { + basePayload.maxRecords = maxRecords; + } + const listRecordsUrl = `${this.baseUrl}/${this.baseId}/${tableIdOrName}/listRecords`; + const aggregateResponses = []; + let numRequestsMade = 0; + let offset = null; + while (numRequestsMade === 0 || offset) { + if (numRequestsMade > 500) { + /** + * This safety net prevents an infinite loop of requests that might + * happen if `offset` is (somehow) never set back to null in the + * body of the `while` loop. + * + * 50,000 records per base (divided amongst all tables in that base) + * is the maximum number of records on all non-enterprise plans. + */ + throw new Error(`Airtable findMany should not make more than 500 paginated requests`); + } + const payload = { ...basePayload }; + if (offset && typeof offset === `string`) { + payload.offset = offset; + } + const body = JSON.stringify(payload); + await this.throttleIfNeeded(); + this.setLastRequestAt(); + const res = await fetch(listRecordsUrl, { + body, + headers: this.headers, + // We use a POST instead of a GET request with query parameters. + // It's more ergonomic than encoding query parameters, + // especially when using `filterByFormula`. + method: `POST`, + }); + numRequestsMade += 1; + if (!res.ok) { + throw new Error(`Airtable findMany failed with HTTP status ${res.status} ${res.statusText}`); + } + const data = await res.json(); + if (Array.isArray(data.records) && data.records.length > 0) { + aggregateResponses.push(data.records); + } + if (data.offset && typeof data.offset === `string`) { + ({ offset } = data); + } + else { + // No more records to fetch. + offset = null; + } + } + /** + * Basic mapping function to extract fields from a record. + * Drops the outer `id` & `createdTime` fields. + */ + const basicMapFn = (record) => record.fields; + /** + * Complex mapping function to include the Airtable ID as `_airtableId` + * along with the record's fields. + */ + const complexMapFn = (record) => ({ + _airtableId: record.id, + ...record.fields, + }); + const records = aggregateResponses + // Flatten the array of arrays into a single array of records. + .flat() + // Determine which mapping function to use based on `includeAirtableId`. + .map(includeAirtableId ? complexMapFn : basicMapFn); + return records; + } /** * Retrieve a single record using an Airtable `recordId`. * Any "empty" fields (e.g. "", [], or false) in the record will not be returned. diff --git a/src/index.ts b/src/index.ts index 251372a..f0703be 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,18 +12,35 @@ export type AirtableClientOpts = { baseUrl?: string; }; +export type FieldsObj = Record< + string, + | boolean + | Date + | null + | number + | Record + | string + | string[] + | undefined +>; + +export type AirtableRecord = { + /** A date timestamp in the ISO format. */ + createdTime: string; + + fields: FieldsObj; + + /** Airtable record ID (begins with 'rec'). */ + id: string; +}; + export type AirtableResponse = { - data: unknown; + data: AirtableRecord; ok: boolean; status: number; statusText: string; }; -export type FieldsObj = Record< - string, - boolean | Date | null | number | string | undefined ->; - export type CreateRecordOpts = { fields: FieldsObj; tableIdOrName: string; @@ -48,9 +65,8 @@ export type FindManyOpts = { includeAirtableId?: boolean; /** - * The number of records to return in each paginated request. + * The maximum total number of records to return across all (paginated) requests. * Can be used as an optimization in "find one" scenarios. - * Defaults to 100. The maximum is 100. */ maxRecords?: number | null; @@ -213,11 +229,203 @@ export class AirtableClient { method: `POST`, }); - const data: unknown = await res.json(); + const data: AirtableRecord = await res.json(); return { data, ok: res.ok, status: res.status, statusText: res.statusText }; } + /** + * Retrieve many (or all) records from a table. + * This method makes paginated requests as necessary. + * Returns an array of records. + * @see https://airtable.com/developers/web/api/list-records + */ + public async findMany({ + fields, + filterByFormula, + includeAirtableId, + maxRecords, + modifiedSinceHours, + tableIdOrName, + }: FindManyOpts): Promise< + (FieldsObj | (FieldsObj & { _airtableId: string }))[] + > { + if (fields) { + const fieldsArrIsValid = + Array.isArray(fields) && + fields.length > 0 && + fields.every((field) => !!field && typeof field === `string`); + + if (!fieldsArrIsValid) { + throw new TypeError( + `Airtable findMany expected 'fields' to be a lengthy array of strings`, + ); + } + } + // Else, `fields` wasn't provided. We'll retrieve all fields. + + if (filterByFormula && typeof filterByFormula !== `string`) { + throw new TypeError( + `Airtable findMany expected 'filterByFormula' to be a string`, + ); + } + + if ( + (maxRecords && (!Number.isInteger(maxRecords) || maxRecords < 1)) || + maxRecords === 0 + ) { + throw new TypeError( + `Airtable findMany expected 'maxRecords' to be a positive integer`, + ); + } + // Else, `maxRecords` wasn't provided. We'll retrieve all records. + + if ( + (modifiedSinceHours && + (!Number.isInteger(modifiedSinceHours) || modifiedSinceHours < 1)) || + modifiedSinceHours === 0 + ) { + throw new TypeError( + `Airtable findMany expected 'modifiedSinceHours' to be a positive integer`, + ); + } + // Else, `modifiedSinceHours` wasn't provided or is `null`. We'll retrieve all records. + + if (filterByFormula && modifiedSinceHours) { + throw new Error( + `Airtable findMany cannot use both 'filterByFormula' and 'modifiedSinceHours'`, + ); + } + + if (!tableIdOrName || typeof tableIdOrName !== `string`) { + throw new TypeError( + `Airtable findMany expected 'tableIdOrName' to be a non-empty string`, + ); + } + + type Payload = { + fields?: string[]; + filterByFormula?: string; + maxRecords?: number; + + /** + * The Airtable API may include an `offset` field in the response. + * To fetch the next page of records, we include the offset + * from the previous request in the next request. + */ + offset?: string; + }; + + const basePayload: Payload = {}; + + if (fields) { + basePayload.fields = fields; + } + + if (filterByFormula) { + basePayload.filterByFormula = filterByFormula; + } else if (modifiedSinceHours) { + basePayload.filterByFormula = `{lastModifiedTime}>=DATETIME_FORMAT(DATEADD(NOW(),-${modifiedSinceHours},'hours'))`; + } + + if (maxRecords) { + basePayload.maxRecords = maxRecords; + } + + const listRecordsUrl = `${this.baseUrl}/${this.baseId}/${tableIdOrName}/listRecords`; + const aggregateResponses: AirtableRecord[][] = []; + + let numRequestsMade = 0; + let offset: string | null = null; + + while (numRequestsMade === 0 || offset) { + if (numRequestsMade > 500) { + /** + * This safety net prevents an infinite loop of requests that might + * happen if `offset` is (somehow) never set back to null in the + * body of the `while` loop. + * + * 50,000 records per base (divided amongst all tables in that base) + * is the maximum number of records on all non-enterprise plans. + */ + throw new Error( + `Airtable findMany should not make more than 500 paginated requests`, + ); + } + + const payload: Payload = { ...basePayload }; + + if (offset && typeof offset === `string`) { + payload.offset = offset; + } + + const body = JSON.stringify(payload); + + await this.throttleIfNeeded(); + this.setLastRequestAt(); + + const res: Response = await fetch(listRecordsUrl, { + body, + headers: this.headers, + + // We use a POST instead of a GET request with query parameters. + // It's more ergonomic than encoding query parameters, + // especially when using `filterByFormula`. + method: `POST`, + }); + + numRequestsMade += 1; + + if (!res.ok) { + throw new Error( + `Airtable findMany failed with HTTP status ${res.status} ${res.statusText}`, + ); + } + + const data: { + offset?: string; + records: AirtableRecord[]; + } = await res.json(); + + if (Array.isArray(data.records) && data.records.length > 0) { + aggregateResponses.push(data.records); + } + + if (data.offset && typeof data.offset === `string`) { + ({ offset } = data); + } else { + // No more records to fetch. + offset = null; + } + } + + /** + * Basic mapping function to extract fields from a record. + * Drops the outer `id` & `createdTime` fields. + */ + const basicMapFn = (record: AirtableRecord): FieldsObj => record.fields; + + /** + * Complex mapping function to include the Airtable ID as `_airtableId` + * along with the record's fields. + */ + const complexMapFn = ( + record: AirtableRecord, + ): FieldsObj & { _airtableId: string } => ({ + _airtableId: record.id, + ...record.fields, + }); + + const records: (FieldsObj | (FieldsObj & { _airtableId: string }))[] = + aggregateResponses + // Flatten the array of arrays into a single array of records. + .flat() + // Determine which mapping function to use based on `includeAirtableId`. + .map(includeAirtableId ? complexMapFn : basicMapFn); + + return records; + } + /** * Retrieve a single record using an Airtable `recordId`. * Any "empty" fields (e.g. "", [], or false) in the record will not be returned. @@ -255,12 +463,12 @@ export class AirtableClient { await this.throttleIfNeeded(); this.setLastRequestAt(); - const res = await fetch(getRecordUrl, { + const res: Response = await fetch(getRecordUrl, { headers: this.headers, method: `GET`, }); - const data: unknown = await res.json(); + const data: AirtableRecord = await res.json(); return { data, ok: res.ok, status: res.status, statusText: res.statusText }; } @@ -324,13 +532,13 @@ export class AirtableClient { await this.throttleIfNeeded(); this.setLastRequestAt(); - const res = await fetch(updateRecordUrl, { + const res: Response = await fetch(updateRecordUrl, { body, headers: this.headers, method: method.toUpperCase(), }); - const data: unknown = await res.json(); + const data: AirtableRecord = await res.json(); return { data, ok: res.ok, status: res.status, statusText: res.statusText }; } From 95249433415d3753a02eeeca604b36523ad73ed4 Mon Sep 17 00:00:00 2001 From: Eric Carraway Date: Sat, 11 May 2024 13:52:23 -0400 Subject: [PATCH 3/3] initial spec test for `findMany` --- tests.spec.ts | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests.spec.ts b/tests.spec.ts index 5ef4417..73c1063 100644 --- a/tests.spec.ts +++ b/tests.spec.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import createFetchMock from 'vitest-fetch-mock'; +import type { FindManyOpts } from './dist'; import { AirtableClient } from './dist'; const fetchMock = createFetchMock(vi); @@ -50,6 +51,39 @@ describe(`AirtableClient`, () => { ); }); + it(`findMany with no optional options`, async () => { + fetchMock.mockResponseOnce( + JSON.stringify({ + records: [ + { + fields: { foo: `bar` }, + id: `rec_mock_123456789`, + }, + ], + }), + ); + + const client: AirtableClient = getClient(); + + const opts: FindManyOpts = { tableIdOrName }; + const res = await client.findMany(opts); + + expect(res).toStrictEqual([{ foo: `bar` }]); + expect(fetchMock.requests().length).toBe(1); + + const { body, headers, method, url } = fetchMock.requests()[0]; + + // Since we didn't pass any optional options, + // the POST body should have been an empty object. + expect(parseBody(body)).toStrictEqual({}); + + expect(headers).toStrictEqual(expectedHeaders); + expect(method).toBe(`POST`); + expect(url).toBe( + `https://api.airtable.com/v0/app_mock_123456789/table_mock/listRecords`, + ); + }); + it(`getRecord`, async () => { fetchMock.mockResponseOnce(JSON.stringify({ getRecordResponse: true }));