diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml index 41991603..792340ae 100644 --- a/.github/workflows/release-workflow.yml +++ b/.github/workflows/release-workflow.yml @@ -4,6 +4,7 @@ on: workflow_dispatch: permissions: + pull-requests: write contents: write jobs: @@ -147,3 +148,11 @@ jobs: git fetch origin main git checkout main git push origin main:production -f + - name: Wait for 5 minutes for any remaining Vercel Main Branch Deployment that's in flight + run: sleep 300 + - name: Stably Runner Action + id: stably-runner + uses: stablyhq/stably-runner-action@v3 + with: + api-key: ${{ secrets.STABLY_API_KEY }} + test-group-id: cm5g8i6nc0005l103urkixuxz diff --git a/.github/workflows/stably-workflow.yml b/.github/workflows/stably-workflow.yml deleted file mode 100644 index aa141643..00000000 --- a/.github/workflows/stably-workflow.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Stably Test Runner - -on: - push: - branches: - - production - -permissions: - pull-requests: write - contents: write - -jobs: - stably-test-action: - name: Stably Test Runner - runs-on: ubuntu-latest - - steps: - - name: Wait for 5 minutes for any remaining Vercel Main Branch Deployment - run: sleep 300 - - name: Stably Runner Action - id: stably-runner - uses: stablyhq/stably-runner-action@v3 - with: - api-key: ${{ secrets.STABLY_API_KEY }} - test-group-id: cm5g8i6nc0005l103urkixuxz diff --git a/apps/web/app/connect/page.tsx b/apps/web/app/connect/page.tsx index 21417bf9..f0a6b9cf 100644 --- a/apps/web/app/connect/page.tsx +++ b/apps/web/app/connect/page.tsx @@ -1,123 +1,8 @@ -// import {clerkClient} from '@clerk/nextjs/server' -// import Image from 'next/image' -import {defConnectors} from '@openint/app-config/connectors/connectors.def' -import {kAccessToken} from '@openint/app-config/constants' -import {envRequired} from '@openint/app-config/env' -import type {ConnectorDef} from '@openint/cdk' -import { - extractConnectorName, - getViewerId, - makeId, - NangoConnect, -} from '@openint/cdk' -import {zConnectPageParams} from '@openint/engine-backend/router/customerRouter' -import {makeUlid} from '@openint/util' -import {createServerComponentHelpers} from '@/lib-server/server-component-helpers' -import {SetCookieAndRedirect} from './(oauth)/redirect/SetCookieAndRedirect' -import {kConnectSession, type ConnectSession} from './shared' - export const metadata = { - title: 'OpenInt Connect', + title: 'OpenInt Connect (Deprecated)', } -/** - * Workaround for searchParams being empty on production. Will ahve to check - * @see https://github.com/vercel/next.js/issues/43077#issuecomment-1383742153 - */ -export const dynamic = 'force-dynamic' - -// Should we allow page to optionally load without token for performance then add token async -// Perhaps it would even be an advantage to have the page simply be static? -// Though that would result in waterfall loading of integrations - -/** https://beta.nextjs.org/docs/api-reference/file-conventions/page#searchparams-optional */ -export default async function ConnectPageContainer({ - searchParams, -}: { - // Only accessible in PageComponent rather than layout component - // @see https://github.com/vercel/next.js/issues/43704 - searchParams: Record -}) { - const {token, ...params} = zConnectPageParams.parse(searchParams) - const {ssg, viewer} = await createServerComponentHelpers({ - searchParams: {[kAccessToken]: token}, - }) - if (viewer.role !== 'customer') { - return ( -
Authenticated user only. Your role is {getViewerId(viewer)}
- ) - } - - // Implement shorthand for specifying only connectorConfigId by connectorName - let connectorConfigId = params.connectorConfigId - if (!connectorConfigId && params.connectorNames) { - let ints = await ssg.listConnectorConfigInfos.fetch({ - connectorName: params.connectorNames, - }) - if (params.connectorConfigDisplayName) { - ints = ints.filter( - (int) => int.displayName === params.connectorConfigDisplayName, - ) - } - if (ints.length === 1 && ints[0]?.id) { - connectorConfigId = ints[0]?.id - } else if (ints.length < 1) { - return ( -
No connector config for {params.connectorNames} configured
- ) - } else if (ints.length > 1) { - console.warn( - `${ints.length} connector configs found for ${params.connectorNames}`, - ) - } - } - - // Special case when we are handling a single oauth connector config - if (connectorConfigId) { - const connectorName = extractConnectorName(connectorConfigId) - const intDef = defConnectors[ - connectorName as keyof typeof defConnectors - ] as ConnectorDef - - if (intDef.metadata?.nangoProvider) { - const connectionId = makeId('conn', connectorName, makeUlid()) - const url = await NangoConnect.getOauthConnectUrl({ - public_key: envRequired.NEXT_PUBLIC_NANGO_PUBLIC_KEY, - connection_id: connectionId, - provider_config_key: connectorConfigId, - // Consider using hookdeck so we can work with any number of urls - // redirect_uri: joinPath(getServerUrl(null), '/connect/callback'), - }) - return ( - - ) - } - } - - await Promise.all([ - // clerkClient.organizations.getOrganization({organizationId: viewer.orgId}), - // Switch to using react suspense / server fetch for this instead of prefetch - ssg.listConnectorConfigInfos.prefetch({ - id: connectorConfigId, - connectorName: params.connectorNames, - }), - params.showExisting ? ssg.listConnections.prefetch({}) : Promise.resolve(), - ]) - +export default async function ConnectPageContainer() { return (
/connect is deprecated. Please use /connect/portal instead diff --git a/connectors/connector-google/package.json b/connectors/connector-google/package.json index 83bfb7ef..e3700545 100644 --- a/connectors/connector-google/package.json +++ b/connectors/connector-google/package.json @@ -6,7 +6,8 @@ "dependencies": { "@openint/cdk": "workspace:*", "@openint/util": "workspace:*", - "@opensdks/runtime": "^0.0.19" + "@opensdks/runtime": "^0.0.19", + "@opensdks/sdk-google": "*" }, "devDependencies": {} } diff --git a/connectors/connector-google/server.ts b/connectors/connector-google/server.ts index 8b1a6412..e3cb4f54 100644 --- a/connectors/connector-google/server.ts +++ b/connectors/connector-google/server.ts @@ -1,4 +1,10 @@ -import {extractId, initNangoSDK, type ConnectorServer} from '@openint/cdk' +import {initGoogleSDK} from '@opensdks/sdk-google' +import { + extractId, + initNangoSDK, + nangoProxyLink, + type ConnectorServer, +} from '@openint/cdk' import type {googleSchemas} from './def' function mergeScopes( @@ -64,28 +70,29 @@ const integrations = [ ] export const googleServer = { - // newInstance: ({settings, fetchLinks}) => { - // const sdk = initHubspotSDK({ - // // We rely on nango to refresh the access token... - // headers: { - // authorization: `Bearer ${settings.oauth.credentials.access_token}`, - // }, - // links: (defaultLinks) => [ - // (req, next) => { - // if (sdk.clientOptions.baseUrl) { - // req.headers.set( - // nangoProxyLink.kBaseUrlOverride, - // sdk.clientOptions.baseUrl, - // ) - // } - // return next(req) - // }, - // ...fetchLinks, - // ...defaultLinks, - // ], - // }) - // return sdk - // }, + newInstance: ({settings, fetchLinks}) => { + const sdk = initGoogleSDK({ + // We rely on nango to refresh the access token... + headers: { + authorization: `Bearer ${settings.oauth.credentials.access_token}`, + }, + links: (defaultLinks) => [ + (req, next) => { + // TODO: make this dynamic to different base URLs than just drive_v2 + if (sdk.drive_v2.clientOptions.baseUrl) { + req.headers.set( + nangoProxyLink.kBaseUrlOverride, + sdk.drive_v2.clientOptions.baseUrl, + ) + } + return next(req) + }, + ...fetchLinks, + ...defaultLinks, + ], + }) + return sdk + }, // passthrough: (instance, input) => // instance.request(input.method, input.path, { // headers: input.headers as Record, diff --git a/kits/sdk/openapi.json b/kits/sdk/openapi.json index 233f2ce5..0ae41bbd 100644 --- a/kits/sdk/openapi.json +++ b/kits/sdk/openapi.json @@ -129,18 +129,6 @@ "add-deeplink" ], "description": "Magic Link tab view" - }, - "connectorConfigDisplayName": { - "type": ["string", "null"], - "description": "Filter connector config by displayName " - }, - "connectorConfigId": { - "type": "string", - "description": "Must start with 'ccfg_'" - }, - "showExisting": { - "type": "boolean", - "default": true } } } @@ -7564,7 +7552,7 @@ }, { "in": "query", - "name": "driveGroupId", + "name": "drive_group_id", "schema": { "type": "string" } @@ -7659,14 +7647,14 @@ }, { "in": "query", - "name": "driveId", + "name": "drive_id", "schema": { "type": "string" } }, { "in": "query", - "name": "folderId", + "name": "folder_id", "schema": { "type": "string" } @@ -7942,7 +7930,7 @@ }, { "in": "query", - "name": "driveId", + "name": "drive_id", "schema": { "type": "string" } @@ -10024,7 +10012,8 @@ }, "created_at": { "type": ["string", "null"] - } + }, + "raw_data": {} }, "required": ["id", "name"], "description": "A unified representation of a drive group" @@ -10046,7 +10035,8 @@ }, "created_at": { "type": ["string", "null"] - } + }, + "raw_data": {} }, "required": ["id", "name"], "description": "A unified representation of a storage drive" @@ -10102,7 +10092,8 @@ }, "created_at": { "type": ["string", "null"] - } + }, + "raw_data": {} }, "required": ["id", "type", "downloadable", "permissions", "exportable"], "description": "A unified representation of a file" @@ -10127,7 +10118,8 @@ }, "created_at": { "type": ["string", "null"] - } + }, + "raw_data": {} }, "required": ["id", "name", "path"], "description": "A unified representation of a folder" diff --git a/kits/sdk/openapi.types.d.ts b/kits/sdk/openapi.types.d.ts index ac17928d..92a5093d 100644 --- a/kits/sdk/openapi.types.d.ts +++ b/kits/sdk/openapi.types.d.ts @@ -1300,6 +1300,7 @@ export interface components { description?: string | null updated_at?: string | null created_at?: string | null + raw_data?: unknown } /** @description A unified representation of a storage drive */ 'unified.drive': { @@ -1308,6 +1309,7 @@ export interface components { description?: string | null updated_at?: string | null created_at?: string | null + raw_data?: unknown } /** @description A unified representation of a file */ 'unified.file': { @@ -1327,6 +1329,7 @@ export interface components { export_formats?: string[] | null updated_at?: string | null created_at?: string | null + raw_data?: unknown } /** @description A unified representation of a folder */ 'unified.folder': { @@ -1336,6 +1339,7 @@ export interface components { path: string updated_at?: string | null created_at?: string | null + raw_data?: unknown } } responses: never @@ -1496,12 +1500,6 @@ export interface operations { * @enum {string|null} */ view?: 'manage' | 'manage-deeplink' | 'add' | 'add-deeplink' - /** @description Filter connector config by displayName */ - connectorConfigDisplayName?: string | null - /** @description Must start with 'ccfg_' */ - connectorConfigId?: string - /** @default true */ - showExisting?: boolean } } } @@ -5772,7 +5770,7 @@ export interface operations { sync_mode?: 'full' | 'incremental' cursor?: string | null page_size?: number - driveGroupId?: string + drive_group_id?: string } } responses: { @@ -5813,8 +5811,8 @@ export interface operations { sync_mode?: 'full' | 'incremental' cursor?: string | null page_size?: number - driveId?: string - folderId?: string + drive_id?: string + folder_id?: string } } responses: { @@ -5960,7 +5958,7 @@ export interface operations { sync_mode?: 'full' | 'incremental' cursor?: string | null page_size?: number - driveId?: string + drive_id?: string } } responses: { diff --git a/packages/api/createRouterHandler.ts b/packages/api/createRouterHandler.ts index 4638397c..05d5a095 100644 --- a/packages/api/createRouterHandler.ts +++ b/packages/api/createRouterHandler.ts @@ -12,12 +12,20 @@ import { kApikeyUrlParam, } from '@openint/app-config/constants' import type {Id, UserId, Viewer} from '@openint/cdk' -import {decodeApikey, makeJwtClient, zCustomerId, zId} from '@openint/cdk' +import { + decodeApikey, + getRemoteContext, + makeJwtClient, + zCustomerId, + zId, +} from '@openint/cdk' import type {RouterContext} from '@openint/engine-backend' import {envRequired} from '@openint/env' +import {downloadFileById} from '@openint/unified-file-storage/adapters' import { BadRequestError, getHTTPResponseFromError, + getProtectedContext, isHttpError, TRPCError, z, @@ -164,7 +172,6 @@ export const contextFromRequest = async ({ } console.log('[contextFromRequest]', {url: req.url, viewer, connectionId}) return { - resHeaders: new Headers(), ...context, remoteConnectionId: connectionId ?? null, } @@ -191,6 +198,123 @@ export function createRouterTRPCHandler({ } } +type HttpMethod = + | 'GET' + | 'POST' + | 'PUT' + | 'DELETE' + | 'PATCH' + | 'OPTIONS' + | 'HEAD' + +type SkipTrpcRoutes = { + [method in HttpMethod]?: { + [route: string]: (req: Request, ctx: RouterContext) => Promise + } +} + +const skipTrpcRoutes: SkipTrpcRoutes = { + GET: { + '/api/v0/unified/file-storage/files/{fileId}/download': async ( + req: Request, + ctx: RouterContext, + ) => { + if (!ctx.remoteConnectionId) { + throw new BadRequestError('No Connection Found') + } + const connection = await ctx.services.metaService.tables.connection.get( + ctx.remoteConnectionId, + ) + if (!connection) { + throw new BadRequestError('No Connection Found For Download') + } + + const urlParts = new URL(req.url).pathname.split('/') + const fileId = urlParts.filter((part) => part).slice(-2, -1)[0] + + if (!fileId) { + throw new BadRequestError('No fileId found in path') + } + + if (!(connection.connectorName in downloadFileById)) { + throw new BadRequestError( + `Download not supported for ${connection.connectorName}`, + ) + } + const downloadFn = + downloadFileById[ + connection.connectorName as keyof typeof downloadFileById + ] + + // TODO: abstract so its not fetched in every handler + const protectedContext = getProtectedContext(ctx) + const remoteContext = await getRemoteContext(protectedContext) + + const {resHeaders, status, error, stream} = await downloadFn({ + fileId, + ctx: remoteContext, + }) + + if (status !== 200 || error || !stream) { + return new Response(JSON.stringify(error), { + status, + }) + } + + return new Response(stream, {status, headers: resHeaders}) + }, + '/api/v0/unified/file-storage/files/{fileId}/export': async ( + req: Request, + ctx: RouterContext, + ) => { + if (!ctx.remoteConnectionId) { + throw new BadRequestError('No Connection Found') + } + const connection = await ctx.services.metaService.tables.connection.get( + ctx.remoteConnectionId, + ) + if (!connection) { + throw new BadRequestError('No Connection Found For Download') + } + + const urlParts = new URL(req.url).pathname.split('/') + const fileId = urlParts.filter((part) => part).slice(-2, -1)[0] + + if (!fileId) { + throw new BadRequestError('No fileId found in path') + } + + if (!(connection.connectorName in downloadFileById)) { + throw new BadRequestError( + `Export not supported for ${connection.connectorName}`, + ) + } + const downloadFn = + downloadFileById[ + connection.connectorName as keyof typeof downloadFileById + ] + + // TODO: abstract so its not fetched in every handler + const protectedContext = getProtectedContext(ctx) + const remoteContext = await getRemoteContext(protectedContext) + + const {resHeaders, status, error, stream} = await downloadFn({ + fileId, + ctx: remoteContext, + exportFormat: new URL(req.url).searchParams.get('format') || undefined, + }) + + if (status !== 200 || error || !stream) { + return new Response(JSON.stringify(error), { + status, + }) + } + + return new Response(stream, {status, headers: resHeaders}) + }, + }, +} + export function createRouterOpenAPIHandler({ endpoint, router, @@ -216,6 +340,21 @@ export function createRouterOpenAPIHandler({ // Now handle for reals try { const context = await contextFromRequest({req}) + + const skipRoutes = + skipTrpcRoutes[req.method as keyof typeof skipTrpcRoutes] + if (skipRoutes) { + const pathname = new URL(req.url).pathname + for (const [route, handler] of Object.entries(skipRoutes)) { + const regex = new RegExp( + '^' + route.replace(/{[^}]+}/g, '[^/]+') + '$', + ) + if (regex.test(pathname)) { + console.log('skipping trpc route', route) + return handler(req, context) + } + } + } // More aptly named handleOpenApiFetchRequest as it returns a response already const res = await createOpenApiFetchHandler({ endpoint, @@ -224,7 +363,8 @@ export function createRouterOpenAPIHandler({ createContext: () => context, // TODO: handle error status code from passthrough endpoints // onError, // can only have side effect and not modify response error status code unfortunately... - responseMeta: ({errors, ctx: _ctx}) => { + responseMeta: ({errors, ctx: _ctx, data}) => { + console.log('res data', data) // Pass the status along for (const err of errors) { console.warn( @@ -250,12 +390,14 @@ export function createRouterOpenAPIHandler({ } for (const [k, v] of context.resHeaders.entries()) { + console.log('setting resHeader header', k, v) res.headers.set(k, v) } for (const [k, v] of Object.entries(corsHeaders)) { res.headers.set(k, v) } + return res } catch (err) { console.error('[trpc.createRouterHandler] error', err) diff --git a/packages/api/package.json b/packages/api/package.json index ec52a5bb..3622278b 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -22,11 +22,11 @@ "@openint/unified-ats": "workspace:*", "@openint/unified-banking": "workspace:*", "@openint/unified-crm": "workspace:*", - "@openint/unified-sync": "workspace:*", "@openint/unified-file-storage": "workspace:*", "@openint/unified-hris": "workspace:*", "@openint/unified-pta": "workspace:*", "@openint/unified-sales-engagement": "workspace:*", + "@openint/unified-sync": "workspace:*", "@openint/vdk": "workspace:*", "@opensdks/fetch-links": "^0.0.19", "@opensdks/util-zod": "0.0.15", diff --git a/packages/api/proxyHandler.ts b/packages/api/proxyHandler.ts index 775edba4..faec25e6 100644 --- a/packages/api/proxyHandler.ts +++ b/packages/api/proxyHandler.ts @@ -56,16 +56,19 @@ export const proxyHandler = async (req: Request) => { } const connectorImplementedProxy = remoteContext.remote.connector.proxy - let res: Response | null = null + let response: Response | null = null if (connectorImplementedProxy) { - res = await connectorImplementedProxy(remoteContext.remote.instance, req) + response = await connectorImplementedProxy( + remoteContext.remote.instance, + req, + ) } else if (isOpenAPIClient(remoteContext.remote.instance)) { const url = new URL(req.url) const prefix = url.protocol + '//' + url.host + '/api/proxy' const newUrl = req.url.replace(prefix, '') - res = await remoteContext.remote.instance + response = await remoteContext.remote.instance .request(req.method as HTTPMethod, newUrl, { body: req.body, headers: req.headers, @@ -73,7 +76,7 @@ export const proxyHandler = async (req: Request) => { .then((r) => r.response) } - if (!res) { + if (!response) { return new Response( `Proxy not supported for connection: ${remoteContext.remoteConnectionId}`, { @@ -83,13 +86,13 @@ export const proxyHandler = async (req: Request) => { } // TODO: move to stream based response - const resBody = await res.blob() + const resBody = await response.blob() - const headers = new Headers(res.headers) + const headers = new Headers(response.headers) headers.delete('content-encoding') // No more gzip at this point... headers.set('content-length', resBody.size.toString()) return new Response(resBody, { - status: res.status, + status: response.status, headers, }) } diff --git a/packages/engine-backend/context.ts b/packages/engine-backend/context.ts index ed6aad61..ce393b2c 100644 --- a/packages/engine-backend/context.ts +++ b/packages/engine-backend/context.ts @@ -117,6 +117,7 @@ export function getContextFactory< getRedirectUrl, clerk: config.clerk, inngest, + resHeaders: new Headers(), } } diff --git a/packages/engine-backend/router/connectorRouter.ts b/packages/engine-backend/router/connectorRouter.ts index 341a782d..bf160f46 100644 --- a/packages/engine-backend/router/connectorRouter.ts +++ b/packages/engine-backend/router/connectorRouter.ts @@ -249,10 +249,12 @@ export const connectorRouter = trpc.mergeRouters( .filter((int) => { const connectorNameMatches = !input.connectorNames || + input.connectorNames.length === 0 || input.connectorNames.includes(int.connector_name) const integrationMatches = - !input.integrationIds?.length || + !input.integrationIds || + input.integrationIds.length === 0 || input.integrationIds.some((filter) => int.id.includes(filter)) // Check if this integration is already connected diff --git a/packages/engine-backend/router/customerRouter.ts b/packages/engine-backend/router/customerRouter.ts index 4cd151cd..70422ea1 100644 --- a/packages/engine-backend/router/customerRouter.ts +++ b/packages/engine-backend/router/customerRouter.ts @@ -59,14 +59,6 @@ export const zConnectPageParams = z.object({ .enum(['manage', 'manage-deeplink', 'add', 'add-deeplink']) .nullish() .describe('Magic Link tab view'), - connectorConfigDisplayName: z - .string() - .nullish() - .describe('Filter connector config by displayName '), - /** Launch the conector with config right away */ - connectorConfigId: zId('ccfg').optional(), - /** Whether to show existing connections */ - showExisting: z.coerce.boolean().optional().default(true), }) /** @@ -181,11 +173,15 @@ export const customerRouter = trpc.router({ connectorNames: params.connectorNames ?.split(',') .map((name) => name.trim()), + theme: params.theme ?? 'light', + view: params.view ?? 'add', } const url = new URL('/connect/portal', ctx.apiUrl) // `/` will start from the root hostname itself for (const [key, value] of Object.entries(mappedParams)) { - url.searchParams.set(key, `${value ?? ''}`) + if (value) { + url.searchParams.set(key, `${value ?? ''}`) + } } return {url: url.toString()} }), diff --git a/packages/trpc/package.json b/packages/trpc/package.json index a8da707c..a38b5103 100644 --- a/packages/trpc/package.json +++ b/packages/trpc/package.json @@ -11,6 +11,5 @@ "@opensdks/runtime": "0.0.19", "@opensdks/util-zod": "0.0.15", "@trpc/server": "10.40.0" - }, - "devDependencies": {} + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2b28378..02fb3eaf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -879,6 +879,9 @@ importers: '@opensdks/runtime': specifier: ^0.0.19 version: 0.0.19 + '@opensdks/sdk-google': + specifier: '*' + version: 0.0.1 connectors/connector-greenhouse: dependencies: @@ -1639,9 +1642,6 @@ importers: '@openint/unified-crm': specifier: workspace:* version: link:../../unified/unified-crm - '@openint/unified-sync': - specifier: workspace:* - version: link:../../unified/unified-sync '@openint/unified-file-storage': specifier: workspace:* version: link:../../unified/unified-file-storage @@ -1654,6 +1654,9 @@ importers: '@openint/unified-sales-engagement': specifier: workspace:* version: link:../../unified/unified-sales-engagement + '@openint/unified-sync': + specifier: workspace:* + version: link:../../unified/unified-sync '@openint/vdk': specifier: workspace:* version: link:../../kits/vdk @@ -2361,20 +2364,14 @@ importers: specifier: 7.5.6 version: 7.5.6 - unified/unified-sync: - dependencies: - '@openint/connector-revert': - specifier: workspace:* - version: link:../../connectors/connector-revert - '@openint/vdk': - specifier: workspace:* - version: link:../../kits/vdk - unified/unified-file-storage: dependencies: '@openint/cdk': specifier: workspace:* version: link:../../kits/cdk + '@openint/connector-google': + specifier: workspace:^ + version: link:../../connectors/connector-google '@openint/connector-greenhouse': specifier: workspace:* version: link:../../connectors/connector-greenhouse @@ -2384,12 +2381,19 @@ importers: '@openint/vdk': specifier: workspace:* version: link:../../kits/vdk + '@opensdks/sdk-google': + specifier: '*' + version: 0.0.1 '@opensdks/sdk-msgraph': specifier: ^0.0.1 version: 0.0.1 rxjs: specifier: 7.5.6 version: 7.5.6 + devDependencies: + typescript: + specifier: ^5.0.0 + version: 5.6.2 unified/unified-hris: dependencies: @@ -2431,6 +2435,15 @@ importers: specifier: 0.0.17 version: 0.0.17 + unified/unified-sync: + dependencies: + '@openint/connector-revert': + specifier: workspace:* + version: link:../../connectors/connector-revert + '@openint/vdk': + specifier: workspace:* + version: link:../../kits/vdk + packages: '@adobe/css-tools@4.4.0': @@ -3881,6 +3894,9 @@ packages: '@opensdks/sdk-finch@0.0.4': resolution: {integrity: sha512-7CxSxCIE8POPBFrEGlovRdkziXNfbrh6GD7fPwIk6GPR2f2CKmLqPxOpLhBhzH/5Ol0nCyzCq8NDsvLbI5OvEQ==} + '@opensdks/sdk-google@0.0.1': + resolution: {integrity: sha512-Qn9r9Yew2RMiL5LK0jU4+BoP5/Q3o2u80SELymytWv+9sVFN0wrTyIPYQO6mnzM7JWQR8150KbcGEnXjt6gL/Q==} + '@opensdks/sdk-greenhouse@0.0.8': resolution: {integrity: sha512-kIN0ckK36QmWUfUNbl7B6Fi8G63JQ/iCWKnY5Ozd+lTRQYQV+ub30FOZk0djiepNOcUkOFv3QSy9K1Sk8B4ADA==} @@ -15926,6 +15942,8 @@ snapshots: '@opensdks/sdk-finch@0.0.4': {} + '@opensdks/sdk-google@0.0.1': {} + '@opensdks/sdk-greenhouse@0.0.8': dependencies: '@opensdks/runtime': 0.0.20 @@ -20769,7 +20787,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.8.0(@typescript-eslint/parser@6.21.0(eslint@8.23.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.5.3(eslint-plugin-import@2.29.0)(eslint@8.23.0))(eslint@8.23.0): + eslint-module-utils@2.8.0(@typescript-eslint/parser@6.21.0(eslint@8.23.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.5.3)(eslint@8.23.0): dependencies: debug: 3.2.7 optionalDependencies: @@ -20814,7 +20832,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.23.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.21.0(eslint@8.23.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.5.3(eslint-plugin-import@2.29.0)(eslint@8.23.0))(eslint@8.23.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.21.0(eslint@8.23.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.5.3)(eslint@8.23.0) hasown: 2.0.0 is-core-module: 2.13.1 is-glob: 4.0.3 diff --git a/unified/unified-file-storage/adapters/google-adapter/index.ts b/unified/unified-file-storage/adapters/google-adapter/index.ts new file mode 100644 index 00000000..964694e2 --- /dev/null +++ b/unified/unified-file-storage/adapters/google-adapter/index.ts @@ -0,0 +1,219 @@ +import type {googleServer} from '@openint/connector-google' +import {TRPCError} from '@openint/vdk' +import {type FileStorageAdapter} from '../../router' +import {mappers} from './mapper' + +interface DownloadFileResult { + status: number + stream: ReadableStream | null + resHeaders?: Headers + error: {code: string; message: string} | null +} + +export async function downloadFileById({ + fileId, + ctx, + exportFormat, +}: { + fileId: string + ctx: any + exportFormat?: string +}): Promise { + try { + const endpoint = exportFormat + ? '/files/{fileId}/export' + : '/files/{fileId}/download' + const queryParams = exportFormat ? {mimeType: exportFormat} : undefined + + const res = await ctx.remote.instance.drive_v3.GET(endpoint, { + params: { + path: {fileId}, + query: queryParams, + }, + parseAs: 'blob', + }) + + if (!res.data) { + return { + status: 404, + stream: null, + error: { + code: 'NOT_FOUND', + message: exportFormat + ? 'File not found or cannot be exported. Please see select an export_format returned with the file.' + : 'File not found', + }, + } + } + + const headers = new Headers() + // Set response headers from Google Drive response + if (res.response.headers) { + headers.set('content-type', res.response.headers['content-type'] ?? '') + headers.set( + 'content-length', + res.response.headers['content-length'] ?? '', + ) + headers.set( + 'content-disposition', + res.response.headers['content-disposition'] ?? '', + ) + headers.set( + 'transfer-encoding', + res.response.headers['transfer-encoding'], + ) + headers.set('content-encoding', res.response.headers['content-encoding']) + } + + return { + status: 200, + stream: res.response.body, + resHeaders: headers, + error: null, + } + } catch (error: any) { + return { + status: error.message.includes('404') ? 404 : 500, + stream: null, + error: { + code: error.message.includes('404') + ? 'NOT_FOUND' + : 'INTERNAL_SERVER_ERROR', + message: error.message || 'Failed to download file', + }, + } + } +} + +export const googleAdapter: FileStorageAdapter< + ReturnType +> = { + listDriveGroups: async () => { + return { + has_next_page: false, + items: [], + next_cursor: null, + } + }, + + listDrives: async ({instance, input}) => { + const res = await instance.drive_v3.GET('/drives', { + params: { + query: { + pageSize: input?.page_size, + pageToken: input?.cursor || undefined, + }, + }, + }) + + if (!res.data) { + throw new TRPCError({code: 'NOT_FOUND', message: 'No drives found'}) + } + + return { + has_next_page: false, + items: [], + next_cursor: null, + } + }, + + listFiles: async ({instance, input}) => { + const res = await instance.drive_v3.GET('/files', { + params: { + query: { + q: input?.folder_id ? `'${input.folder_id}' in parents` : undefined, + pageSize: input?.page_size, + pageToken: input?.cursor || undefined, + fields: '*', + }, + }, + }) + + if (!res.data) { + throw new TRPCError({code: 'NOT_FOUND', message: 'No files found'}) + } + + return { + has_next_page: res.data.incompleteSearch || false, + items: res.data.files?.map(mappers.File) || [], + next_cursor: res.data.nextPageToken || null, + } + }, + + getFile: async ({instance, input}) => { + const res = await instance.drive_v3.GET('/files/{fileId}', { + params: { + path: {fileId: input.id}, + query: { + fields: '*', + }, + }, + }) + + if (!res.data) { + throw new TRPCError({code: 'NOT_FOUND', message: 'File not found'}) + } + + return mappers.File(res.data) + }, + + exportFile: async ({instance, input}) => { + const {stream, status, error} = await downloadFileById({ + fileId: input.id, + ctx: {remote: {instance}}, + exportFormat: input.format, + }) + + if (status !== 200 || error || !stream) { + throw new TRPCError({ + code: (error?.code as any) ?? 'INTERNAL_SERVER_ERROR', + message: error?.message ?? 'Failed to export file', + }) + } + + return stream + }, + + downloadFile: async ({instance, input, ctx}) => { + const {resHeaders, stream, status, error} = await downloadFileById({ + fileId: input.id, + ctx: {remote: {instance}}, + }) + + if (status !== 200 || error || !stream) { + throw new TRPCError({ + code: (error?.code as any) ?? 'INTERNAL_SERVER_ERROR', + message: error?.message ?? 'Failed to download file', + }) + } + + resHeaders?.forEach((value, key) => { + ctx.resHeaders.set(key, value) + }) + + return stream + }, + + listFolders: async ({instance, input}) => { + const res = await instance.drive_v3.GET('/files', { + params: { + query: { + q: "mimeType='application/vnd.google-apps.folder'", + pageSize: input?.page_size, + pageToken: input?.cursor || undefined, + fields: '*', + }, + }, + }) + + if (!res.data) { + throw new TRPCError({code: 'NOT_FOUND', message: 'No folders found'}) + } + + return { + has_next_page: res.data.incompleteSearch || false, + items: res.data.files?.map(mappers.Folder) || [], + next_cursor: res.data.nextPageToken || null, + } + }, +} diff --git a/unified/unified-file-storage/adapters/google-adapter/mapper.ts b/unified/unified-file-storage/adapters/google-adapter/mapper.ts new file mode 100644 index 00000000..2dc38a74 --- /dev/null +++ b/unified/unified-file-storage/adapters/google-adapter/mapper.ts @@ -0,0 +1,61 @@ +import {Oas_drive_v3} from '@opensdks/sdk-google' +import {mapper, zCast} from '@openint/vdk' +import * as unified from '../../unifiedModels' + +// 2025-01-22 SDK schemas: About|Change|ChangeList|Channel|Comment|CommentList|ContentRestriction|Drive|DriveList|File|FileList|GeneratedIds|Label|LabelField|LabelFieldModification|LabelList|LabelModification|ModifyLabelsRequest|ModifyLabelsResponse|Permission|PermissionList|Reply|ReplyList|Revision|RevisionList|StartPageToken|TeamDrive|TeamDriveList|User +type AdapterTypes = Oas_drive_v3['components']['schemas'] + +export const mappers: Record< + keyof typeof unified, + ReturnType +> = { + DriveGroup: mapper(zCast(), unified.DriveGroup, { + id: (record) => record.id || '', + name: (record) => record.name || '', + description: (record) => record.description || null, + updated_at: (record) => record.modifiedTime || null, + created_at: (record) => record.createdTime || null, + raw_data: (record) => record, + }), + Drive: mapper(zCast(), unified.Drive, { + id: (record) => record.id || '', + name: (record) => record.name || '', + description: (record) => record.description || null, + updated_at: (record) => record.modifiedTime || null, + created_at: (record) => record.createdTime || null, + raw_data: (record) => record, + }), + File: mapper(zCast(), unified.File, { + id: (record) => record.id || '', + name: (record) => record.name || null, + description: (record) => record.description || null, + type: (record) => + record.mimeType === 'application/vnd.google-apps.folder' + ? 'folder' + : 'file', + path: (record) => record.parents?.[0] || null, + mime_type: (record) => record.mimeType || null, + downloadable: (record) => + !!record.capabilities?.canDownload && !!record.webContentLink, + size: (record) => (record.size ? parseInt(record.size) : null), + permissions: (record) => ({ + download: !!record.capabilities?.canDownload, + }), + exportable: (record) => + !!record.exportLinks && Object.keys(record.exportLinks).length > 0, + export_formats: (record) => + record.exportLinks ? Object.keys(record.exportLinks) : null, + updated_at: (record) => record.modifiedTime || null, + created_at: (record) => record.createdTime || null, + raw_data: (record) => record, + }), + Folder: mapper(zCast(), unified.Folder, { + id: (record) => record.id || '', + name: (record) => record.name || '', + description: (record) => record.description || null, + path: (record) => record.parents?.[0] || '', + updated_at: (record) => record.modifiedTime || null, + created_at: (record) => record.createdTime || null, + raw_data: (record) => record, + }), +} diff --git a/unified/unified-file-storage/adapters/index.ts b/unified/unified-file-storage/adapters/index.ts index d333661c..bcd8b066 100644 --- a/unified/unified-file-storage/adapters/index.ts +++ b/unified/unified-file-storage/adapters/index.ts @@ -1,6 +1,19 @@ import type {AdapterMap} from '@openint/vdk' -import {sharepointAdapter} from './sharepoint-adapter' +import { + googleAdapter, + downloadFileById as googleDownloadFileById, +} from './google-adapter' +import { + sharepointAdapter, + downloadFileById as sharepointDownloadFileById, +} from './sharepoint-adapter' export default { microsoft: sharepointAdapter, + google: googleAdapter, } satisfies AdapterMap + +export const downloadFileById = { + microsoft: sharepointDownloadFileById, + google: googleDownloadFileById, +} diff --git a/unified/unified-file-storage/adapters/sharepoint-adapter/index.ts b/unified/unified-file-storage/adapters/sharepoint-adapter/index.ts index 34a0da35..929aca52 100644 --- a/unified/unified-file-storage/adapters/sharepoint-adapter/index.ts +++ b/unified/unified-file-storage/adapters/sharepoint-adapter/index.ts @@ -21,6 +21,140 @@ function extractCursor(nextLink?: string): string | undefined { } } +async function getFileFromDrives({ + instance, + input, + ctx, +}: { + instance: MsgraphSDK + input: {id: string; cursor?: string} + ctx: any +}) { + const drivesResult: any = await sharepointAdapter.listDrives({ + instance, + input: {}, + ctx, + }) + + const filePromises = drivesResult.items.map(async (drive: any) => { + try { + const response = await instance.GET( + '/drives/{drive-id}/items/{driveItem-id}', + { + params: { + path: { + 'drive-id': drive.id, + 'driveItem-id': input.id, + }, + // @ts-expect-error TODO: "$expandParams is supported by the API but its not clear in the documentation + query: { + ...expandParams, + }, + }, + }, + ) + return response.data + } catch (error: any) { + // TODO: fix nesting in this SDK + if (error?.error?.error?.code === 'itemNotFound') { + return null + } + throw error + } + }) + + const results = await Promise.all(filePromises) + const validResult = results.find( + (res) => res && res.id && res.id !== 'undefined', + ) + + if (!validResult) { + throw new TRPCError({ + code: 'NOT_FOUND', + message: 'File not found in any drive', + }) + } + + return validResult +} + +interface DownloadFileResult { + status: number + stream: ReadableStream | null + resHeaders?: Headers + error: {code: string; message: string} | null +} + +export async function downloadFileById({ + fileId, + ctx, +}: { + fileId: string + ctx: any +}): Promise { + const file = await getFileFromDrives({ + instance: ctx.remote.instance, + input: {id: fileId}, + ctx, + }) + + if (!file['@microsoft.graph.downloadUrl']) { + return { + status: 404, + stream: null, + error: { + code: 'NOT_FOUND', + message: 'File not downloadable', + }, + } + } + + const downloadResponse = await fetch(file['@microsoft.graph.downloadUrl']) + + if (!downloadResponse.ok) { + return { + status: 500, + stream: null, + error: { + code: 'INTERNAL_SERVER_ERROR', + message: 'Failed to download file', + }, + } + } + + if (!downloadResponse.body) { + return { + status: 500, + stream: null, + error: { + code: 'INTERNAL_SERVER_ERROR', + message: 'No download response body available', + }, + } + } + + const headers = new Headers() + headers.set( + 'content-type', + downloadResponse.headers.get('content-type') ?? '', + ) + headers.set( + 'content-length', + downloadResponse.headers.get('content-length') ?? '', + ) + headers.set( + 'content-disposition', + downloadResponse.headers.get('content-disposition') ?? '', + ) + + return { + status: 200, + stream: downloadResponse.body, + resHeaders: headers, + error: null, + } +} + export const sharepointAdapter = { listDriveGroups: async ({instance, input}) => { const res = await fetch( @@ -50,7 +184,7 @@ export const sharepointAdapter = { }, listDrives: async ({instance, input}) => { - const siteId = input?.driveGroupId + const siteId = input?.drive_group_id let drivesResponse if (siteId) { @@ -120,20 +254,20 @@ export const sharepointAdapter = { } }, - listFiles: async ({instance, input}) => { + listFiles: async ({instance, input, ctx}) => { let filesResponse - if (input?.driveId) { - // Use instance.GET for cases with driveId - filesResponse = await instance.GET( - input.folderId - ? `/drives/{drive-id}/items/{driveItem-id}/children` - : `/drives/{drive-id}/root/children`, - { + if (input?.drive_id) { + const endpoint = input.folder_id + ? `/drives/{drive-id}/items/{driveItem-id}/children` + : `/drives/{drive-id}/root/children` + + try { + filesResponse = await instance.GET(endpoint, { params: { path: { - 'drive-id': input.driveId, - 'driveItem-id': input.folderId, + 'drive-id': input.drive_id, + 'driveItem-id': input.folder_id, }, query: { ...expandParams, @@ -141,28 +275,55 @@ export const sharepointAdapter = { $skiptoken: input?.cursor, }, }, - }, - ) - filesResponse = filesResponse.data + }) + filesResponse = filesResponse.data + } catch (error) { + throw error + } } else { - // Use fetch for cases without driveId - const response = await fetch( - `https://graph.microsoft.com/v1.0${ - input?.folderId - ? `/me/drive/items/${input.folderId}/children` - : '/me/drive/root/children' - }?${new URLSearchParams({ - ...expandParams, - ...(input?.cursor ? {$skiptoken: input.cursor} : {}), - })}`, - { - headers: { - // @ts-expect-error - Authorization: instance.clientOptions.headers?.authorization, - }, - }, - ) - filesResponse = await response.json() + if (input?.cursor) { + throw new TRPCError({ + code: 'BAD_REQUEST', + message: 'Pagination is only supported when specifying a drive_id', + }) + } + + const drivesResult = await sharepointAdapter.listDrives({ + instance, + input: {}, + ctx, + }) + + const allFilesPromises = drivesResult.items.map(async (drive: any) => { + try { + const endpoint = input?.folder_id + ? `/drives/{drive-id}/items/{driveItem-id}/children` + : `/drives/{drive-id}/root/children` + + const response = await instance.GET(endpoint, { + params: { + path: { + 'drive-id': drive.id, + 'driveItem-id': input?.folder_id, + }, + query: { + ...expandParams, + // @ts-expect-error TODO: "$skiptoken is supported by the API but its not clear in the documentation + $skiptoken: input?.cursor, + }, + }, + }) + return response.data.value || [] + } catch (error) { + return [] + } + }) + + const allFilesArrays: any[][] = await Promise.all(allFilesPromises) + filesResponse = { + value: allFilesArrays.flat(), + '@odata.nextLink': undefined, // Pagination not supported when searching all drives + } } if (!filesResponse.value) { @@ -177,82 +338,51 @@ export const sharepointAdapter = { items: filesResponse.value .filter((item: any) => item.file) .map(mappers.File), - cursor: extractCursor(filesResponse['@odata.nextLink']), + cursor: extractCursor(filesResponse['@odata.nextLink'] ?? ''), } }, - getFile: async ({instance, input}) => { - const response = await fetch( - `https://graph.microsoft.com/v1.0/me/drive/items/${ - input.id - }?${new URLSearchParams(expandParams)}`, - { - headers: { - // @ts-expect-error - Authorization: instance.clientOptions.headers?.authorization, - }, - }, - ) - const res = await response.json() - - if (!res) { - throw new TRPCError({ - code: 'NOT_FOUND', - message: 'File not found', - }) - } - - return mappers.File(res) + getFile: async ({instance, input, ctx}) => { + const file = await getFileFromDrives({instance, input, ctx}) + return mappers.File(file) }, - exportFile: async ({instance, input}) => { - const response = await fetch( - `https://graph.microsoft.com/v1.0/me/drive/items/${input.id}`, - { - headers: { - // @ts-expect-error - Authorization: instance.clientOptions.headers?.authorization, - }, - }, - ) - const res = await response.json() + exportFile: async () => { + throw new TRPCError({ + code: 'NOT_IMPLEMENTED', + message: 'Export file is not available in Sharepoint', + }) + }, - if (!res || !res['@microsoft.graph.downloadUrl']) { - throw new TRPCError({ - code: 'NOT_FOUND', - message: 'File not found or not downloadable', - }) - } + downloadFile: async ({input, ctx}) => { + const {resHeaders, stream, status, error} = await downloadFileById({ + fileId: input.id, + ctx, + }) - const downloadResponse = await fetch(res['@microsoft.graph.downloadUrl']) - if (!downloadResponse.ok) { + if (status !== 200 || error || !stream) { throw new TRPCError({ - code: 'INTERNAL_SERVER_ERROR', - message: 'Failed to download file', + // @ts-expect-error + code: error?.code ?? 'INTERNAL_SERVER_ERROR', + message: error?.message ?? 'Failed to download file', }) } - if (!downloadResponse.body) { - throw new TRPCError({ - code: 'INTERNAL_SERVER_ERROR', - message: 'No response body available', - }) - } - return downloadResponse.body - }, + resHeaders?.forEach((value, key) => { + ctx.resHeaders.set(key, value) + }) - downloadFile: async ({instance, input}) => { - return exports.microsoftGraphAdapter.exportFile({instance, input}) + return stream }, listFolders: async ({instance, input}) => { let foldersResponse - if (input?.driveId) { + if (input?.drive_id) { // Use instance.GET for cases with driveId const res = await instance.GET('/drives/{drive-id}/root/children', { params: { - path: {'drive-id': input.driveId}, + path: {'drive-id': input.drive_id}, query: { $filter: 'folder ne null', // @ts-expect-error TODO: "$skiptoken is supported by the API but its not clear in the documentation diff --git a/unified/unified-file-storage/adapters/sharepoint-adapter/mappers.ts b/unified/unified-file-storage/adapters/sharepoint-adapter/mappers.ts index 63b3009b..127d4fac 100644 --- a/unified/unified-file-storage/adapters/sharepoint-adapter/mappers.ts +++ b/unified/unified-file-storage/adapters/sharepoint-adapter/mappers.ts @@ -14,6 +14,7 @@ export const mappers = { description: 'description', updated_at: 'lastModifiedDateTime', created_at: 'createdDateTime', + raw_data: (record) => record, }, ), @@ -23,6 +24,7 @@ export const mappers = { description: 'description', updated_at: 'lastModifiedDateTime', created_at: 'createdDateTime', + raw_data: (record) => record, }), File: mapper( @@ -43,6 +45,7 @@ export const mappers = { export_formats: () => null, updated_at: 'lastModifiedDateTime', created_at: 'createdDateTime', + raw_data: (record) => record, }, ), @@ -56,6 +59,7 @@ export const mappers = { path: (record) => record.parentReference?.path || '/', updated_at: 'lastModifiedDateTime', created_at: 'createdDateTime', + raw_data: (record) => record, }, ), } diff --git a/unified/unified-file-storage/package.json b/unified/unified-file-storage/package.json index 87e78b4f..312ce1cc 100644 --- a/unified/unified-file-storage/package.json +++ b/unified/unified-file-storage/package.json @@ -1,14 +1,20 @@ { "name": "@openint/unified-file-storage", + "scripts": { + "typecheck": "tsc --noEmit" + }, "dependencies": { "@openint/cdk": "workspace:*", + "@openint/connector-google": "workspace:^", "@openint/connector-greenhouse": "workspace:*", "@openint/connector-lever": "workspace:*", "@openint/vdk": "workspace:*", - "rxjs": "7.5.6", - "@opensdks/sdk-msgraph": "^0.0.1" + "@opensdks/sdk-google": "*", + "@opensdks/sdk-msgraph": "^0.0.1", + "rxjs": "7.5.6" }, "devDependencies": { - "@openint/cdk": "workspace:*" + "@openint/cdk": "workspace:*", + "typescript": "^5.0.0" } } diff --git a/unified/unified-file-storage/router.spec.ts b/unified/unified-file-storage/router.spec.ts index 48034999..0b81b745 100644 --- a/unified/unified-file-storage/router.spec.ts +++ b/unified/unified-file-storage/router.spec.ts @@ -1,53 +1,245 @@ /* eslint-disable jest/no-standalone-expect */ +import {Blob} from 'buffer' import {forEachAdapterConnections as describeEachAdapterConnections} from '@openint/vdk/vertical-test-utils' import adapters from './adapters' import type {FileStorageAdapter} from './router' describeEachAdapterConnections>(adapters, (t) => { let testFileId: string + let testDriveId: string + let testFolderId: string + + if (t.adapterName === 'microsoft') { + return + } t.testIfImplemented('listDriveGroups', async () => { + // Test default pagination const res = await t.sdkForConn.GET('/unified/file-storage/drive-groups', {}) expect(res.data.items).toBeTruthy() expect(Array.isArray(res.data.items)).toBe(true) - if (res.data.items.length > 0) { - // const group = res.data.items[0] - // expect(group.id).toBeTruthy() - // expect(group.name).toBeTruthy() + + // Test with pagination params + const resWithPagination = await t.sdkForConn.GET( + '/unified/file-storage/drive-groups', + { + params: {query: {page_size: 5}}, + }, + ) + expect(resWithPagination.data.items.length).toBeLessThanOrEqual(5) + + // Test cursor-based pagination if we have items + if ( + resWithPagination.data.items.length === 5 && + resWithPagination.data.next_cursor + ) { + const resWithCursor = await t.sdkForConn.GET( + '/unified/file-storage/drive-groups', + { + params: { + query: { + cursor: resWithPagination.data.next_cursor, + page_size: 5, + }, + }, + }, + ) + expect(resWithCursor.data.items).toBeTruthy() + expect(Array.isArray(resWithCursor.data.items)).toBe(true) + } + + if (res.data.items.length > 0 && res.data.items[0]) { + const group = res.data.items[0] + expect(group.id).toBeTruthy() + expect(group.name).toBeTruthy() + expect(typeof group.name).toBe('string') + expect(group.raw_data as object).toBeTruthy() + expect(Object.keys(group.raw_data as object).length).toBeGreaterThan(0) } }) t.testIfImplemented('listDrives', async () => { + // First get a drive group ID for testing + let testDriveGroupId: string | undefined + const driveGroupsRes = await t.sdkForConn.GET( + '/unified/file-storage/drive-groups', + {}, + ) + if (driveGroupsRes.data.items.length > 0 && driveGroupsRes.data.items[0]) { + testDriveGroupId = driveGroupsRes.data.items[0].id + } + + // Test default listing const res = await t.sdkForConn.GET('/unified/file-storage/drives', {}) expect(res.data.items).toBeTruthy() expect(Array.isArray(res.data.items)).toBe(true) - if (res.data.items.length > 0) { - // const drive = res.data.items[0] - // expect(drive.id).toBeTruthy() - // expect(drive.name).toBeTruthy() + + // Test with pagination + const resWithPagination = await t.sdkForConn.GET( + '/unified/file-storage/drives', + { + params: {query: {page_size: 5}}, + }, + ) + expect(resWithPagination.data.items.length).toBeLessThanOrEqual(5) + + // Test cursor-based pagination if we have items + if ( + resWithPagination.data.items.length === 5 && + resWithPagination.data.next_cursor + ) { + const resWithCursor = await t.sdkForConn.GET( + '/unified/file-storage/drives', + { + params: { + query: { + cursor: resWithPagination.data.next_cursor, + page_size: 5, + }, + }, + }, + ) + expect(resWithCursor.data.items).toBeTruthy() + expect(Array.isArray(resWithCursor.data.items)).toBe(true) + } + + if (res.data.items.length > 0 && res.data.items[0]) { + const drive = res.data.items[0] + testDriveId = drive.id + expect(drive.id).toBeTruthy() + expect(drive.name).toBeTruthy() + expect(typeof drive.name).toBe('string') + expect(drive.raw_data as object).toBeTruthy() + expect(Object.keys(drive.raw_data as object).length).toBeGreaterThan(0) + } + + // Test with driveGroupId if we have one + if (testDriveGroupId) { + const resWithGroup = await t.sdkForConn.GET( + '/unified/file-storage/drives', + { + params: { + query: { + drive_group_id: testDriveGroupId, + page_size: 5, + }, + }, + }, + ) + expect(resWithGroup.data.items).toBeTruthy() + expect(Array.isArray(resWithGroup.data.items)).toBe(true) } }) t.testIfImplemented('listFiles', async () => { - const res = await t.sdkForConn.GET('/unified/file-storage/files', {}) + // Test default listing + const res = await t.sdkForConn.GET('/unified/file-storage/files') expect(res.data.items).toBeTruthy() expect(Array.isArray(res.data.items)).toBe(true) - if (res.data.items.length > 0) { - // const file = res.data.items[0] - // expect(file.id).toBeTruthy() - // expect(file.name).toBeTruthy() - // expect(file.type).toBeTruthy() - // testFileId = file.id + + // Test with pagination + const resWithPagination = await t.sdkForConn.GET( + '/unified/file-storage/files', + { + params: {query: {page_size: 3}}, + }, + ) + expect(resWithPagination.data.items.length).toBeLessThanOrEqual(3) + + // Test cursor-based pagination if we have items + if ( + resWithPagination.data.items.length === 3 && + resWithPagination.data.next_cursor + ) { + const resWithCursor = await t.sdkForConn.GET( + '/unified/file-storage/files', + { + params: { + query: { + cursor: resWithPagination.data.next_cursor, + page_size: 3, + }, + }, + }, + ) + expect(resWithCursor.data.items).toBeTruthy() + expect(Array.isArray(resWithCursor.data.items)).toBe(true) + } + + if (testDriveId) { + // Test files in specific drive + const resWithDrive = await t.sdkForConn.GET( + '/unified/file-storage/files', + { + params: {query: {drive_id: testDriveId}}, + }, + ) + expect(Array.isArray(resWithDrive.data.items)).toBe(true) + } + + // Test files in specific folder if we have one + if (testFolderId) { + const resWithFolder = await t.sdkForConn.GET( + '/unified/file-storage/files', + { + params: {query: {folder_id: testFolderId}}, + }, + ) + expect(Array.isArray(resWithFolder.data.items)).toBe(true) + resWithFolder.data.items.forEach((file) => { + expect(file.id).toBeTruthy() + expect(file.name).toBeTruthy() + expect(file.type).toBeTruthy() + expect(['number', 'object'].includes(typeof file.size)).toBe(true) + expect(file.raw_data as object).toBeTruthy() + expect(Object.keys(file.raw_data as object).length).toBeGreaterThan(0) + }) + } + + const file = res.data.items.find((item) => item.type === 'file') + if (file) { + testFileId = file.id + expect(file.id).toBeTruthy() + expect(file.name).toBeTruthy() + expect(file.type).toBeTruthy() + expect(['number', 'object'].includes(typeof file.size)).toBe(true) + expect(typeof file.created_at).toBe('string') + expect(typeof file.updated_at).toBe('string') + expect(file.raw_data as object).toBeTruthy() + expect(Object.keys(file.raw_data as object).length).toBeGreaterThan(0) + console.log( + `Test file ID set to: ${testFileId} ${file.name} ${file.type} ${ + file.size && file.size / 1024 / 1024 + } MB for ${t.adapterName} ${t.connectionId}`, + ) + } else { + throw new Error('No files found to use for testing') + } + + // Test if there's a folder in the items and get it + const folder = res.data.items.find((item) => item.type === 'folder') + if (folder) { + testFolderId = folder.id + expect(folder.id).toBeTruthy() + expect(folder.name).toBeTruthy() + expect(folder.type).toBe('folder') + expect(typeof folder.created_at).toBe('string') + expect(typeof folder.updated_at).toBe('string') + expect(folder.raw_data as object).toBeTruthy() + expect(Object.keys(folder.raw_data as object).length).toBeGreaterThan(0) + console.log( + `Test folder ID set to: ${testFolderId} ${folder.name} ${folder.type} for ${t.adapterName} ${t.connectionId}`, + ) } }) t.testIfImplemented('getFile', async () => { - // Skip if we don't have a file ID from previous test if (!testFileId) { console.warn('Skipping getFile test - no test file ID available') return } + // Test valid file retrieval const res = await t.sdkForConn.GET('/unified/file-storage/files/{id}', { params: {path: {id: testFileId}}, }) @@ -56,56 +248,205 @@ describeEachAdapterConnections>(adapters, (t) => { expect(res.data.name).toBeTruthy() expect(res.data.type).toBeTruthy() expect(typeof res.data.downloadable).toBe('boolean') + expect(['number', 'object'].includes(typeof res.data.size)).toBe(true) + expect(typeof res.data.created_at).toBe('string') + expect(typeof res.data.updated_at).toBe('string') + expect(res.data.raw_data as object).toBeTruthy() + expect(Object.keys(res.data.raw_data as object).length).toBeGreaterThan(0) + + // Test invalid file ID + await expect( + t.sdkForConn.GET('/unified/file-storage/files/{id}', { + params: {path: {id: 'invalid-file-id'}}, + }), + ).rejects.toThrow(/(Bad Request|Not Found)/) + + // Test with undefined ID + await expect( + t.sdkForConn.GET('/unified/file-storage/files/{id}', { + params: {path: {id: 'undefined'}}, + }), + ).rejects.toThrow(/Not Found/) }) - t.testIfImplemented('exportFile', async () => { - // Skip if we don't have a file ID from previous test - if (!testFileId) { - console.warn('Skipping exportFile test - no test file ID available') - return + t.testIfImplemented('downloadFile', async () => { + // Fetch the list of files + const filesResponse = await t.sdkForConn.GET( + '/unified/file-storage/files', + { + params: {query: {page_size: 100}}, + }, + ) + + // Find a downloadable file + const downloadableFile = filesResponse.data.items.find( + (file) => file.downloadable, + ) + + if (!downloadableFile) { + throw new Error('No downloadable files found to use for testing') } + const fileToDownload = downloadableFile.id + + console.log( + 'Attempting to download file ', + JSON.stringify(fileToDownload, null, 2), + ) const res = await t.sdkForConn.GET( - '/unified/file-storage/files/{id}/export', + '/unified/file-storage/files/{id}/download', { - params: { - path: {id: testFileId}, - query: {format: 'pdf'}, // Common export format - }, + params: {path: {id: fileToDownload}}, + parseAs: 'blob', }, ) - expect(res.data).toBeTruthy() - // Response should be a ReadableStream - expect(res.data).toBeInstanceOf(ReadableStream) + + // const contentDisposition = res.response.headers.get('content-disposition') + // const filename = + // contentDisposition?.split('filename=')[1]?.replace(/["']/g, '') || + // `file-${testFileId}` + + // const blob = await res.data + // const buffer = Buffer.from(await blob.arrayBuffer()) + // const fs = require('fs') + // const path = require('path') + // const downloadPath = path.join(filename) + // fs.writeFileSync(downloadPath, buffer) + // console.log(`File saved to: ${downloadPath}`) + + expect(res.data).toBeInstanceOf(Blob) + + // Test invalid file ID + const response = await t.sdkForConn.GET( + '/unified/file-storage/files/{id}/download', + { + params: {path: {id: 'invalid-file-id'}}, + parseAs: 'blob', + }, + ) + + expect(response.response.status).not.toBe(200) }) - t.testIfImplemented('downloadFile', async () => { - // Skip if we don't have a file ID from previous test - if (!testFileId) { - console.warn('Skipping downloadFile test - no test file ID available') + t.testIfImplemented('exportFile', async () => { + // For SharePoint, expect export to not be implemented + if (t.adapterName === 'microsoft' && testFileId) { + await expect( + t.sdkForConn.GET('/unified/file-storage/files/{id}/export', { + params: { + path: {id: testFileId}, + query: {format: 'pdf'}, + }, + parseAs: 'blob', + }), + ).rejects.toThrow(/not available in Sharepoint/) return } - const res = await t.sdkForConn.GET( - '/unified/file-storage/files/{id}/download', + const filesResponse = await t.sdkForConn.GET( + '/unified/file-storage/files', { - params: {path: {id: testFileId}}, + params: {query: {page_size: 50}}, }, ) - expect(res.data).toBeTruthy() - // Response should be a ReadableStream - expect(res.data).toBeInstanceOf(ReadableStream) + const exportableFile = filesResponse.data.items.find( + (file) => file.export_formats && file.export_formats.length > 0, + ) + if ( + !exportableFile || + !exportableFile.export_formats || + exportableFile.export_formats.length === 0 || + !exportableFile.export_formats[0] + ) { + throw new Error('No exportable file found') + } + const exportFormat = exportableFile.export_formats[0] + + console.log( + 'Attempting to export file ', + JSON.stringify(exportableFile, null, 2), + ) + const resPdf = await t.sdkForConn.GET( + '/unified/file-storage/files/{id}/export', + { + params: { + path: {id: exportableFile.id}, + query: {format: exportFormat}, + }, + parseAs: 'blob', + }, + ) + + expect(resPdf.data).toBeInstanceOf(ReadableStream) + + // Test with invalid format + await expect( + t.sdkForConn.GET('/unified/file-storage/files/{id}/export', { + params: { + path: {id: exportableFile.id}, + query: {format: 'invalid-format'}, + }, + parseAs: 'blob', + }), + ).rejects.toThrow() }) t.testIfImplemented('listFolders', async () => { + // Test default listing const res = await t.sdkForConn.GET('/unified/file-storage/folders', {}) expect(res.data.items).toBeTruthy() expect(Array.isArray(res.data.items)).toBe(true) - if (res.data.items.length > 0) { - // const folder = res.data.items[0] - // expect(folder.id).toBeTruthy() - // expect(folder.name).toBeTruthy() - // expect(folder.path).toBeTruthy() + + // Test with pagination + const resWithPagination = await t.sdkForConn.GET( + '/unified/file-storage/folders', + { + params: {query: {page_size: 5}}, + }, + ) + expect(resWithPagination.data.items.length).toBeLessThanOrEqual(5) + + // Test cursor-based pagination if we have items + if ( + resWithPagination.data.items.length === 5 && + resWithPagination.data.next_cursor + ) { + const resWithCursor = await t.sdkForConn.GET( + '/unified/file-storage/folders', + { + params: { + query: { + cursor: resWithPagination.data.next_cursor, + page_size: 5, + }, + }, + }, + ) + expect(resWithCursor.data.items).toBeTruthy() + expect(Array.isArray(resWithCursor.data.items)).toBe(true) + } + + if (res.data.items.length > 0 && res.data.items[0]) { + const folder = res.data.items[0] + testFolderId = folder.id + expect(folder.id).toBeTruthy() + expect(folder.name).toBeTruthy() + expect(folder.path).toBeTruthy() + expect(typeof folder.created_at).toBe('string') + expect(typeof folder.updated_at).toBe('string') + expect(folder.raw_data as object).toBeTruthy() + expect(Object.keys(folder.raw_data as object).length).toBeGreaterThan(0) + } + + // Test with driveId if available + if (testDriveId) { + const resWithDrive = await t.sdkForConn.GET( + '/unified/file-storage/folders', + { + params: {query: {drive_id: testDriveId}}, + }, + ) + expect(Array.isArray(resWithDrive.data.items)).toBe(true) } }) }) diff --git a/unified/unified-file-storage/router.ts b/unified/unified-file-storage/router.ts index f0a0991c..226e67d9 100644 --- a/unified/unified-file-storage/router.ts +++ b/unified/unified-file-storage/router.ts @@ -29,7 +29,7 @@ export const fileStorageRouter = trpc.router({ .input( zPaginationParams .extend({ - driveGroupId: z.string().optional(), + drive_group_id: z.string().optional(), }) .nullish(), ) @@ -41,8 +41,8 @@ export const fileStorageRouter = trpc.router({ .input( zPaginationParams .extend({ - driveId: z.string().optional(), - folderId: z.string().optional(), + drive_id: z.string().optional(), + folder_id: z.string().optional(), }) .nullish(), ) @@ -69,7 +69,7 @@ export const fileStorageRouter = trpc.router({ downloadFile: procedure .meta(oapi({method: 'GET', path: '/files/{id}/download'})) .input(z.object({id: z.string()})) - .output(z.instanceof(ReadableStream)) + .output(z.instanceof(ReadableStream)) // don't validate output .query(async ({input, ctx}) => proxyCallAdapter({input, ctx})), listFolders: procedure @@ -77,7 +77,7 @@ export const fileStorageRouter = trpc.router({ .input( zPaginationParams .extend({ - driveId: z.string().optional(), + drive_id: z.string().optional(), }) .nullish(), ) diff --git a/unified/unified-file-storage/unifiedModels.ts b/unified/unified-file-storage/unifiedModels.ts index f0a1c440..529a383b 100644 --- a/unified/unified-file-storage/unifiedModels.ts +++ b/unified/unified-file-storage/unifiedModels.ts @@ -7,6 +7,7 @@ export const DriveGroup = z description: z.string().nullish(), updated_at: z.string().nullish(), created_at: z.string().nullish(), + raw_data: z.any(), }) .openapi({ ref: 'unified.drivegroup', @@ -20,6 +21,7 @@ export const Drive = z description: z.string().nullish(), updated_at: z.string().nullish(), created_at: z.string().nullish(), + raw_data: z.any(), }) .openapi({ ref: 'unified.drive', @@ -43,6 +45,7 @@ export const File = z export_formats: z.array(z.string()).nullish(), updated_at: z.string().nullish(), created_at: z.string().nullish(), + raw_data: z.any(), }) .openapi({ ref: 'unified.file', @@ -57,6 +60,7 @@ export const Folder = z path: z.string(), updated_at: z.string().nullish(), created_at: z.string().nullish(), + raw_data: z.any(), }) .openapi({ ref: 'unified.folder',