Neon database client migration

.github/workflows/release-workflow.yml

@@ -147,9 +147,17 @@ jobs:
           git fetch origin main
           git checkout main
           git push origin main:production -f
-
-      - name: Trigger Stably Workflow
+      - uses: UnlyEd/github-action-await-vercel@v1 # TODO best practices recommend to use a fixed version instead of @v1 for production usage (i.e: @v1.2.32)
+        id: await-vercel
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          gh workflow run stably-workflow.yml
+          VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+        with:
+          deployment-url: openint-git-main-openint-dev.vercel.app # TODO Replace by the domain you want to test
+          timeout: 10 # Wait for 10 seconds before failing
+          poll-interval: 1 # Wait for 1 second before each retry
+      - name: Stably Runner Action
+        id: stably-runner
+        uses: stablyhq/stably-runner-action@v3
+        with:
+          api-key: ${{ secrets.STABLY_API_KEY }}
+          test-group-id: cm5g8i6nc0005l103urkixuxz

.github/workflows/stably-workflow.yml

@@ -16,8 +16,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Wait for 5 minutes for any remaining Vercel Main Branch Deployment
-        run: sleep 300
+      - uses: UnlyEd/github-action-await-vercel@v1 # TODO best practices recommend to use a fixed version instead of @v1 for production usage (i.e: @v1.2.32)
+        id: await-vercel
+        env:
+          VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+        with:
+          deployment-url: openint-git-main-openint-dev.vercel.app # TODO Replace by the domain you want to test
+          timeout: 10 # Wait for 10 seconds before failing
+          poll-interval: 1 # Wait for 1 second before each retry
       - name: Stably Runner Action
         id: stably-runner
         uses: stablyhq/stably-runner-action@v3

.github/workflows/validate-workflow.yml

@@ -9,45 +9,22 @@ jobs:
     name: Run type checks, lint, and tests
     runs-on: ubuntu-latest
     timeout-minutes: 15
-    services:
-      # https://docs.github.com/en/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:latest
-        # service environment variables
-        env:
-          # `POSTGRES_HOST` is `postgres`
-          # optional (defaults to `postgres`)
-          POSTGRES_DB: test
-          # required
-          POSTGRES_PASSWORD: test
-          # optional (defaults to `5432`)
-          POSTGRES_PORT: 5432
-          # optional (defaults to `postgres`)
-          POSTGRES_USER: postgres
-        ports:
-          # maps tcp port 5432 on service container to the host
-          - 5432:5432
-        # set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      # This is not working for some reason...
-      # inngest:
-      #   image: inngest/inngest
-      #   ports:
-      #     - 8288:8288
 
     env:
       NODE_OPTIONS: --max-old-space-size=4096
       VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
-      DATABASE_URL: postgres://postgres:test@localhost:5432/test
+      DATABASE_URL: postgres://postgres:[email protected]:5432/postgres
 
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
+
+      - name: Start Docker services
+        run: docker compose up -d
+
       - name: Install Node.js
         uses: actions/setup-node@v3
         with:

apps/web/__tests__/end-to-end.spec.ts

@@ -1,27 +1,32 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import plaidSdkDef, {initPlaidSDK} from '@opensdks/sdk-plaid'
-import {drizzle, eq, schema, sql} from '@openint/db'
+import {drizzle, eq, neon, neonConfig, schema, sql} from '@openint/db'
 import {createAppTrpcClient} from '@openint/engine-frontend/lib/trpcClient'
 import {env, testEnv, testEnvRequired} from '@openint/env'
 import {initOpenIntSDK} from '@openint/sdk'
 import {setupTestOrg, tearDownTestOrg} from './test-utils'
 
 jest.setTimeout(30 * 1000) // long timeout because we have to wait for next.js to compile
+neonConfig.fetchEndpoint = (host) => {
+  const [protocol, port] =
+    host === 'db.localtest.me' ? ['http', 4444] : ['https', 443]
+  return `${protocol}://${host}:${port}/sql`
+}
 
 let fixture: Awaited<ReturnType<typeof setupTestOrg>>
 let sdk: ReturnType<typeof initOpenIntSDK>
 
 let trpc: ReturnType<typeof createAppTrpcClient>
 
-const db = drizzle(env.DATABASE_URL, {logger: true, schema})
+const db = drizzle(neon(env.DATABASE_URL), {logger: true, schema})
 async function setupTestDb(dbName: string) {
-  await db.execute(`DROP DATABASE IF EXISTS ${dbName}`)
+  await db.execute(`DROP DATABASE IF EXISTS ${dbName} WITH (FORCE)`)
   await db.execute(`CREATE DATABASE ${dbName}`)
   const url = new URL(env.DATABASE_URL)
   url.pathname = `/${dbName}`
   console.log('setupTestDb url:', url.toString())
   console.log(url.toString())
-  return {url, db: drizzle(url.toString(), {logger: true})}
+  return {url, db: drizzle(neon(url.toString()), {logger: true})}
 }
 
 beforeAll(async () => {
@@ -41,7 +46,7 @@ beforeAll(async () => {
 afterAll(async () => {
   if (!testEnv.DEBUG) {
     await tearDownTestOrg(fixture)
-    await testDb.db.$client.end()
+    // No need to close connections with neon
     // Cannot drop because database connection is still kept open by connector-postgres/server
     // await db.execute(`DROP DATABASE IF EXISTS test_${fixture.testId}`)
   }
@@ -125,7 +130,7 @@ test('create and sync plaid connection', async () => {
   const rows = await testDb.db.execute(
     sql`SELECT * FROM openint.banking_transaction`,
   )
-  expect(rows[0]).toMatchObject({
+  expect(rows.rows[0]).toMatchObject({
     source_id: connId,
     id: expect.any(String),
     customer_id: null,
@@ -178,7 +183,7 @@ test('create and sync greenhouse connection', async () => {
   })
 
   const rows = await testDb.db.execute(sql`SELECT * FROM openint.job`)
-  expect(rows[0]).toMatchObject({
+  expect(rows.rows[0]).toMatchObject({
     source_id: connId,
     id: expect.any(String),
     customer_id: null,

apps/web/app/api/connections/[connectionId]/sql/route.ts

@@ -59,7 +59,7 @@ export async function GET(
       format === 'csv'
         ? new NextResponse(
             Papa.unparse([
-              ...rows.map((r) =>
+              ...rows.rows.map((r) =>
                 R.mapValues(r, (v) =>
                   v instanceof Date
                     ? v.toISOString()

docker-compose.yml

@@ -3,28 +3,40 @@
 version: '3'
 services:
   postgres:
-    image: postgres
+    image: postgres:17
     ports:
       - 5432:5432
     # command: postgres -c log_statement=all
     restart: always
     environment:
+      POSTGRES_USER: postgres
       POSTGRES_DB: postgres # Only database named `postgres` works with pg_cron by default
       POSTGRES_PASSWORD: password
-    command: ["postgres", "-c", "log_statement=all", "-c", "max_connections=500"]
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U postgres']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    command: '-d 1'
+    volumes:
+      - db_data:/var/lib/postgresql/data
 
-  supabase:
-    image: supabase/postgres:15.8.1.017 # For some reason "lastest tag" is having issues... so we pin to a specific version latest as of 2024-12-20_0055
-    ports:
-      - 5433:5432 # Change the port to avoid conflict with the default postgres
-    command: postgres -c config_file=/etc/postgresql/postgresql.conf -c log_statement=all -c max_connections=500
-    restart: always
+  neon-proxy:
+    image: ghcr.io/timowilhelm/local-neon-http-proxy:main
     environment:
-      POSTGRES_DB: supabase # Only database named `postgres` works with pg_cron by default
-      POSTGRES_PASSWORD: password
+      ## Ensure that this matches the .env.dev DATABASE_URL
+      - PG_CONNECTION_STRING=postgres://postgres:password@postgres:5432/postgres
+    ports:
+      - '4444:4444'
+    depends_on:
+      postgres:
+        condition: service_healthy
 
   inngest:
-      image: inngest/inngest
-      command: 'inngest dev'
-      ports:
-        - '8288:8288'
+    image: inngest/inngest
+    command: 'inngest dev'
+    ports:
+      - '8288:8288'
+
+volumes:
+  db_data:

kits/cdk/internal/oauthConnector.ts

@@ -159,7 +159,12 @@ export function makeOauthConnectorServer({
             },
           },
         })
-        .then((r) => r.data as OauthBaseTypes['connectionSettings']['oauth'])
+        .then(
+          (r) =>
+            r.data as OauthBaseTypes['connectionSettings']['oauth'] & {
+              error: z.infer<typeof zOauthConnectionError>
+            },
+        )
 
       const parsed = zOauthCredentials.safeParse(res.credentials)
       if (!parsed.success) {

packages/db/index.ts

@@ -1,29 +1,29 @@
+import {neon, neonConfig} from '@neondatabase/serverless'
 import type {DrizzleConfig, SQL} from 'drizzle-orm'
 import {sql} from 'drizzle-orm'
-import {drizzle} from 'drizzle-orm/postgres-js'
-import postgres from 'postgres'
+import {drizzle} from 'drizzle-orm/neon-http'
 import {env} from '@openint/env'
 import * as schema from './schema'
 
 export * from 'drizzle-orm'
 export * from './stripeNullByte'
 export * from './upsert'
-export {schema, drizzle, postgres}
+export {schema, drizzle, neon, neonConfig}
 
 export function getDb<
   TSchema extends Record<string, unknown> = Record<string, never>,
 >(urlString: string, config?: DrizzleConfig<TSchema>) {
-  const pg = postgres(urlString)
-  const db = drizzle(pg, {logger: !!env['DEBUG'], ...config})
+  const sql = neon(urlString)
+  const db = drizzle(sql, {logger: !!env['DEBUG'], ...config})
 
   const url = new URL(urlString)
   if (env.DEBUG) {
     console.log('[db] host', url.host)
   }
-  return {db, pg}
+  return {db, sql}
 }
 
-export const {pg, db} = getDb(env.DATABASE_URL, {schema})
+export const {sql: pg, db} = getDb(env.DATABASE_URL, {schema})
 
 export async function ensureSchema(
   thisDb: ReturnType<typeof getDb>['db'],
@@ -34,7 +34,7 @@ export async function ensureSchema(
     .execute(
       sql`SELECT true as exists FROM information_schema.schemata WHERE schema_name = ${schema}`,
     )
-    .then((r) => r[0]?.['exists'] === true)
+    .then((r) => r.rows[0]?.['exists'] === true)
   if (exists) {
     return
   }

packages/db/package.json

@@ -12,6 +12,7 @@
     "push": "drizzle-kit push"
   },
   "dependencies": {
+    "@neondatabase/serverless": "^0.10.4",
     "@openint/env": "workspace:*",
     "drizzle-kit": "^0.30.1",
     "drizzle-orm": "^0.38.2",