Db v1 setup

.github/workflows/validate-workflow.yml

@@ -51,7 +51,7 @@ jobs:
       - name: Install Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 20 # TODO: Can we get this from "engine" field in package.json?
+          node-version: 22 # TODO: Can we get this from "engine" field in package.json?
 
       - uses: pnpm/action-setup@v4
         name: Install pnpm
@@ -90,7 +90,8 @@ jobs:
         run: |
           npm install --global vercel@latest
           vercel link --token $VERCEL_TOKEN --scope openint-dev --yes
-          vercel env pull --token $VERCEL_TOKEN ./apps/web/.env.local
+          vercel env pull --token $VERCEL_TOKEN ./apps/web/.env.local.orig
+          cat ./apps/web/.env.local.orig | pnpm --silent bun scripts/escape-env.ts > ./apps/web/.env.local
 
       - name: Ensure OpenAPI spec and docs are up to date
         run: pnpm --dir ./kits/sdk run gen && pnpm --dir ./docs generate && git diff --exit-code

bin/tp.ts

@@ -84,7 +84,11 @@ const name = [label, `${url.username}@${url.hostname}${port ? `:${port}` : ''}`]
 
 const {label: _, ...options} = args.values
 
-if (url.hostname === 'localhost' || url.hostname === '127.0.0.1') {
+if (
+  url.hostname === 'localhost' ||
+  url.hostname === '127.0.0.1' ||
+  url.hostname.includes('localtest.me')
+) {
   options.env = options.env || 'local'
   options.statusColor = options.statusColor || '007F3D'
   options.safeModeLevel = '0'

docker-compose.yml

@@ -4,6 +4,9 @@ version: '3'
 services:
   postgres:
     image: postgres
+    # build:
+    #   context: ./docker/postgres/
+    #   dockerfile: Dockerfile
     ports:
       - 5432:5432
     # command: postgres -c log_statement=all
@@ -12,6 +15,11 @@ services:
       POSTGRES_DB: postgres # Only database named `postgres` works with pg_cron by default
       POSTGRES_PASSWORD: password
     command: ["postgres", "-c", "log_statement=all", "-c", "max_connections=500"]
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U postgres']
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   supabase:
     image: supabase/postgres:15.8.1.017 # For some reason "lastest tag" is having issues... so we pin to a specific version latest as of 2024-12-20_0055
@@ -24,7 +32,20 @@ services:
       POSTGRES_PASSWORD: password
 
   inngest:
-      image: inngest/inngest
-      command: 'inngest dev'
-      ports:
-        - '8288:8288'
+    image: inngest/inngest
+    command: 'inngest dev'
+    ports:
+      - '8288:8288'
+
+  neon-proxy:
+    image: ghcr.io/timowilhelm/local-neon-http-proxy:main
+    # build:
+    #   context: ./docker/neon-proxy/
+    #   dockerfile: Dockerfile
+    environment:
+      PG_CONNECTION_STRING: postgres://postgres:password@postgres:5432/postgres
+    ports:
+      - '4444:4444'
+    depends_on:
+      postgres:
+        condition: service_healthy

docker/neon-proxy/Caddyfile

@@ -0,0 +1,9 @@
+:4444 {
+	reverse_proxy https://localhost:4445 {
+		transport http {
+			tls_trusted_ca_certs server.pem
+			tls_server_name db.localtest.me
+		}
+		header_up Host db.localtest.me:4445
+	}
+}

docker/neon-proxy/Dockerfile

@@ -0,0 +1,74 @@
+# hadolint global ignore=DL3008
+
+ARG NEON_RELEASE_TAG=release-7845
+
+FROM rust:bookworm AS rust-builder
+ARG NEON_RELEASE_TAG
+
+ARG DEBIAN_FRONTEND=noninteractive
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# install apt dependencies
+RUN \
+  apt-get update -qq \
+  && apt-get install -qq --no-install-recommends -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-confdef \
+  build-essential \
+  pkg-config \
+  git \
+  libssl-dev \
+  && apt-get clean -qq && rm -rf /var/lib/apt/lists/*
+
+# get and build the proxy
+RUN git clone --depth=1 --branch $NEON_RELEASE_TAG https://github.com/neondatabase/neon.git
+WORKDIR /neon
+RUN cargo build --bin proxy --features "testing"
+
+
+FROM debian:bookworm-slim
+
+ARG DEBIAN_FRONTEND=noninteractive
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# install apt dependencies
+RUN \
+  apt-get update -qq \
+  && apt-get install -qq --no-install-recommends -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-confdef \
+  curl \
+  ca-certificates \
+  openssl \
+  postgresql-client \
+  && apt-get clean -qq && rm -rf /var/lib/apt/lists/*
+
+# install caddy
+RUN \
+  apt-get update -qq \
+  && apt-get install -qq --no-install-recommends -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-confdef \
+  gnupg2 debian-keyring debian-archive-keyring apt-transport-https \
+  && curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg2 --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg \
+  && curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list \
+  && apt-get update -qq \
+  && apt-get install -qq --no-install-recommends -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-confdef \
+  caddy \
+  && apt-get clean -qq && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# create a self-signed cert for *.localtest.me (see https://readme.localtest.me/)
+RUN openssl req -new -x509 \
+  -days 365 \
+  -nodes -text \
+  -out server.pem \
+  -keyout server.key \
+  -subj "/CN=*.localtest.me" \
+  -addext "subjectAltName = DNS:*.localtest.me"
+
+# copy the proxy binary
+COPY --from=rust-builder /neon/target/debug/proxy ./neon-proxy
+
+COPY ./Caddyfile Caddyfile
+COPY ./start.sh start.sh
+
+RUN chmod +x start.sh
+
+EXPOSE 4444
+ENTRYPOINT ["./start.sh"]

docker/neon-proxy/start.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+if [ -z "$PG_CONNECTION_STRING" ]; then
+  echo "PG_CONNECTION_STRING is not set"
+  exit 1
+fi
+
+# Create required tables
+psql -Atx $PG_CONNECTION_STRING \
+  -c "CREATE SCHEMA IF NOT EXISTS neon_control_plane" \
+  -c "CREATE TABLE neon_control_plane.endpoints (endpoint_id VARCHAR(255) PRIMARY KEY, allowed_ips VARCHAR(255))"
+  -c "CREATE TABLE neon_control_plane.endpoint_jwks (id VARCHAR PRIMARY KEY, jwks_url TEXT NOT NULL, audience TEXT NOT NULL, role_names TEXT[] NOT NULL, endpoint_id UUID NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now())"
+
+# Start the neon-proxy
+./neon-proxy \
+  -c server.pem \
+  -k server.key \
+  --auth-backend=postgres \
+  --auth-endpoint=$PG_CONNECTION_STRING \
+  --wss=0.0.0.0:4445 \
+  --is-auth-broker=true \
+  &
+
+# Start caddy reverse proxy
+caddy run \
+  --config ./Caddyfile \
+  --adapter caddyfile \
+  &
+
+wait -n
+exit $?

docker/postgres/Dockerfile

@@ -0,0 +1,20 @@
+# Not working at the moment... Would be nice to be able to get the actual pg_session_jwt extension working
+FROM rust:bookworm AS rust-builder
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN mkdir /ext-src
+WORKDIR /ext-src
+RUN apt-get update && apt-get install -y wget
+RUN wget https://github.com/neondatabase/pg_session_jwt/archive/refs/tags/v0.2.0.tar.gz -O pg_session_jwt.tar.gz && \
+    echo "5ace028e591f2e000ca10afa5b1ca62203ebff014c2907c0ec3b29c36f28a1bb pg_session_jwt.tar.gz" | sha256sum --check && \
+    mkdir pg_session_jwt-src && cd pg_session_jwt-src && tar xzf ../pg_session_jwt.tar.gz --strip-components=1 -C . && \
+    sed -i 's/pgrx = "0.12.6"/pgrx = { version = "0.12.9", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \
+    sed -i 's/version = "0.12.6"/version = "0.12.9"/g' pgrx-tests/Cargo.toml && \
+    sed -i 's/pgrx = "=0.12.6"/pgrx = { version = "=0.12.9", features = [ "unsafe-postgres" ] }/g' pgrx-tests/Cargo.toml && \
+    sed -i 's/pgrx-macros = "=0.12.6"/pgrx-macros = "=0.12.9"/g' pgrx-tests/Cargo.toml && \
+    sed -i 's/pgrx-pg-config = "=0.12.6"/pgrx-pg-config = "=0.12.9"/g' pgrx-tests/Cargo.toml
+RUN cargo pgrx install --release
+
+FROM postgres:latest
+
+COPY --from=rust-builder /ext-src /usr/share/postgresql/17/extension/

package.json

@@ -25,7 +25,10 @@
     "pgdump": "pg_dump --schema public --schema-only --no-owner --exclude-schema=graphile_migrate --file=packages/db/schema.sql $DATABASE_URL",
     "worker:setup": "tsx ./bin/openint setupWorker",
     "worker:run": "tsx ./bin/openint runWorker",
-    "env:pull": "vercel env pull --environment development .env.dev && vercel env pull --environment preview --git-branch $(git rev-parse --abbrev-ref HEAD) .env.pre && vercel env pull --environment production .env.prod"
+    "env:pull:development": "vercel env pull --environment development .env.dev.orig && cat .env.dev.orig | pnpm --silent bun scripts/escape-env.ts > .env.dev",
+    "env:pull:preview": "vercel env pull --environment preview --git-branch $(git rev-parse --abbrev-ref HEAD) .env.pre.orig && cat .env.pre.orig | pnpm --silent bun scripts/escape-env.ts > .env.pre",
+    "env:pull:production": "vercel env pull --environment production .env.prod.orig && cat .env.prod.orig | pnpm --silent bun scripts/escape-env.ts > .env.prod",
+    "env:pull": "run-s env:pull:*"
   },
   "lint-staged": {
     "**/*.{js,ts,tsx,json,css,yml,yaml}": "prettier --write",
@@ -46,6 +49,7 @@
     "@types/prettier": "3.0.0",
     "@typescript-eslint/eslint-plugin": "6.21.0",
     "@typescript-eslint/parser": "6.21.0",
+    "bun": "latest",
     "esbuild": "0.17.5",
     "esbuild-jest": "0.5.0",
     "eslint": "8.23.0",

packages-v1/api-v1/app.ts

@@ -1,17 +1,44 @@
 import {swagger} from '@elysiajs/swagger'
 import {Elysia} from 'elysia'
+import type {Database} from '@openint/db-v1'
+import {createDatabase, createNeonDatabase} from '@openint/db-v1'
+import {envRequired} from '@openint/env'
 import {createOpenApiHandler, createTrpcHandler} from './trpc/handlers'
 import {generateOpenAPISpec} from './trpc/openapi'
 
+async function checkLatency(db: Database) {
+  const start = new Date()
+  await db.execute('SELECT 1')
+  const durationMs = Date.now() - start.getTime()
+  return durationMs
+}
+
 export const app = new Elysia({prefix: '/api'})
-  .get('/health', () => ({status: 'ok'}))
+  .get('/health', async () => {
+    const postgresJsLatencyMs = await checkLatency(
+      createDatabase({url: envRequired.DATABASE_URL}),
+    )
+    const neonLatencyMs = await checkLatency(
+      createNeonDatabase({
+        url: envRequired.DATABASE_URL,
+      }) as unknown as Database,
+    )
+    return {healthy: true, postgresJsLatencyMs, neonLatencyMs}
+  })
   .use(
     swagger({
       // For some reason spec.content doesn't work. so we are forced tos specify url instead
       scalarConfig: {spec: {url: '/api/v1/openapi.json'}},
       path: '/v1',
     }),
   )
+  // maybe this should be part of trpc and openapi spec itself?
+  .get('/.well-known/jwks.json', () => {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+    const publicKeyJwk = JSON.parse(envRequired.NEXT_PUBLIC_JWT_PUBLIC_KEY)
+    const jwks = {keys: [publicKeyJwk]}
+    return jwks
+  })
   .get('/v1/openapi.json', () => generateOpenAPISpec({}))
   // These two ways of mounting are very inconsistent, but I don't know why.
   // empirically, the first one without * works for trpc, and the second one with * works for openapi

packages-v1/api-v1/models.ts

@@ -76,3 +76,7 @@ export const core = {
       title: 'Connector Config',
     }),
 }
+
+export type Core = {
+  [k in keyof typeof core]: z.infer<(typeof core)[k]>
+}

packages-v1/api-v1/package.json

@@ -13,11 +13,13 @@
     "@clerk/nextjs": "^6.11.2",
     "@elysiajs/swagger": "^1.2.0",
     "@openint/all-connectors": "workspace:*",
+    "@openint/db-v1": "workspace:*",
     "@openint/env": "workspace:*",
     "@openint/events": "workspace:*",
     "@sinclair/typebox": "^0.34.20",
     "@trpc/server": "next",
     "elysia": "^1.2.12",
+    "jose": "^6.0.7",
     "remeda": "^2.20.1",
     "trpc-to-openapi": "^2.1.3",
     "zod": "^3.24.2",

packages-v1/api-v1/scripts/generateAndExportKeys.ts

@@ -0,0 +1,37 @@
+import {exportJWK, generateKeyPair} from 'jose'
+import {z} from 'zod'
+
+const envName = z
+  .enum(['development', 'preview', 'production'])
+  .default('development')
+  .parse(process.argv[2])
+
+// Function to generate a new RSA key pair, converts the keys to JWK (JSON Web Key) format,
+// adds necessary metadata, and saves them to separate files.
+async function generateAndExportKeys() {
+  // Generate a new RSA key pair using the RS256 algorithm.
+  const {publicKey, privateKey} = await generateKeyPair('RS256', {
+    extractable: true,
+  })
+  // Convert the generated keys to JWK format
+  const privateJwk = await exportJWK(privateKey)
+  const publicJwk = await exportJWK(publicKey)
+  // Add metadata to the private key JWK.
+  // 'use': Indicates the key's intended use (e.g., 'sig' for signing).
+  // 'kid': A unique identifier for the key, useful for key management and rotation.
+  // 'alg': Specifies the algorithm to be used with the key (Neon RLS Authorize supports only RS256 and ES256 currently).
+  privateJwk.use = 'sig'
+  privateJwk.kid = `openint-${envName}`
+  privateJwk.alg = 'RS256'
+  // Add the same metadata to the public key JWK for consistency.
+  publicJwk.use = 'sig'
+  publicJwk.kid = `openint-${envName}`
+  publicJwk.alg = 'RS256'
+  // Save the keys to separate JSON files.
+  console.log('--- privateKey.jwk.json')
+  console.log(JSON.stringify(privateJwk))
+  console.log('--- publicKey.jwk.json')
+  console.log(JSON.stringify(publicJwk))
+  console.log('--- Keys generated and saved to files.')
+}
+generateAndExportKeys()

packages-v1/api-v1/trpc/_base.ts

@@ -1,12 +1,41 @@
-import {initTRPC} from '@trpc/server'
+import {initTRPC, TRPCError} from '@trpc/server'
 import {type OpenApiMeta} from 'trpc-to-openapi'
-import type {Viewer} from '@openint/cdk'
+import {hasRole, type Viewer} from '@openint/cdk'
+import type {Database} from '@openint/db-v1'
 
 export interface RouterContext {
   viewer: Viewer
+  db: Database
 }
 
-export const trpc = initTRPC.meta<OpenApiMeta>().context<RouterContext>().create()
+export const trpc = initTRPC
+  .meta<OpenApiMeta>()
+  .context<RouterContext>()
+  .create()
 
 export const router = trpc.router
 export const publicProcedure = trpc.procedure
+
+export const authenticatedProcedure = publicProcedure.use(({next, ctx}) => {
+  const viewer = ctx.viewer
+  if (!hasRole(viewer, ['customer', 'user', 'org'])) {
+    throw new TRPCError({code: 'FORBIDDEN', message: 'Admin only'})
+  }
+  return next({ctx: {...ctx, viewer}})
+})
+
+export const customerProcedure = publicProcedure.use(({next, ctx}) => {
+  const viewer = ctx.viewer
+  if (!hasRole(ctx.viewer, ['customer'])) {
+    throw new TRPCError({code: 'FORBIDDEN', message: 'Customer only'})
+  }
+  return next({ctx: {...ctx, viewer}})
+})
+
+export const adminProcedure = publicProcedure.use(({next, ctx}) => {
+  const viewer = ctx.viewer
+  if (!hasRole(ctx.viewer, ['user', 'org'])) {
+    throw new TRPCError({code: 'FORBIDDEN', message: 'Admin only'})
+  }
+  return next({ctx: {...ctx, viewer}})
+})

packages-v1/api-v1/trpc/context.ts

@@ -1,3 +1,9 @@
-export function contextFromRequest(_req: Request) {
-  return {viewer: {role: 'system' as const}}
+import {createDatabase} from '@openint/db-v1'
+import {env} from '@openint/env'
+import type {RouterContext} from './_base'
+
+export function contextFromRequest(_req: Request): RouterContext {
+  // create db once rather than on every request
+  const db = createDatabase({url: env.DATABASE_URL})
+  return {viewer: {role: 'system' as const}, db}
 }