ok

e2e/utils/db.ts

@@ -44,7 +44,6 @@ export async function populateLogs() {
       cost: 3.75e-5,
       external_user_id: 91823,
       feedback: null,
-      sibling_run_id: null,
       template_version_id: null,
       runtime: "langchain-js",
       metadata: "{}",

packages/backend/src/jobs/data-retention.ts

@@ -0,0 +1,65 @@
+import sql from "../utils/db"
+import * as Sentry from "@sentry/node"
+
+export default async function purgeRuns() {
+  try {
+    const deletedRunsCount = await sql.begin(async (sql) => {
+      await sql`create temporary table runs_to_delete_temp (run_id uuid)`
+
+      await sql`
+        with orgs as (
+          select
+            id,
+            data_retention_days
+          from
+            org
+          where
+            data_retention_days is not null
+            and plan != 'free'
+          
+          union all
+          
+          select
+            id,
+            30 as data_retention_days
+          from
+            org
+          where
+            plan = 'free'
+        ),
+        runs_to_delete as (
+          select
+            r.id as run_id
+          from
+            orgs o
+            join project p on o.id = p.org_id
+            join run r on p.id = r.project_id
+          where
+            r.created_at < now() - interval '1 day' * o.data_retention_days
+            order by r.created_at desc
+        )
+        insert into runs_to_delete_temp
+        select
+          run_id
+        from
+          runs_to_delete
+      `
+
+      await sql`delete from radar_result where radar_result.run_id in (select run_id from runs_to_delete_temp)`
+      await sql`delete from evaluation_result_v2 where evaluation_result_v2.run_id in (select run_id from runs_to_delete_temp)`
+      const { count } =
+        await sql`delete from run where id in (select run_id from runs_to_delete_temp)`
+
+      await sql`drop table runs_to_delete_temp`
+      return count
+    })
+    console.info(
+      `[JOB] Data retention job completed: ${deletedRunsCount} runs purged`,
+    )
+  } catch (error) {
+    console.error(error)
+    if (process.env.NODE_ENV === "production") {
+      Sentry.captureException(error)
+    }
+  }
+}

packages/backend/src/utils/cron.ts

@@ -1,16 +1,22 @@
-import cron from "node-cron"
-import sql from "./db"
 import resetUsage from "@/src/jobs/resetUsage"
+import cron from "node-cron"
+import purgeRuns from "../jobs/data-retention"
 import stripeCounters from "../jobs/stripeMeters"
 
 const EVERY_HOUR = "0 * * * *"
+const EVERY_DAY_AT_10AM = "0 10 * * *"
+const EVERY_DAY_AT_MIDNIGHT = "0 0 * * *"
 
 export function setupCronJobs() {
-  cron.schedule("0 10 * * *", resetUsage, {
+  cron.schedule(EVERY_DAY_AT_10AM, resetUsage, {
     name: "reset usage",
   })
 
   cron.schedule(EVERY_HOUR, stripeCounters, {
     name: "stripe meters",
   })
+
+  cron.schedule(EVERY_DAY_AT_MIDNIGHT, purgeRuns, {
+    name: "purge runs",
+  })
 }

packages/db/0036.sql

@@ -0,0 +1,3 @@
+alter table org add column data_retention_days int4 default null;
+alter table run drop column sibling_run_id;
+create index on radar_result(run_id);