feat: improv analytics (#397)

package.json

@@ -25,8 +25,7 @@
   ],
   "dependencies": {
     "concurrently": "^8.2.2",
-    "patch-package": "^8.0.0",
-    "react-markdown": "^9.0.1"
+    "patch-package": "^8.0.0"
   },
   "devDependencies": {
     "@playwright/test": "^1.42.1",

packages/backend/src/api/v1/analytics/utils.ts

@@ -1,6 +1,6 @@
 import { convertChecksToSQL } from "@/src/utils/checks"
 import sql from "@/src/utils/db"
-
+import Context from "@/src/utils/koa"
 import { deserializeLogic } from "shared"
 import { z } from "zod"
 

packages/backend/src/api/v1/index.ts

@@ -16,7 +16,6 @@ import templates from "./templates"
 import users from "./users"
 import evaluators from "./evaluator"
 import analytics from "./analytics"
-import views from "./views"
 
 const v1 = new Router({
   prefix: "/v1",
@@ -49,6 +48,5 @@ v1.use(evaluations.routes())
 v1.use(projectUsers.routes())
 v1.use(checklists.routes())
 v1.use(analytics.routes())
-v1.use(views.routes())
 
 export default v1

packages/backend/src/checks/index.ts

@@ -83,17 +83,13 @@ function postgresOperators(operator: string) {
 
 export const CHECK_RUNNERS: CheckRunner[] = [
   {
-    id: "fulltype",
+    id: "type",
     sql: ({ type }) =>
       type === "trace"
         ? // matches agent and chain runs with no parent or parent is a chat run
           sql`(r.type in ('agent','chain') and (parent_run_id is null OR EXISTS (SELECT 1 FROM run AS parent_run WHERE parent_run.id = r.parent_run_id AND parent_run.type = 'chat')))`
         : sql`(r.type = ${type})`,
   },
-  {
-    id: "type",
-    sql: ({ type }) => sql`(r.type = ${type})`,
-  },
   {
     id: "models",
     sql: ({ models }) => sql`(r.name = any(${models}))`,

packages/backend/src/index.ts

@@ -22,7 +22,9 @@ import { startMaterializedViewRefreshJob } from "./jobs/materializedViews"
 checkDbConnection()
 setupCronJobs()
 
-startMaterializedViewRefreshJob()
+if (process.env.NODE_ENV === "production") {
+  startMaterializedViewRefreshJob()
+}
 initSentry()
 
 const app = new Koa()

packages/backend/src/jobs/materializedViews.ts

@@ -1,8 +1,6 @@
 import sql from "../utils/db"
 import { sleep } from "../utils/misc"
 
-const REFRESH_FREQUENCY = process.env.NODE_ENV === "production" ? 1000 : 10000
-
 export async function startMaterializedViewRefreshJob() {
   try {
     const views = [
@@ -23,7 +21,7 @@ export async function startMaterializedViewRefreshJob() {
         )
       }
 
-      await sleep(REFRESH_FREQUENCY)
+      await sleep(1000)
     }
   } catch (error) {
     console.error(error)

packages/backend/src/utils/calcCost.ts

@@ -80,16 +80,16 @@ const MODEL_COSTS: ModelCost[] = [
     inputCost: 0.008,
     outputCost: 0.024,
   },
-  {
-    models: ["claude-3.5-sonnet"],
-    inputCost: 0.003,
-    outputCost: 0.015,
-  },
   {
     models: ["claude-3-opus"],
     inputCost: 0.015,
     outputCost: 0.075,
   },
+  {
+    models: ["claude-3-5-sonnet"],
+    inputCost: 0.003,
+    outputCost: 0.075,
+  },
   {
     models: ["claude-3-sonnet"],
     inputCost: 0.003,

packages/backend/src/utils/ingest.ts

@@ -166,8 +166,6 @@ export const ingestChatEvent = async (
     metadata: metadata || extra,
   })
 
-  console.log({ coreMessage })
-
   if (typeof parentRunId === "undefined") {
     throw new Error("parentRunId is undefined")
   }

packages/frontend/components/SmartViewer/Message.tsx

@@ -14,7 +14,6 @@ import {
   TextInput,
   Textarea,
   ThemeIcon,
-  useComputedColorScheme,
 } from "@mantine/core"
 import {
   IconInfoCircle,
@@ -27,6 +26,7 @@ import Image from "next/image"
 import ProtectedText from "../blocks/ProtectedText"
 import { RenderJson } from "./RenderJson"
 
+import { useColorScheme } from "@mantine/hooks"
 import { circularPro } from "@/utils/theme"
 import { useEffect } from "react"
 
@@ -441,8 +441,7 @@ export function ChatMessage({
 }) {
   // TODO FIX
   // Flickering dark mode bug: this is due to scheme being 'light' for a few ms
-
-  const scheme = useComputedColorScheme()
+  const scheme = useColorScheme()
 
   const color = getColorForRole(data?.role)
 

packages/frontend/components/analytics/BarList.tsx

@@ -1,11 +1,5 @@
-import {
-  Flex,
-  Progress,
-  Table,
-  Text,
-  Tooltip,
-  useComputedColorScheme,
-} from "@mantine/core"
+import { Flex, Progress, Table, Text, Tooltip } from "@mantine/core"
+import { useColorScheme } from "@mantine/hooks"
 
 type BarListProps = {
   data: any[]
@@ -25,8 +19,7 @@ function BarList({ data, columns, filterZero = true }: BarListProps) {
   const dataColumns = columns.filter((col) => !col.bar && col.key)
   const main = dataColumns.find((col) => col.main) || dataColumns[0]
   const mainTotal = data.reduce((acc, item) => acc + (item[main.key] || 0), 0)
-
-  const colorScheme = useComputedColorScheme()
+  const scheme = useColorScheme()
 
   if (!data) return <>No data.</>
 
@@ -91,7 +84,7 @@ function BarList({ data, columns, filterZero = true }: BarListProps) {
                         justify="center"
                       >
                         <Text
-                          c={colorScheme === "dark" ? "white" : "dark"}
+                          c={scheme === "dark" ? "white" : "dark"}
                           mb={-3}
                           size="12px"
                           style={{