fix(gateway): Cody API version config

- Fix Claude model versioning to use API version 0 for older models but not 3.5 or newer.
- Fix chat client adding empty assistant message as the last message for all requests which is not longer supported by newer models.
- Add support for the latest Cody API version (8) for chat completions.
- Refactor the chat client to simplify logic and improve readability.

## Test plan

- Verified that chat completions work as expected.
- Verified that Fireworks tracing is enabled for Sourcegraph teammates.
- Verified that Claude models older than 3.5 use API version 0.
- Verified that Claude models 3.5 or newer does not use API version 0.
- Verified that the chat client does not add an empty assistant message as the last message for all requests unless they are using older claude models.

lib/shared/src/chat/chat.ts

@@ -18,6 +18,8 @@ const DEFAULT_CHAT_COMPLETION_PARAMETERS: Omit<ChatParameters, 'maxTokensToSampl
     topP: -1,
 }
 
+const claudeRegex = /claude-(\d+\.\d+)-/
+
 export class ChatClient {
     constructor(private completions: SourcegraphCompletionsClient) {}
 
@@ -52,49 +54,39 @@ export class ChatClient {
             throw new Error('unable to determine Cody API version')
         }
 
-        const useApiV1 =
-            params.model?.includes('claude-3') && !(versions instanceof Error) && versions.codyAPIVersion
-        const isLastMessageFromHuman = messages.length > 0 && messages.at(-1)!.speaker === 'human'
-
-        const isFireworks =
-            params?.model?.startsWith('fireworks/') || params?.model?.startsWith('fireworks::')
-        const augmentedMessages =
-            isFireworks || useApiV1
-                ? sanitizeMessages(messages)
-                : isLastMessageFromHuman
-                  ? messages.concat([{ speaker: 'assistant' }])
-                  : messages
-
-        // We only want to send up the speaker and prompt text, regardless of whatever other fields
-        // might be on the messages objects (`file`, `displayText`, `contextFiles`, etc.).
-        const messagesToSend = augmentedMessages.map(({ speaker, text, cacheEnabled, content }) => ({
-            text,
-            speaker,
-            cacheEnabled,
-            content,
-        }))
+        const requestParams = { apiVersion: versions.codyAPIVersion, interactionId, customHeaders: {} }
+
+        const isFireworks = params?.model?.startsWith('fireworks')
+        // Enabled Fireworks tracing for Sourcegraph teammates.
+        // https://readme.fireworks.ai/docs/enabling-tracing
+        if (isFireworks && authStatus_.isFireworksTracingEnabled) {
+            requestParams.customHeaders = { 'X-Fireworks-Genie': 'true' }
+            messages = sanitizeMessages(messages)
+        }
+
+        // Set api version to 0 (unversion) for Claude models older than 3.5.
+        // Example: claude-3-haiku or claude-2-sonnet or claude-2.1-instant v.s. claude-3-5-haiku or 3.5-haiku or 3-7-haiku
+        // Regex for matching old claude models by checking if the model digits are less than 3.5
+        const claudeVersionMatch = params.model?.match(claudeRegex)
+        if (claudeVersionMatch && Number.parseFloat(claudeVersionMatch[1]) < 3.5) {
+            requestParams.apiVersion = 0
+            messages = messages.concat([{ speaker: 'assistant' }])
+        }
 
         const completionParams = {
             ...DEFAULT_CHAT_COMPLETION_PARAMETERS,
             ...params,
-            messages: messagesToSend,
+            // We only want to send up the speaker and prompt text, regardless of whatever other fields
+            // might be on the messages objects (`file`, `displayText`, `contextFiles`, etc.).
+            messages: messages.map(({ speaker, text, cacheEnabled, content }) => ({
+                text,
+                speaker,
+                cacheEnabled,
+                content,
+            })),
         }
 
-        // Enabled Fireworks tracing for Sourcegraph teammates.
-        // https://readme.fireworks.ai/docs/enabling-tracing
-
-        const customHeaders: Record<string, string> =
-            isFireworks && authStatus_.isFireworksTracingEnabled ? { 'X-Fireworks-Genie': 'true' } : {}
-
-        return this.completions.stream(
-            completionParams,
-            {
-                apiVersion: useApiV1 ? versions.codyAPIVersion : 0,
-                interactionId: interactionId,
-                customHeaders,
-            },
-            abortSignal
-        )
+        return this.completions.stream(completionParams, requestParams, abortSignal)
     }
 }
 

lib/shared/src/sourcegraph-api/siteVersion.ts

@@ -113,17 +113,19 @@ export function checkVersion({
     return (insider && isInsiderBuild) || semver.gte(currentVersion, minimumVersion)
 }
 
-type CodyApiVersion = 0 | 1 | 2
+// @link latestSupportedCompletionsStreamAPIVersion
+// Docs: https://sourcegraph.sourcegraph.com/search?q=context:global+latestSupportedCompletionsStreamAPIVersion
+type CodyApiVersion = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8
+const latestSupportedCompletionsStreamAPIVersion = 8
 
 /** @internal Exported for testing only. */
 export function inferCodyApiVersion(version: string, isDotCom: boolean): CodyApiVersion {
     const parsedVersion = semver.valid(version)
     const isLocalBuild = parsedVersion === '0.0.0'
 
     if (isDotCom || isLocalBuild) {
-        // The most recent version is api-version=2, which was merged on 2024-09-11
-        // https://github.com/sourcegraph/sourcegraph/pull/470
-        return 2
+        // The most recent version is api-version=8, latestSupportedCompletionsStreamAPIVersion
+        return 8
     }
 
     // On Cloud deployments from main, the version identifier will use a format
@@ -132,17 +134,25 @@ export function inferCodyApiVersion(version: string, isDotCom: boolean): CodyApi
     // allowing us to selectively enable new API versions on instances like SG02
     // (that deploy frequently) without crashing on other Cloud deployments that
     // release less frequently.
-    const isCloudBuildFromMain = parsedVersion === null
-    if (isCloudBuildFromMain) {
+    if (parsedVersion === null) {
+        // api-version=2 was merged on 2024-09-11:
+        // https://github.com/sourcegraph/sourcegraph/pull/470
         const date = parseDateFromPreReleaseVersion(version)
         if (date && date >= new Date('2024-09-11')) {
-            return 2
+            return latestSupportedCompletionsStreamAPIVersion
         }
         // It's safe to bump this up to api-version=2 after the 5.8 release
         return 1
     }
 
+    // 6.1.0+ is the first version to support api-version=8.
+    // https://github.com/sourcegraph/sourcegraph/pull/3507
+    if (semver.gte(parsedVersion, '6.1.0')) {
+        return 8
+    }
+
     // 5.8.0+ is the first version to support api-version=2.
+    // https://github.com/sourcegraph/sourcegraph/pull/470
     if (semver.gte(parsedVersion, '5.8.0')) {
         return 2
     }