test(amazonq): retry inline tests if no valid response was found

packages/amazonq/test/e2e/inline/inline.test.ts

@@ -15,11 +15,12 @@ import {
     using,
 } from 'aws-core-vscode/test'
 import { RecommendationHandler, RecommendationService, session } from 'aws-core-vscode/codewhisperer'
-import { Commands, globals, sleep, waitUntil } from 'aws-core-vscode/shared'
+import { Commands, globals, sleep, waitUntil, collectionUtil } from 'aws-core-vscode/shared'
 import { loginToIdC } from '../amazonq/utils/setup'
 
 describe('Amazon Q Inline', async function () {
     let tempFolder: string
+    const retries = 3
     const waitOptions = {
         interval: 500,
         timeout: 10000,
@@ -37,13 +38,24 @@ describe('Amazon Q Inline', async function () {
         const folder = await TestFolder.create()
         tempFolder = folder.path
         await closeAllEditors()
-        await resetCodeWhispererGlobalVariables(false)
+        await resetCodeWhispererGlobalVariables()
     })
 
     afterEach(async function () {
         await closeAllEditors()
+        if (this.currentTest?.state === undefined || this.currentTest?.isFailed() || this.currentTest?.isPending()) {
+            logUserDecisionStatus()
+        }
     })
 
+    function logUserDecisionStatus() {
+        const events = getUserTriggerDecision()
+        console.table({
+            'telemetry events': JSON.stringify(events),
+            'recommendation service status': RecommendationService.instance.isRunning,
+        })
+    }
+
     async function setupEditor({ name, contents }: { name?: string; contents?: string } = {}) {
         const fileName = name ?? 'test.ts'
         const textContents =
@@ -58,16 +70,28 @@ describe('Amazon Q Inline', async function () {
     }
 
     async function waitForRecommendations() {
-        const ok = await waitUntil(
-            async () =>
-                RecommendationHandler.instance.isSuggestionVisible() || session.getSuggestionState(0) === 'Showed',
+        const suggestionShown = await waitUntil(async () => session.getSuggestionState(0) === 'Showed', waitOptions)
+        if (!suggestionShown) {
+            throw new Error(`Suggestion did not show. Suggestion States: ${JSON.stringify(session.suggestionStates)}`)
+        }
+        const suggestionVisible = await waitUntil(
+            async () => RecommendationHandler.instance.isSuggestionVisible(),
             waitOptions
         )
-        if (!ok) {
-            assert.fail(
+        if (!suggestionVisible) {
+            throw new Error(
                 `Suggestions failed to become visible. Suggestion States: ${JSON.stringify(session.suggestionStates)}`
             )
         }
+        console.table({
+            'suggestions states': JSON.stringify(session.suggestionStates),
+            'valid recommendation': RecommendationHandler.instance.isValidResponse(),
+            'recommendation service status': RecommendationService.instance.isRunning,
+            recommendations: session.recommendations,
+        })
+        if (!RecommendationHandler.instance.isValidResponse()) {
+            throw new Error('Did not find a valid response')
+        }
     }
 
     /**
@@ -82,17 +106,23 @@ describe('Amazon Q Inline', async function () {
             })
             return events.some((event) => event.codewhispererSuggestionState === suggestionState)
         }, waitOptions)
-        const events = globals.telemetry.logger.query({
-            metricName,
-        })
         if (!ok) {
-            assert.fail(`Telemetry failed to be emitted. Current events: ${JSON.stringify(events)}`)
+            assert.fail(`Telemetry for ${metricName} with suggestionState ${suggestionState} was not emitted`)
         }
+        const events = getUserTriggerDecision()
         if (events.length > 1 && events[events.length - 1].codewhispererSuggestionState !== suggestionState) {
-            assert.fail(`Telemetry events were emitted in the wrong order. Current events: ${JSON.stringify(events)}`)
+            assert.fail(`Telemetry events were emitted in the wrong order`)
         }
     }
 
+    function getUserTriggerDecision() {
+        return globals.telemetry.logger
+            .query({
+                metricName: 'codewhisperer_userTriggerDecision',
+            })
+            .map((e) => collectionUtil.partialClone(e, 3, ['credentialStartUrl'], '[omitted]'))
+    }
+
     for (const [name, invokeCompletion] of [
         ['automatic', async () => await vscode.commands.executeCommand('type', { text: '\n' })],
         ['manual', async () => Commands.tryExecute('aws.amazonq.invokeInlineCompletion')],
@@ -101,7 +131,7 @@ describe('Amazon Q Inline', async function () {
             let originalEditorContents: string | undefined
 
             describe('supported filetypes', () => {
-                beforeEach(async () => {
+                async function setup() {
                     await setupEditor()
 
                     /**
@@ -119,6 +149,31 @@ describe('Amazon Q Inline', async function () {
 
                     // wait until the ghost text appears
                     await waitForRecommendations()
+                }
+
+                beforeEach(async () => {
+                    /**
+                     * Every once and a while the backend won't respond with any recommendations.
+                     * In those cases, re-try the setup up-to ${retries} times
+                     */
+                    let attempt = 0
+                    while (attempt < retries) {
+                        try {
+                            await setup()
+                            console.log(`test run ${attempt} succeeded`)
+                            logUserDecisionStatus()
+                            break
+                        } catch (e) {
+                            console.log(`test run ${attempt} failed`)
+                            console.log(e)
+                            logUserDecisionStatus()
+                            attempt++
+                            await resetCodeWhispererGlobalVariables()
+                        }
+                    }
+                    if (attempt === retries) {
+                        assert.fail(`Failed to invoke ${name} tests after ${attempt} attempts`)
+                    }
                 })
 
                 it(`${name} invoke accept`, async function () {

packages/core/src/shared/index.ts

@@ -62,3 +62,4 @@ export { i18n } from './i18n-helper'
 export * from './icons'
 export * as textDocumentUtil from './utilities/textDocumentUtilities'
 export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants'
+export * as collectionUtil from './utilities/collectionUtils'

packages/core/src/test/codewhisperer/testUtil.ts

@@ -28,15 +28,13 @@ import * as model from '../../codewhisperer/models/model'
 import { stub } from '../utilities/stubber'
 import { Dirent } from 'fs' // eslint-disable-line no-restricted-imports
 
-export async function resetCodeWhispererGlobalVariables(clearGlobalState: boolean = true) {
+export async function resetCodeWhispererGlobalVariables() {
     vsCodeState.isIntelliSenseActive = false
     vsCodeState.isCodeWhispererEditing = false
     CodeWhispererCodeCoverageTracker.instances.clear()
     globals.telemetry.logger.clear()
     session.reset()
-    if (clearGlobalState) {
-        await globals.globalState.clear()
-    }
+    await globals.globalState.clear()
     await CodeSuggestionsState.instance.setSuggestionsEnabled(true)
     await RecommendationHandler.instance.clearInlineCompletionStates()
 }