From 6d53ccb7ab38cb02b70c09a161927260999d7c9e Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Mon, 24 Feb 2025 11:04:13 -0500 Subject: [PATCH] test(amazonq): retry inline tests if no valid response was found --- .../amazonq/test/e2e/inline/inline.test.ts | 81 ++++++++++++++++--- packages/core/src/shared/index.ts | 1 + .../core/src/test/codewhisperer/testUtil.ts | 6 +- 3 files changed, 71 insertions(+), 17 deletions(-) diff --git a/packages/amazonq/test/e2e/inline/inline.test.ts b/packages/amazonq/test/e2e/inline/inline.test.ts index f070840b3f8..31b94572537 100644 --- a/packages/amazonq/test/e2e/inline/inline.test.ts +++ b/packages/amazonq/test/e2e/inline/inline.test.ts @@ -15,11 +15,12 @@ import { using, } from 'aws-core-vscode/test' import { RecommendationHandler, RecommendationService, session } from 'aws-core-vscode/codewhisperer' -import { Commands, globals, sleep, waitUntil } from 'aws-core-vscode/shared' +import { Commands, globals, sleep, waitUntil, collectionUtil } from 'aws-core-vscode/shared' import { loginToIdC } from '../amazonq/utils/setup' describe('Amazon Q Inline', async function () { let tempFolder: string + const retries = 3 const waitOptions = { interval: 500, timeout: 10000, @@ -37,13 +38,24 @@ describe('Amazon Q Inline', async function () { const folder = await TestFolder.create() tempFolder = folder.path await closeAllEditors() - await resetCodeWhispererGlobalVariables(false) + await resetCodeWhispererGlobalVariables() }) afterEach(async function () { await closeAllEditors() + if (this.currentTest?.state === undefined || this.currentTest?.isFailed() || this.currentTest?.isPending()) { + logUserDecisionStatus() + } }) + function logUserDecisionStatus() { + const events = getUserTriggerDecision() + console.table({ + 'telemetry events': JSON.stringify(events), + 'recommendation service status': RecommendationService.instance.isRunning, + }) + } + async function setupEditor({ name, contents }: { name?: string; contents?: string } = {}) { const fileName = name ?? 'test.ts' const textContents = @@ -58,16 +70,28 @@ describe('Amazon Q Inline', async function () { } async function waitForRecommendations() { - const ok = await waitUntil( - async () => - RecommendationHandler.instance.isSuggestionVisible() || session.getSuggestionState(0) === 'Showed', + const suggestionShown = await waitUntil(async () => session.getSuggestionState(0) === 'Showed', waitOptions) + if (!suggestionShown) { + throw new Error(`Suggestion did not show. Suggestion States: ${JSON.stringify(session.suggestionStates)}`) + } + const suggestionVisible = await waitUntil( + async () => RecommendationHandler.instance.isSuggestionVisible(), waitOptions ) - if (!ok) { - assert.fail( + if (!suggestionVisible) { + throw new Error( `Suggestions failed to become visible. Suggestion States: ${JSON.stringify(session.suggestionStates)}` ) } + console.table({ + 'suggestions states': JSON.stringify(session.suggestionStates), + 'valid recommendation': RecommendationHandler.instance.isValidResponse(), + 'recommendation service status': RecommendationService.instance.isRunning, + recommendations: session.recommendations, + }) + if (!RecommendationHandler.instance.isValidResponse()) { + throw new Error('Did not find a valid response') + } } /** @@ -82,17 +106,23 @@ describe('Amazon Q Inline', async function () { }) return events.some((event) => event.codewhispererSuggestionState === suggestionState) }, waitOptions) - const events = globals.telemetry.logger.query({ - metricName, - }) if (!ok) { - assert.fail(`Telemetry failed to be emitted. Current events: ${JSON.stringify(events)}`) + assert.fail(`Telemetry for ${metricName} with suggestionState ${suggestionState} was not emitted`) } + const events = getUserTriggerDecision() if (events.length > 1 && events[events.length - 1].codewhispererSuggestionState !== suggestionState) { - assert.fail(`Telemetry events were emitted in the wrong order. Current events: ${JSON.stringify(events)}`) + assert.fail(`Telemetry events were emitted in the wrong order`) } } + function getUserTriggerDecision() { + return globals.telemetry.logger + .query({ + metricName: 'codewhisperer_userTriggerDecision', + }) + .map((e) => collectionUtil.partialClone(e, 3, ['credentialStartUrl'], '[omitted]')) + } + for (const [name, invokeCompletion] of [ ['automatic', async () => await vscode.commands.executeCommand('type', { text: '\n' })], ['manual', async () => Commands.tryExecute('aws.amazonq.invokeInlineCompletion')], @@ -101,7 +131,7 @@ describe('Amazon Q Inline', async function () { let originalEditorContents: string | undefined describe('supported filetypes', () => { - beforeEach(async () => { + async function setup() { await setupEditor() /** @@ -119,6 +149,31 @@ describe('Amazon Q Inline', async function () { // wait until the ghost text appears await waitForRecommendations() + } + + beforeEach(async () => { + /** + * Every once and a while the backend won't respond with any recommendations. + * In those cases, re-try the setup up-to ${retries} times + */ + let attempt = 0 + while (attempt < retries) { + try { + await setup() + console.log(`test run ${attempt} succeeded`) + logUserDecisionStatus() + break + } catch (e) { + console.log(`test run ${attempt} failed`) + console.log(e) + logUserDecisionStatus() + attempt++ + await resetCodeWhispererGlobalVariables() + } + } + if (attempt === retries) { + assert.fail(`Failed to invoke ${name} tests after ${attempt} attempts`) + } }) it(`${name} invoke accept`, async function () { diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index dffa40df3ee..4e2b770c1c7 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -62,3 +62,4 @@ export { i18n } from './i18n-helper' export * from './icons' export * as textDocumentUtil from './utilities/textDocumentUtilities' export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants' +export * as collectionUtil from './utilities/collectionUtils' diff --git a/packages/core/src/test/codewhisperer/testUtil.ts b/packages/core/src/test/codewhisperer/testUtil.ts index dfa6c932878..f3b82fd3850 100644 --- a/packages/core/src/test/codewhisperer/testUtil.ts +++ b/packages/core/src/test/codewhisperer/testUtil.ts @@ -28,15 +28,13 @@ import * as model from '../../codewhisperer/models/model' import { stub } from '../utilities/stubber' import { Dirent } from 'fs' // eslint-disable-line no-restricted-imports -export async function resetCodeWhispererGlobalVariables(clearGlobalState: boolean = true) { +export async function resetCodeWhispererGlobalVariables() { vsCodeState.isIntelliSenseActive = false vsCodeState.isCodeWhispererEditing = false CodeWhispererCodeCoverageTracker.instances.clear() globals.telemetry.logger.clear() session.reset() - if (clearGlobalState) { - await globals.globalState.clear() - } + await globals.globalState.clear() await CodeSuggestionsState.instance.setSuggestionsEnabled(true) await RecommendationHandler.instance.clearInlineCompletionStates() }