TINY-10604: LambdaTest improvements

CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Fixed
+- LambdaTest tunnel now correctly shutdowns if Webdriver creation fails
+- Webdriver now uses tunnel name for LambdaTest tunnels #TINY-10604
+
+### Improved
+- Tests now correctly reflect their status in LambdaTest dashboards
+- Tests use the `--name` argument to correclty name test runs in LambdaTest dashboards
+
 ## 14.1.1 - 2024-01-09
 
 ### Added

modules/server/src/main/ts/BedrockAuto.ts

@@ -32,8 +32,10 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
 
   const routes = RunnerRoutes.generate('auto', settings.projectdir, settings.basedir, settings.config, settings.bundler, settings.testfiles, settings.chunk, settings.retries, settings.singleTimeout, settings.stopOnFailure, basePage, settings.coverage, settings.polyfills);
 
+  const shutdownServices: (() => Promise<void>)[] = [];
+  const shutdown = (services: (() => Promise<void>)[]) => () => Promise.allSettled(services.map((fn) => fn()));
+
   routes.then(async (runner) => {
-    const shutdownServices: (() => Promise<any>)[] = [];
 
     // LambdaTest Tunnel must know dev server port, but tunnel must be created before dev server.
     const servicePort = await portfinder.getPortPromise({
@@ -54,6 +56,7 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
     if (remoteWebdriver == 'aws') {
         console.log('INFO: Webdriver creation waits for device farm session to activate. Takes 30-45s.');
     }
+
     const driver = await Driver.create({
       browser: browserName,
       basedir: settings.basedir,
@@ -71,7 +74,9 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
       devicefarmRegion: settings.devicefarmRegion,
       deviceFarmArn: settings.devicefarmArn,
       browserVersion: settings.browserVersion,
-      platformName: settings.platformName
+      platformName: settings.platformName,
+      tunnel,
+      name: settings.name ? settings.name : 'bedrock-auto'
     });
 
     const webdriver = driver.webdriver;
@@ -85,8 +90,6 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
     });
     shutdownServices.push(service.shutdown, driver.shutdown);
 
-    const shutdown = () => Promise.allSettled(shutdownServices.map((shutdown_fn) => shutdown_fn()));
-
     try {
       if (!isHeadless) {
         console.log('bedrock-auto ' + Version.get() + ' available at: ' + location);
@@ -106,13 +109,15 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
         return Reporter.writePollExit(settings, data);
       });
 
-      return Lifecycle.done(result, webdriver, shutdown, settings.gruntDone, settings.delayExit);
+      return Lifecycle.done(result, webdriver, shutdown(shutdownServices), settings.gruntDone, settings.delayExit);
     } catch (e) {
-      return Lifecycle.error(e, webdriver, shutdown, settings.gruntDone, settings.delayExit);
+      return Lifecycle.error(e, webdriver, shutdown(shutdownServices), settings.gruntDone, settings.delayExit);
     }
-  }).catch((err) => {
+  }).catch(async (err) => {
     // Chalk does not use a formatter. Using node's built-in to expand Objects, etc.
     console.error(chalk.red('Error creating webdriver', format(err)));
+    // Shutdown tunnels in case webdriver fails
+    await shutdown(shutdownServices)();
     Lifecycle.exit(settings.gruntDone, ExitCodes.failures.unexpected);
   });
 };

modules/server/src/main/ts/bedrock/auto/Driver.ts

@@ -9,6 +9,7 @@ import * as DriverLoader from './DriverLoader';
 import * as RemoteDriver from './RemoteDriver';
 import deepmerge = require('deepmerge');
 import { RemoteOptions } from 'webdriverio';
+import { Tunnel } from './Tunnel';
 
 export interface DriverSettings {
   basedir: string;
@@ -30,6 +31,8 @@ export interface DriverSettings {
   deviceFarmArn?: string;
   platformName?: string;
   browserVersion: string;
+  tunnel?: Tunnel;
+  name?: string;
 }
 
 export interface Driver {

modules/server/src/main/ts/bedrock/auto/RemoteDriver.ts

@@ -52,7 +52,7 @@ const createFarm = async (browserName: string, remoteOpts: WebdriverIO.RemoteOpt
       webdriver: driver,
       shutdown: (_: boolean | undefined) => {
         console.log('Shutting down Device Farm. This currently does nothing.');
-        return Promise.resolve();
+        return driver.deleteSession();
       }
     };
   } catch (e) {
@@ -70,14 +70,29 @@ export const getApi = async (settings: DriverSettings, browser: string, opts: We
     const driver = await WebdriverIO.remote(opts);
     return {
       webdriver: driver,
-      shutdown: () => Promise.resolve()
+      shutdown: () => driver.deleteSession()
     };
   }
   return Promise.reject('Unrecognized remote provider: [' + remoteWebdriver + ']');
 };
 
 const addDriverSpecificOpts = (opts: WebdriverIO.RemoteOptions, settings: DriverSettings): WebdriverIO.RemoteOptions => {
   if (settings.remoteWebdriver === 'lambdatest') {
+    // For naming in LT we use PROJECT_BUILD[_NAME] or BUILD
+    const getProjectNaming = (name: string) => {
+      const names = name.split('_');
+      if (names.length > 1) {
+        return {
+          project: names[0],
+          build: names[1],
+          ...(names.length > 2 ? { name: names[2] } : {})
+        };
+      } else {
+        return { build: names[0] };
+      }
+    };
+    const names = settings.name ? getProjectNaming(settings.name) : {};
+    const tunnelName = settings.tunnel?.name ? { tunnelName: settings.tunnel.name } : {};
     const platformName = settings.platformName ? { platformName: settings.platformName } : {};
     return deepmerge(opts, {
       user: settings.username,
@@ -90,7 +105,9 @@ const addDriverSpecificOpts = (opts: WebdriverIO.RemoteOptions, settings: Driver
           console: true,
           w3c: true,
           plugin: 'node_js-webdriverio',
-          ...platformName
+          ...platformName,
+          ...tunnelName,
+          ...names
         }
       }
     });

modules/server/src/main/ts/bedrock/auto/Tunnel.ts

@@ -6,6 +6,7 @@ import { Tunnel as LambdaTunnel } from '@lambdatest/node-tunnel';
 
 export interface Tunnel {
   url: URL;
+  name?: string;
   shutdown: () => Promise<void>;
 }
 
@@ -85,8 +86,11 @@ const createSSH = async (port: number | string, domain: string): Promise<Tunnel>
 // and no proper typing for it. Excuse the hard type casting
 const createLambda = async (port: number | string, credentials: LambdaCredentials): Promise<Tunnel> => {
   const tunnel = new LambdaTunnel();
+  const suffix = crypto.randomUUID();
+  const tunnelName = 'bedrock-tunnel-' + suffix;
 
   const tunnelArguments = {
+    tunnelName,
     user: credentials.user,
     key: credentials.key,
     port: port.toString()
@@ -99,6 +103,7 @@ const createLambda = async (port: number | string, credentials: LambdaCredential
 
   const result: Tunnel = {
     url: new URL('http://localhost:' + port),
+    name: tunnelName,
     shutdown
   };
 

modules/server/src/main/ts/bedrock/core/Lifecycle.ts

@@ -12,6 +12,13 @@ const exitDelay = (driver: Browser, delayExiting: boolean) => {
   return delayExiting ? driver.pause(17 * 60 * 1000) : Promise.resolve();
 };
 
+// TINY-10604: Mark test runs for LambdaTest dashboard
+const markLambdaTest = async (driver: Browser, status: 'failed' | 'passed') => {
+  // Tests running on LambdaTests machines should have access to the `lambda_status` var.
+  // Setting this var will mark the result of the test for the LT dashboard
+  await driver.executeScript('if (window.lambda_status) { lambda_status=' + status + ' }', []);
+};
+
 export const exit = (gruntDone: GruntDoneFn, exitCode: number): void => {
   if (gruntDone !== undefined) {
     gruntDone(exitCode === 0);
@@ -23,10 +30,12 @@ export const exit = (gruntDone: GruntDoneFn, exitCode: number): void => {
 export const done = async (result: Attempt<string[], TestResult[]>, driver: Browser, shutdown: ShutdownFn, gruntDone: GruntDoneFn, delayExiting: boolean): Promise<void> => {
   // Only delay exiting if tests failed.
   const exitCode = await Attempt.cata(result, async (errs) => {
+    await markLambdaTest(driver, 'failed');
     await exitDelay(driver, delayExiting);
     console.log(chalk.red(errs.join('\n')));
     return ExitCodes.failures.tests;
   }, async () => {
+    await markLambdaTest(driver, 'passed');
     console.log(chalk.green('All tests passed.'));
     return ExitCodes.success;
   });