TINY-11177: Vastly improve remote testing (#145)

Co-authored-by: Seb <[email protected]>

CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+## Added
+- New server-side APIs to accept a batch of results instead of a single result #TINY-11177
+
+## Changed
+- Reverted TINY-10708 which was a server-side fix
+- Client no longer waits for log requests to complete between tests, which should speed up remote testing #TINY-11177
+- Console HUD no longer updates for individual tests #TINY-11177
+- Client now posts test status only in batches every 30 seconds, this is the only time the console HUD will update #TINY-11177
+
+## Removed
+- Single result server-side API #TINY-11177
+- Server-side monitoring of single test timeouts. This is still monitored client side. #TINY-11177
+- The Promise polyfill is no longer allowed on modern NodeJS frameworks so it has been removed. #TINY-11177
+
 ## 14.1.5 - 2024-10-18
 
 ### Added

Jenkinsfile

@@ -40,7 +40,7 @@ timestamps {
         sh 'yarn build'
       },
       testDirs: [ 'modules/sample/src/test/ts/**/pass' ],
-      custom: '--config modules/sample/tsconfig.json --customRoutes modules/sample/routes.json --polyfills Promise Symbol'
+      custom: '--config modules/sample/tsconfig.json --customRoutes modules/sample/routes.json'
     )
   }
 

lerna.json

@@ -1,7 +1,7 @@
 {
   "npmClient": "yarn",
   "useWorkspaces": true,
-  "version": "14.1.5",
+  "version": "15.0.0-alpha.10",
   "publish": {
     "push": false
   }

modules/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ephox/bedrock-client",
-  "version": "14.1.1",
+  "version": "15.0.0-alpha.10",
   "author": "Tiny Technologies Inc",
   "license": "Apache-2.0",
   "scripts": {
@@ -10,7 +10,7 @@
     "buildAndTest": "yarn prepublishOnly && yarn test"
   },
   "dependencies": {
-    "@ephox/bedrock-common": "^14.1.1",
+    "@ephox/bedrock-common": "^15.0.0-alpha.7",
     "@ephox/dispute": "^1.0.3"
   },
   "main": "./lib/main/ts/api/Main.js",

modules/client/src/main/ts/api/UnitTest.ts

@@ -1,5 +1,5 @@
 import { Context, Failure, TestError, TestLabel, TestLogs } from '@ephox/bedrock-common';
-import { it } from './Bdd';
+import { describe, it } from './Bdd';
 
 type TestLogs = TestLogs.TestLogs;
 type TestError = TestError.TestError;
@@ -10,11 +10,13 @@ export type FailureCallback = (error: TestThrowable, logs?: TestLogs) => void;
 
 /** An asynchronous test with callbacks. */
 export const asyncTest = (name: string, test: (this: Context, success: SuccessCallback, failure: FailureCallback) => void): void => {
-  it(name, function (done) {
-    test.call(this, () => done(), ((err, logs) => {
-      const r = Failure.prepFailure(err, logs);
-      done(r);
-    }));
+  describe('old-style test', () => {
+    it(name, function (done) {
+      test.call(this, () => done(), ((err, logs) => {
+        const r = Failure.prepFailure(err, logs);
+        done(r);
+      }));
+    });
   });
 };
 
@@ -23,10 +25,14 @@ export const asynctest = asyncTest;
 
 /** A synchronous test that fails if an exception is thrown */
 export const test = (name: string, test: (this: Context) => void): void => {
-  it(name, test);
+  describe('old-style test', () => {
+    it(name, test);
+  });
 };
 
 /** Tests an async function (function that returns a Promise). */
 export const promiseTest = (name: string, test: (this: Context) => Promise<void>): void => {
-  it(name, test);
+  describe('old-style test', () => {
+    it(name, test);
+  });
 };

modules/client/tsconfig.json

@@ -4,9 +4,9 @@
     "strict": true,
     "noUnusedLocals": true,
     "useUnknownInCatchVariables": false,
-    "target": "es5",
-    "module": "es2015",
-    "lib": ["es2015", "dom"],
+    "target": "es2019",
+    "module": "es2020",
+    "lib": ["es2022", "dom"],
     "declaration": true,
     "sourceMap": true,
     "outDir": "lib",

modules/common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ephox/bedrock-common",
-  "version": "14.1.1",
+  "version": "15.0.0-alpha.7",
   "author": "Tiny Technologies Inc",
   "license": "Apache-2.0",
   "scripts": {

modules/common/tsconfig.json

@@ -4,12 +4,13 @@
     "strict": true,
     "noUnusedLocals": true,
     "useUnknownInCatchVariables": false,
-    "target": "es5",
-    "module": "es2015",
-    "lib": ["es2015", "dom"],
+    "target": "es2019",
+    "module": "es2020",
+    "lib": ["es2022", "dom"],
     "declaration": true,
     "sourceMap": true,
     "outDir": "lib",
+
     "rootDir": "src",
     "types": []
   },

modules/runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ephox/bedrock-runner",
-  "version": "14.1.1",
+  "version": "15.0.0-alpha.10",
   "author": "Tiny Technologies Inc",
   "license": "Apache-2.0",
   "scripts": {
@@ -10,19 +10,18 @@
     "buildAndTest": "yarn prepublishOnly && yarn test"
   },
   "dependencies": {
-    "@ephox/bedrock-common": "^14.1.1",
-    "@ephox/wrap-promise-polyfill": "^2.2.0",
+    "@ephox/bedrock-common": "^15.0.0-alpha.7",
     "jquery": "^3.4.1",
     "querystringify": "^2.1.1"
   },
   "devDependencies": {
     "@types/diff": "^5.0.0",
     "@types/jquery": "^3.5.3",
     "@types/querystringify": "^2.0.0",
-    "rollup": "^1.20.2",
+    "rollup": "^4.30.1",
     "rollup-plugin-commonjs": "^10.1.0",
     "rollup-plugin-node-resolve": "^5.2.0",
-    "rollup-plugin-sourcemaps": "^0.4.2",
+    "rollup-plugin-sourcemaps": "^0.6.3",
     "sourcemapped-stacktrace": "^1.1.11"
   },
   "main": "./lib/main/ts/api/Main.js",

modules/runner/src/main/ts/api/Main.ts

@@ -1,5 +1,4 @@
 import { Failure, Global } from '@ephox/bedrock-common';
-import Promise from '@ephox/wrap-promise-polyfill';
 import * as Globals from '../core/Globals';
 import * as TestLoader from '../core/TestLoader';
 import { UrlParams } from '../core/UrlParams';
@@ -23,16 +22,16 @@ const setupAndRun = (loadError?: Error) => {
 
   const runner = Runner(Globals.rootSuite(), params, callbacks, reporter, ui);
   runner.init().then((data) => {
-    if (data.mode === 'auto') {
-      // Try to ensure the page has focus
-      window.focus();
-    }
-
     // Run the tests if an error didn't occur during loading
     if (loadError !== undefined) {
       return Promise.reject(loadError);
     } else {
-      return runner.run(data.chunk, data.retries, data.timeout, data.stopOnFailure);
+      const autoMode = data.mode === 'auto';
+      if (autoMode) {
+        // Try to ensure the page has focus
+        window.focus();
+      }
+      return runner.run(data.chunk, data.retries, data.timeout, data.stopOnFailure, autoMode);
     }
   }).catch((e: Error) => {
     console.error('Unexpected error occurred', e);
@@ -43,7 +42,7 @@ const setupAndRun = (loadError?: Error) => {
 };
 
 const run = () => setupAndRun();
-const runError = (e: Error) => setupAndRun(e);
+const runError = (e: Error) => setupAndRun(Failure.prepFailure(e));
 
 const loadAndRun = (scripts: string[]) => {
   // Load the scripts and then run

modules/runner/src/main/ts/core/Globals.ts

@@ -79,6 +79,9 @@ export const afterEach = (title: TitleOrExecuteFn, fn?: ExecuteFn): void => {
 
 export const it = (title: string, fn: ExecuteFn): Test => {
   const suite = getCurrentSuiteOrDie();
+  if (suite === root) {
+    throw new Error('Tests must be in a `describe` block');
+  }
   const test = createTest(title, fn, suite);
 
   suite.tests.push(test);
@@ -123,4 +126,4 @@ export const setup = (global: any = Global): void => {
   });
 };
 
-export const rootSuite = (): Suite => root;
+export const rootSuite = (): Suite => root;

modules/runner/src/main/ts/core/TestLoader.ts

@@ -1,4 +1,3 @@
-import Promise from '@ephox/wrap-promise-polyfill';
 import { ErrorCatcher } from '../errors/ErrorCatcher';
 
 export const load = (scriptUrl: string): Promise<void> =>
@@ -38,4 +37,4 @@ export const load = (scriptUrl: string): Promise<void> =>
 
     // Add the script to the dom to load it
     document.body.appendChild(script);
-  });
+  });

modules/runner/src/main/ts/core/Utils.ts

@@ -1,5 +1,4 @@
 import { Suite, Test } from '@ephox/bedrock-common';
-import Promise from '@ephox/wrap-promise-polyfill';
 import sourceMappedStackTrace from 'sourcemapped-stacktrace';
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -20,8 +19,8 @@ export const makeUrl = (session: string, offset: number, failed: number, skipped
   return baseUrl + makeQueryParams(session, offset, failed, skipped, retry);
 };
 
-export const formatElapsedTime = (start: Date, end: Date): string => {
-  const millis = end.getTime() - start.getTime();
+export const formatElapsedTime = (start: number, end: number): string => {
+  const millis = end - start;
   const seconds = Math.floor(millis / 1000);
   const point = Math.floor(millis - (seconds * 1000) / 100);
   const printable =
@@ -62,4 +61,4 @@ export const setStack = (error: Error, stack: string | undefined): void => {
   } catch (err) {
     // Do nothing
   }
-};
+};

modules/runner/src/main/ts/reporter/Callbacks.ts

@@ -1,7 +1,16 @@
 import { ErrorData, Global } from '@ephox/bedrock-common';
-import Promise from '@ephox/wrap-promise-polyfill';
+
 import { HarnessResponse } from '../core/ServerTypes';
 
+export interface TestReport {
+  readonly file: string;
+  readonly name: string;
+  readonly passed: boolean;
+  readonly time: string;
+  readonly skipped: string | null;
+  readonly error: TestErrorData | null
+}
+
 export interface TestErrorData {
   readonly data: ErrorData;
   readonly text: string;
@@ -11,25 +20,34 @@ export interface Callbacks {
   readonly loadHarness: () => Promise<HarnessResponse>
   readonly sendKeepAlive: (session: string) => Promise<void>;
   readonly sendInit: (session: string) => Promise<void>;
-  readonly sendTestStart: (session: string, totalTests: number, file: string, name: string) => Promise<void>;
-  readonly sendTestResult: (session: string, file: string, name: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string | null) => Promise<void>;
+  readonly sendTestStart: (session: string, number: number, totalTests: number, file: string, name: string) => Promise<void>;
+  readonly sendTestResults: (session: string, results: TestReport[]) => Promise<void>;
   readonly sendDone: (session: string, error?: string) => Promise<void>;
 }
 
 declare const $: JQueryStatic;
 
-const sendJson = <T>(url: string, data: any): Promise<T> => {
+function generateErrorMessage(xhr: JQuery.jqXHR<any>, onError: (reason?: any) => void, url: string, requestDetails: string, statusText: 'timeout' | 'error' | 'abort' | 'parsererror', e: string) {
+  if (xhr.readyState === 0) {
+    onError(`Unable to open connection to ${url}, ${requestDetails}. Status text "${statusText}", error thrown "${e}"`);
+  } else {
+    onError(`Response status ${xhr.status} connecting to ${url}, ${requestDetails}. Status text "${statusText}", error thrown "${e}"`);
+  }
+}
+
+const sendJson = <T>(url: string, jsonData: any): Promise<T> => {
   return new Promise((onSuccess, onError) => {
+    const data = JSON.stringify(jsonData);
     $.ajax({
       method: 'post',
       url,
       contentType: 'application/json; charset=UTF-8',
       dataType: 'json',
       success: onSuccess,
       error: (xhr, statusText, e) => {
-        onError(e);
+        generateErrorMessage(xhr, onError, url, `sending ${data}`, statusText, e);
       },
-      data: JSON.stringify(data),
+      data,
     });
   });
 };
@@ -41,7 +59,7 @@ const getJson = <T>(url: string): Promise<T> => {
       dataType: 'json',
       success: onSuccess,
       error: (xhr, statusText, e) => {
-        onError(e);
+        generateErrorMessage(xhr, onError, url, 'as a get request', statusText, e);
       }
     });
   }));
@@ -64,24 +82,20 @@ export const Callbacks = (): Callbacks => {
     });
   };
 
-  const sendTestStart = (session: string, totalTests: number, file: string, name: string): Promise<void> => {
+  const sendTestStart = (session: string, number: number, totalTests: number, file: string, name: string): Promise<void> => {
     return sendJson('/tests/start', {
+      number,
       totalTests,
       session,
       file,
       name,
     });
   };
 
-  const sendTestResult = (session: string, file: string, name: string, passed: boolean, time: string, error: TestErrorData | null, skipped: string | null): Promise<void> => {
-    return sendJson('/tests/result', {
+  const sendTestResults = (session: string, results: TestReport[]): Promise<void> => {
+    return sendJson('/tests/results', {
       session,
-      file,
-      name,
-      passed,
-      skipped,
-      time,
-      error,
+      results,
     });
   };
 
@@ -101,7 +115,7 @@ export const Callbacks = (): Callbacks => {
     sendInit,
     sendKeepAlive,
     sendTestStart,
-    sendTestResult,
+    sendTestResults,
     sendDone
   };
-};
+};