Tests: Sync various test suites and add enforce with structural test

Gruntfile.js

@@ -68,90 +68,83 @@ module.exports = function( grunt ) {
 						path.resolve( "./build/coverage-bridge.js" ),
 						require.resolve( "grunt-contrib-qunit/chrome/bridge" )
 					],
+
+					// @HTML_FILES
 					urls: [
 						"test/index.html",
+
+						"test/amd.html",
 						"test/autostart.html",
-						"test/startError.html",
-						"test/reorder.html",
-						"test/reorderError1.html",
-						"test/reorderError2.html",
 						"test/events-filters.html",
 						"test/events-in-test.html",
+						"test/headless.html",
 						"test/logs.html",
-						"test/amd.html",
-						"test/reporter-html/legacy-markup.html",
-						"test/reporter-html/no-qunit-element.html",
-						"test/reporter-html/single-testid.html",
-						"test/reporter-html/window-onerror.html",
-						"test/reporter-html/window-onerror-preexisting-handler.html",
-						"test/reporter-html/xhtml-escape-details-source.xhtml",
-						"test/reporter-html/xhtml-single-testid.xhtml",
-						"test/reporter-urlparams.html",
-						"test/reporter-urlparams-hidepassed.html",
+						"test/module-filter.html",
+						"test/module-skip.html",
+						"test/module-todo.html",
 						"test/moduleId.html",
-						"test/onerror/inside-test.html",
-						"test/seed.html",
+						"test/only-each.html",
 						"test/overload.html",
+						"test/performance-mark.html",
 						"test/preconfigured.html",
-						"test/regex-filter.html",
 						"test/regex-exclude-filter.html",
+						"test/regex-filter.html",
+						"test/reorder.html",
+						"test/reorderError1.html",
+						"test/reorderError2.html",
+						"test/reporter-urlparams-hidepassed.html",
+						"test/reporter-urlparams.html",
+						"test/sandboxed-iframe.html",
+						"test/seed.html",
+						"test/startError.html",
 						"test/string-filter.html",
-						"test/module-skip.html",
-						"test/module-todo.html",
-						"test/only-each.html",
+						"test/webWorker.html",
+
+						"test/reporter-html/legacy-markup.html",
+						"test/reporter-html/no-qunit-element.html",
+						"test/reporter-html/single-testid.html",
+						"test/reporter-html/window-onerror-preexisting-handler.html",
+						"test/reporter-html/window-onerror.html",
+						"test/reporter-html/xhtml-escape-details-source.xhtml",
+						"test/reporter-html/xhtml-single-testid.xhtml"
 
-						// ensure this is last - it has the potential to drool
-						// and omit subsequent tests during coverage runs
-						"test/sandboxed-iframe.html"
 					].map( file => `http://localhost:${connectPort}/${file}` )
 				}
 			}
 		},
 
-		// Sync with test/index.html and test/mozjs.js
 		"test-on-node": {
 			files: [
-				"test/logs.js",
+
+				// Sync with test/index.html and test/mozjs.js
 				"test/main/test.js",
 				"test/main/each.js",
 				"test/main/assert.js",
-				"test/main/assert/step.js",
-				"test/main/assert/timeout.js",
+				"test/main/assert-step.js",
+				"test/main/assert-timeout.js",
 				"test/main/async.js",
 				"test/main/promise.js",
 				"test/main/dump.js",
 				"test/main/modules.js",
 				"test/main/deepEqual.js",
 				"test/main/stack.js",
 				"test/main/utilities.js",
+				"test/main/onError.js",
 				"test/main/onUncaughtException.js",
-				"test/events-in-test.js",
-				"test/onerror/inside-test.js",
 				"test/setTimeout.js",
+
+				// Sync with test/*.html files that also make sense for Node.js
+				"test/events-in-test.js",
+				"test/logs.js",
+				"test/module-skip.js",
+				"test/module-todo.js",
+
 				"test/node/storage-1.js",
 				"test/node/storage-2.js",
 
-				"test/cli/fixtures/only/test.js",
-				"test/cli/fixtures/only/module.js",
-				"test/cli/fixtures/only/module-flat.js",
-
 				"test/es2018/async-functions.js",
 				"test/es2018/rejects.js",
 				"test/es2018/throws.js"
-
-				// FIXME: These tests use an ugly hack that re-opens
-				// an already finished test run. This only works reliably
-				// via the HTML Reporter thanks to some delays in the bridge.
-				// These tests are about reporting, not about functional
-				// behaviour. They would be best run either as reflection on the
-				// DOM in an HTML Reporter test, or from the CLI by asserting
-				// TAP output. I suggest we do the latter, and then remove them
-				// from here.
-				//
-				// Ref https://github.com/qunitjs/qunit/issues/1511
-				//
-				// "test/module-skip.js",
-				// "test/module-todo.js",
 			]
 		}
 	} );

test/cli/.eslintrc.json

@@ -1,6 +1,6 @@
 {
   "parserOptions": {
-    "ecmaVersion": 2017
+    "ecmaVersion": 2018
   },
   "env": {
     "node": true,

test/cli/structure.js

@@ -0,0 +1,78 @@
+const fs = require( "fs" );
+const glob = require( "tiny-glob/sync" );
+
+// This is a meta test to validate structural expectations of our
+// tests, such as checking for fixture files that aren't used or
+// missing from one of the test targets.
+QUnit.module( "structure", () => {
+
+	QUnit.module( "test/main/*.js", () => {
+		const files = fs.readdirSync( __dirname + "/../main/" )
+			.map( file => `main/${file}` );
+
+		QUnit.test( "files", assert => {
+			assert.true( files.length > 5, "found files" );
+			assert.deepEqual(
+				files.filter( file => file.endsWith( ".js" ) ),
+				files,
+				"js files"
+			);
+		} );
+
+		QUnit.test( "index.html", assert => {
+			const contents = fs.readFileSync( __dirname + "/../index.html", "utf8" );
+			files.forEach( file => {
+				assert.true( contents.includes( file ), file );
+			} );
+		} );
+
+		QUnit.test( "test-on-node", assert => {
+			const raw = fs.readFileSync( __dirname + "/../../Gruntfile.js", "utf8" );
+			const contents = raw.match( /test-on-node.*?\{.*?\}/s )[ 0 ];
+
+			files.forEach( file => {
+				assert.true( contents.includes( file ), file );
+			} );
+		} );
+
+		QUnit.test( "mozjs", assert => {
+			const contents = fs.readFileSync( __dirname + "/../mozjs.js", "utf8" );
+			files.forEach( file => {
+				assert.true( contents.includes( file ), file );
+			} );
+		} );
+
+		QUnit.test( "Web Worker", assert => {
+			const contents = fs.readFileSync( __dirname + "/../webWorker-worker.js", "utf8" );
+			files.forEach( file => {
+				assert.true( contents.includes( file ), file );
+			} );
+		} );
+	} );
+
+	QUnit.module( "test/**.html", () => {
+
+		// Get a list of the HTML files, including in subdirectories (e.g. "test/reporter-html/").
+		// Ignore file names containing "--", which are subresources (e.g. iframes).
+		const files = glob( "**/*.html", {
+			cwd: __dirname + "/../",
+			filesOnly: true
+		} )
+			.filter( file => !file.includes( "--" ) )
+			.map( file => `test/${file}` );
+
+		QUnit.test( "files", assert => {
+			assert.true( files.length > 5, "found files" );
+		} );
+
+		QUnit.test( "grunt-contrib-qunit", assert => {
+			const raw = fs.readFileSync( __dirname + "/../../Gruntfile.js", "utf8" );
+			const contents = raw.match( /@HTML_FILES.*?\[.*?\]/s )[ 0 ];
+
+			files.forEach( file => {
+				assert.true( contents.includes( file ), file );
+			} );
+		} );
+	} );
+} );
+

test/headless.html

@@ -2,27 +2,33 @@
 <html>
 <head>
 	<meta charset="UTF-8">
-	<title>QUnit Headless Test Suite</title>
+	<title>Headless HTML Reporter</title>
 	<link rel="stylesheet" href="../qunit/qunit.css">
 	<script src="../qunit/qunit.js"></script>
 	<script src="main/test.js"></script>
 	<script src="main/deepEqual.js"></script>
 	<script>
 		( function() {
-			function createCallback( logType ) {
-				QUnit[ logType ]( function() {
-					console.log( logType, arguments );
+			function createCallback( eventName ) {
+				QUnit.on( eventName, function( eventData ) {
+					console.log( eventName, eventData );
 				} );
 			}
 
-			var logs = [ "begin", "testStart", "testDone", "log", "moduleStart", "moduleDone", "done" ];
-			for ( var i = 0; i < logs.length; i++ ) {
-				createCallback( logs[ i ] );
+			// For debugging
+			if ( self.__grunt_contrib_qunit__ === undefined ) {
+				var logs = [ "runStart", "testStart", "testEnd", "runEnd" ];
+				for ( var i = 0; i < logs.length; i++ ) {
+					createCallback( logs[ i ] );
+				}
 			}
 		}() );
 	</script>
 </head>
 <body>
+	<!-- There is no div#qunit element, which means there should
+	     not be any default UI on this page.
+	 -->
 	<div id="qunit-fixture">test markup</div>
 </body>
 </html>

test/index.html

@@ -9,21 +9,22 @@
 	<script src="main/test.js"></script>
 	<script src="main/each.js"></script>
 	<script src="main/assert.js"></script>
-	<script src="main/assert/step.js"></script>
-	<script src="main/assert/timeout.js"></script>
+	<script src="main/assert-step.js"></script>
+	<script src="main/assert-timeout.js"></script>
 	<script src="main/async.js"></script>
 	<script src="main/promise.js"></script>
 	<script src="main/dump.js"></script>
 	<script src="main/modules.js"></script>
 	<script src="main/deepEqual.js"></script>
 	<script src="main/stack.js"></script>
 	<script src="main/utilities.js"></script>
+	<script src="main/onError.js"></script>
 	<script src="main/onUncaughtException.js"></script>
 	<script src="setTimeout.js"></script>
-	<script src="reporter-html/reporter-html.js"></script>
+
 	<script src="reporter-html/diff.js"></script>
+	<script src="reporter-html/reporter-html.js"></script>
 	<script src="reporter-html/unhandled-rejection.js"></script>
-	<script src="onerror/inside-test.js"></script>
 </head>
 <body>
 	<div id="qunit"></div>

test/mozjs.js

@@ -40,15 +40,16 @@ QUnit.on( "runEnd", ( suiteEnd ) => {
 loadRelativeToScript( "../test/main/test.js" );
 loadRelativeToScript( "../test/main/each.js" );
 loadRelativeToScript( "../test/main/assert.js" );
-loadRelativeToScript( "../test/main/assert/step.js" );
-// Requires setTimeout, loadRelativeToScript( "../test/main/assert/timeout.js" );
-// Requires setTimeout, loadRelativeToScript( "../test/main/async.js" );
+loadRelativeToScript( "../test/main/assert-step.js" );
+// loadRelativeToScript( "../test/main/assert-timeout.js" ); // Requires setTimeout
+// loadRelativeToScript( "../test/main/async.js" ); // Requires setTimeout
 loadRelativeToScript( "../test/main/promise.js" );
 loadRelativeToScript( "../test/main/dump.js" );
-// Requires setTimeout, loadRelativeToScript( "../test/main/modules.js" );
+// loadRelativeToScript( "../test/main/modules.js" ); // Requires setTimeout
 loadRelativeToScript( "../test/main/deepEqual.js" );
 loadRelativeToScript( "../test/main/stack.js" );
 loadRelativeToScript( "../test/main/utilities.js" );
+loadRelativeToScript( "../test/main/onError.js" );
 loadRelativeToScript( "../test/main/onUncaughtException.js" );
 
 QUnit.start();

test/only-each.html

@@ -2,10 +2,10 @@
 <html>
 <head>
 	<meta charset="UTF-8">
-	<title>QUnit.test.each.only Suite</title>
+	<title>only-each</title>
 	<link rel="stylesheet" href="../qunit/qunit.css">
 	<script src="../qunit/qunit.js"></script>
-	<script src="main/only-each.js"></script>
+	<script src="./only-each.js"></script>
 </head>
 <body>
 	<div id="qunit"></div>

test/performance-mark.js

@@ -1,5 +1,5 @@
 QUnit.module( "urlParams performance mark module", function() {
-	QUnit.test( "shouldn't fail if performance marks are cleared ", function( assert ) {
+	QUnit.test( "shouldn't fail if performance marks are cleared", function( assert ) {
 		performance.clearMarks();
 
 		assert.true( true );

test/reporter-html/xhtml-escape-details-source.xhtml

@@ -6,7 +6,7 @@
 	<title>QUnit Main Test Suite</title>
 	<link rel="stylesheet" href="../../qunit/qunit.css" />
 	<script src="../../qunit/qunit.js"></script>
-	<script src="test-escape-details-source.js"></script>
+	<script src="xhtml-escape-details-source.js"></script>
 </head>
 <body>
 	<div id="qunit"></div>

test/sandboxed-iframe-contents.html → test/sandboxed-iframe--contents.html

@@ -2,7 +2,7 @@
 <html>
 	<head>
 		<meta charset="UTF-8">
-		<title>QUnit in Sandboxed Iframe Test Suite - Inside Iframe</title>
+		<title>sandboxed iframe contents</title>
 		<link rel="stylesheet" href="../qunit/qunit.css">
 		<script src="../qunit/qunit.js"></script>
 		<script src="sandboxed-iframe.js"></script>

test/sandboxed-iframe.html

@@ -2,9 +2,9 @@
 <html>
 <head>
 	<meta charset="UTF-8">
-	<title>QUnit in Sandboxed Iframe Test Suite</title>
+	<title>sandboxed iframe</title>
 </head>
 <body>
-	<iframe src="sandboxed-iframe-contents.html" sandbox="allow-scripts"></iframe>
+	<iframe src="sandboxed-iframe--contents.html" sandbox="allow-scripts"></iframe>
 </body>
 </html>

test/sandboxed-iframe.js

@@ -1,3 +1,12 @@
+// For debugging
+if ( self.__grunt_contrib_qunit__ === undefined ) {
+	self.__grunt_contrib_qunit__ = function() {
+		var args = [].slice.call( arguments );
+		args.unshift( "[grunt-contrib-qunit]" );
+		console.log.apply( console, args );
+	};
+}
+
 QUnit.module( "QUnit.only", function( hooks ) {
 	var testsRun = 0;