Build: Add scheduled job to verify our reproducible builds

Ref #1560.

.github/workflows/reproducible.yaml

@@ -0,0 +1,26 @@
+name: Reproducible builds
+on:
+  # Once a week on Monday at 00:30 UTC
+  schedule:
+    - cron: '30 0 * * 1'
+  # Or manually
+  workflow_dispatch:
+  # Or when developing this workflow
+  push:
+    paths:
+      - .github/workflows/reproducible.yaml
+
+jobs:
+  run:
+    name: Verify releases
+    if: ${{ github.repository == 'qunitjs/qunit' }} # skip on forks, noisy cron
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js 12
+        uses: actions/setup-node@v2
+        with:
+          node-version: 12.x
+
+      - run: node build/reproducible-builds.js

build/dist-replace.js

@@ -16,6 +16,14 @@ if ( /pre/.test( distVersion ) ) {
 
 const replacements = {
 
+	// Normalize CRLF from fuzzysort.js.
+	//
+	// The way we upload files to npm, Git, and the jQuery CDN all normalize
+	// CRLF to LF. Thus, if we don't do this ourselves during the build, then
+	// reproducible build verification would find that the distribution is
+	// not identical to the reproduced build artefact.
+	"\r\n": "\n",
+
 	// Embed version
 	"@VERSION": distVersion
 };

build/reproducible-builds.js

@@ -0,0 +1,189 @@
+// Helper for the "Reproducible builds" job.
+
+const cp = require( "child_process" );
+const fs = require( "fs" );
+const path = require( "path" );
+const util = require( "util" );
+
+const utils = require( "./utils.js" );
+const execFile = util.promisify( cp.execFile );
+const tempDir = path.join( __dirname, "../temp", "reproducible-builds" );
+const SRC_REPO = "https://github.com/qunitjs/qunit.git";
+
+/**
+ * How many past releases to verify.
+ *
+ * Note that qunit@<2.16.0 were not fully reproducible.
+ *
+ * qunit@<=2.14.1 embedded a timestamp in the file header. This would have to be
+ * ignored (or replaced with the timestamp found in the files you compare against).
+ * In the 2.14.1, timestamps were removed from the output. Also, prior to 2.14.1,
+ * the build wrote files to "/dist" instead of "/qunit".
+ *
+ * [email protected] contained some CR (\r) characters in comments from fuzzysort.js,
+ * which got normalized to LF (\n) by Git, npm, and the CDN on their own. This was
+ * fixed in [email protected] by removing the comment in question, and [email protected]
+ * normalizes CRLF during the build.
+ */
+const VERIFY_COUNT = 2;
+
+async function buildRelease( version, cacheDir = null ) {
+	console.log( `... ${version}: checking out the source` );
+
+	const gitDir = path.join( tempDir, `git-${version}` );
+	utils.cleanDir( gitDir );
+
+	await execFile( "git", [ "clone", "-q", "-b", version, "--depth=5", SRC_REPO, gitDir ] );
+
+	// Remove any artefacts that were checked into Git
+	utils.cleanDir( gitDir + "/qunit/" );
+
+	// Use sync for npm-ci to avoid concurrency bugs with shared cache
+	console.log( `... ${version}: installing development dependencies from npm` );
+	cp.execFileSync( "npm", [ "ci" ], {
+		env: {
+			npm_config_cache: cacheDir,
+			npm_config_update_notifier: "false",
+			PATH: process.env.PATH,
+			PUPPETEER_DOWNLOAD_PATH: path.join( cacheDir, "puppeteer_download" )
+		},
+		cwd: gitDir
+	} );
+
+	console.log( `... ${version}: building release` );
+	await execFile( "npm", [ "run", "build" ], {
+		env: {
+			PATH: process.env.PATH
+		},
+		cwd: gitDir
+	} );
+
+	return {
+		js: fs.readFileSync( gitDir + "/qunit/qunit.js", "utf8" ),
+		css: fs.readFileSync( gitDir + "/qunit/qunit.css", "utf8" )
+	};
+}
+
+const Reproducible = {
+	async fetch() {
+
+		// Keep the stuff that matters in memory. Below, we will run unaudited npm dev deps
+		// as part of build commands, which can modify anything on disk.
+		const releases = {};
+
+		{
+			console.log( "Setting up temp directory..." );
+
+			// This can take a while when running it locally (not CI),
+			// as it first need to remove any old builds.
+			utils.cleanDir( tempDir );
+		}
+		{
+			console.log( "Fetching releases from jQuery CDN..." );
+			const cdnIndexUrl = "https://releases.jquery.com/resources/cdn.json";
+			const data = JSON.parse( await utils.download( cdnIndexUrl ) );
+
+			for ( const release of data.qunit.all.slice( 0, VERIFY_COUNT ) ) {
+				releases[ release.version ] = {
+					cdn: {
+						js: await utils.download( `https://code.jquery.com/${release.filename}` ),
+						css: await utils.download( `https://code.jquery.com/${release.theme}` )
+					}
+				};
+			}
+		}
+		{
+			console.log( "Fetching releases from npmjs.org..." );
+			const npmIndexUrl = "https://registry.npmjs.org/qunit";
+			const data = JSON.parse( await utils.download( npmIndexUrl ) );
+
+			for ( const version of Object.keys( data.versions ).slice( -VERIFY_COUNT ) ) {
+				if ( !releases[ version ] ) {
+					releases[ version ] = {};
+				}
+
+				const tarball = data.versions[ version ].dist.tarball;
+				const tarFile = path.join( tempDir, `npm-${version}${path.extname( tarball )}` );
+				await utils.downloadFile( tarball, tarFile );
+
+				releases[ version ].npm = {
+					js: cp.execFileSync(
+						"tar", [ "-xOf", tarFile, "package/qunit/qunit.js" ],
+						{ encoding: "utf8" }
+					),
+					css: cp.execFileSync(
+						"tar", [ "-xOf", tarFile, "package/qunit/qunit.css" ],
+						{ encoding: "utf8" }
+					)
+				};
+			}
+		}
+		{
+			console.log( "Reproducing release builds..." );
+
+			const cacheDir = path.join( tempDir, "cache" );
+			utils.cleanDir( cacheDir );
+
+			// Start the builds in parallel and await results.
+			// Let the first error propagate and ignore others (avoids "Unhandled rejection" later).
+			const buildPromises = [];
+			for ( const version in releases ) {
+				releases[ version ].buildPromise = buildRelease( version, cacheDir );
+				buildPromises.push( releases[ version ].buildPromise );
+			}
+			await Promise.all( buildPromises );
+
+			const diffs = [];
+			for ( const version in releases ) {
+				const release = releases[ version ];
+				const build = await release.buildPromise;
+
+				// For [email protected], normalize CRLF to match what Git and npm did during upload.
+				if ( version === "2.15.0" ) {
+					build.js = utils.normalizeEOL( build.js );
+				}
+
+				let verified = true;
+				for ( const distro of [ "cdn", "npm" ] ) {
+					for ( const file of [ "js", "css" ] ) {
+						if ( release[ distro ][ file ] !== build[ file ] ) {
+							verified = false;
+							console.error(
+								`QUnit ${version} ${file} from ${distro} differs from build`
+							);
+							diffs.push( [
+								{ name: `qunit-${version}-build.${file}`,
+									contents: build[ file ] },
+								{ name: `qunit-${version}-${distro}.${file}`,
+									contents: release[ distro ][ file ] }
+							] );
+						}
+					}
+				}
+				if ( verified ) {
+					console.log( `QUnit ${version} is reproducible and matches distributions!` );
+				}
+			}
+
+			diffs.forEach( diff => {
+				const fromFile = path.join( tempDir, diff[ 0 ].name );
+				const toFile = path.join( tempDir, diff[ 1 ].name );
+				fs.writeFileSync( fromFile, utils.verboseNonPrintable( diff[ 0 ].contents ) );
+				fs.writeFileSync( toFile, utils.verboseNonPrintable( diff[ 1 ].contents ) );
+				process.stdout.write(
+					utils.getDiff( fromFile, toFile, { ignoreWhitespace: false } )
+				);
+			} );
+			if ( diffs.length ) {
+				throw new Error( "One or more distributions differ from the reproduced build" );
+			}
+		}
+	}
+};
+
+( async function main() {
+	await Reproducible.fetch();
+}() ).catch( e => {
+	console.error( e.toString() );
+	process.exit( 1 );
+} );

build/review-package.js

@@ -6,35 +6,9 @@
 
 const cp = require( "child_process" );
 const fs = require( "fs" );
-const https = require( "https" );
 const path = require( "path" );
 const readline = require( "readline" );
-
-function getDiff( from, to ) {
-
-	// macOS 10.15+ comes with GNU diff (2.8)
-	// https://unix.stackexchange.com/a/338960/37512
-	// https://stackoverflow.com/a/41770560/319266
-	const gnuDiffVersion = cp.execFileSync( "diff", [ "--version" ], { encoding: "utf8" } );
-	const versionStr = /diff.* (\d+\.\d+)/.exec( gnuDiffVersion );
-	const isOld = ( versionStr && Number( versionStr[ 1 ] ) < 3.4 );
-
-	try {
-		cp.execFileSync( "diff", [
-			"-w",
-			"--text",
-			"--unified",
-			...( isOld ? [] : [ "--color=always" ] ),
-			from,
-			to
-		], { encoding: "utf8" } );
-	} catch ( e ) {
-
-		// Expected, `diff` command yields non-zero exit status if files differ
-		return e.stdout;
-	}
-	throw new Error( `Unable to diff between ${from} and ${to}` );
-}
+const { getDiff, downloadFile } = require( "./utils.js" );
 
 async function confirm( text ) {
 	const rl = readline.createInterface( { input: process.stdin, output: process.stdout } );
@@ -50,16 +24,6 @@ async function confirm( text ) {
 	} );
 }
 
-async function download( url, dest ) {
-	const fileStr = fs.createWriteStream( dest );
-	await new Promise( ( resolve, reject ) => {
-		https.get( url, ( resp ) => {
-			resp.pipe( fileStr );
-			fileStr.on( "finish", () => fileStr.close( resolve ) );
-		} ).on( "error", ( err ) => reject( err ) );
-	} );
-}
-
 const ReleaseAssets = {
 	async audit( prevVersion ) {
 		if ( typeof prevVersion !== "string" || !/^\d+\.\d+\.\d+$/.test( prevVersion ) ) {
@@ -86,7 +50,7 @@ const ReleaseAssets = {
 
 			const prevUrl = `https://code.jquery.com/qunit/qunit-${prevVersion}.js`;
 			const tempPrevPath = path.join( __dirname, "../temp", file );
-			await download( prevUrl, tempPrevPath );
+			await downloadFile( prevUrl, tempPrevPath );
 
 			const currentPath = path.join( __dirname, "../qunit", file );
 			process.stdout.write( getDiff( tempPrevPath, currentPath ) );
@@ -98,7 +62,7 @@ const ReleaseAssets = {
 
 			const prevUrl = `https://code.jquery.com/qunit/qunit-${prevVersion}.css`;
 			const tempPrevPath = path.join( __dirname, "../temp", file );
-			await download( prevUrl, tempPrevPath );
+			await downloadFile( prevUrl, tempPrevPath );
 
 			const currentPath = path.join( __dirname, "../qunit", file );
 			process.stdout.write( getDiff( tempPrevPath, currentPath ) );

build/utils.js

@@ -0,0 +1,90 @@
+const cp = require( "child_process" );
+const fs = require( "fs" );
+const https = require( "https" );
+
+function getDiff( from, to, options = {} ) {
+
+	// macOS 10.15+ comes with GNU diff (2.8)
+	// https://unix.stackexchange.com/a/338960/37512
+	// https://stackoverflow.com/a/41770560/319266
+	const gnuDiffVersion = cp.execFileSync( "diff", [ "--version" ], { encoding: "utf8" } );
+	const versionStr = /diff.* (\d+\.\d+)/.exec( gnuDiffVersion );
+	const isOld = ( versionStr && Number( versionStr[ 1 ] ) < 3.4 );
+
+	try {
+		cp.execFileSync( "diff", [
+			...( options.ignoreWhitespace !== false ? [ "-w" ] : [] ),
+			"--text",
+			"--unified",
+			...( isOld ? [] : [ "--color=always" ] ),
+			from,
+			to
+		], { encoding: "utf8" } );
+	} catch ( e ) {
+
+		// Expected, `diff` command yields non-zero exit status if files differ
+		return e.stdout;
+	}
+	throw new Error( `Unable to diff between ${from} and ${to}` );
+}
+
+async function download( url ) {
+	return new Promise( ( resolve, reject ) => {
+		https.get( url, async resp => {
+			try {
+				const chunks = [];
+				for await ( const chunk of resp ) {
+					chunks.push( Buffer.from( chunk ) );
+				}
+				resolve( Buffer.concat( chunks ).toString( "utf8" ) );
+			} catch ( err ) {
+				reject( err );
+			}
+		} );
+	} );
+}
+
+async function downloadFile( url, dest ) {
+	const fileStr = fs.createWriteStream( dest );
+	return new Promise( ( resolve, reject ) => {
+		https.get( url, resp => {
+			resp.pipe( fileStr );
+			fileStr.on( "finish", () => fileStr.close( resolve ) );
+		} ).on( "error", err => reject( err ) );
+	} );
+}
+
+function cleanDir( dirPath ) {
+	if ( fs.existsSync( dirPath ) ) {
+		fs.rmdirSync( dirPath, { recursive: true } );
+	}
+	fs.mkdirSync( dirPath, { recursive: true } );
+}
+
+// Turn invisible chars and non-ASCII chars into escape sequences.
+//
+// This is like `cat --show-nonprinting` and makes diffs easier to understand
+// when e.g. there is an added/removed line with a Window-style CRLF which
+// would otherwise look the same in both lines.
+function verboseNonPrintable( str ) {
+
+	// Match all chars that are not printable ASCII,
+	// except \t (U+0009) and \n (U+000A).
+	return str.replace( /[^\t\n\u0020-\u007F]/g, function( m ) {
+		return `U+${m.codePointAt( 0 ).toString( 16 ).toUpperCase().padStart( 4, "0" )}`;
+	} );
+}
+
+function normalizeEOL( str ) {
+	return str.replace( /\r\n/g, "\n" );
+}
+
+
+module.exports = {
+	getDiff,
+	download,
+	downloadFile,
+	cleanDir,
+	verboseNonPrintable,
+	normalizeEOL
+};