Compress library files

TurboWarp · Feb 12, 2022 · 173a834 · 173a834
1 parent 48a3988
commit 173a834
Show file tree

Hide file tree

Showing 5 changed files with 97 additions and 30 deletions.
diff --git a/scripts/download-library-files.js b/scripts/download-library-files.js
@@ -1,22 +1,27 @@
 const fs = require('fs');
 const pathUtil = require('path');
 const Limiter = require('async-limiter');
-const https = require('https');
 const crypto = require('crypto');
-const {fetch} = require('./lib');
 const promisify = require('util').promisify;
+const zlib = require('zlib');
+
+const compress = promisify(zlib.brotliCompress);
 const writeFile = promisify(fs.writeFile);
+const {fetch} = require('./lib');
 
-const libraryFiles = pathUtil.join(__dirname, '..', 'library-files');
+const libraryFiles = pathUtil.join(__dirname, '..', 'static', 'library-files');
 if (!fs.existsSync(libraryFiles)) {
   console.log('Making library files folder');
-  fs.mkdirSync(libraryFiles);
+  fs.mkdirSync(libraryFiles, {
+    recursive: true
+  });
 }
 
-const costumesManifest = pathUtil.join(__dirname, '..', 'node_modules', 'scratch-gui', 'src', 'lib', 'libraries', 'costumes.json');
-const backdropManifest = pathUtil.join(__dirname, '..', 'node_modules', 'scratch-gui', 'src', 'lib', 'libraries', 'backdrops.json');
-const spriteManifest = pathUtil.join(__dirname, '..', 'node_modules', 'scratch-gui', 'src', 'lib', 'libraries', 'sprites.json');
-const soundManifest = pathUtil.join(__dirname, '..', 'node_modules', 'scratch-gui', 'src', 'lib', 'libraries', 'sounds.json');
+const guiLibraryFolder = pathUtil.join(__dirname, '..', 'node_modules', 'scratch-gui', 'src', 'lib', 'libraries');
+const costumesManifest = pathUtil.join(guiLibraryFolder, 'costumes.json');
+const backdropManifest = pathUtil.join(guiLibraryFolder, 'backdrops.json');
+const spriteManifest = pathUtil.join(guiLibraryFolder, 'sprites.json');
+const soundManifest = pathUtil.join(guiLibraryFolder, 'sounds.json');
 if (!fs.existsSync(costumesManifest)) {
   throw new Error('costumes.json does not exist -- did you forget a step?');
 }
@@ -35,37 +40,36 @@ const backdrops = JSON.parse(fs.readFileSync(backdropManifest));
 const sprites = JSON.parse(fs.readFileSync(spriteManifest));
 const sounds = JSON.parse(fs.readFileSync(soundManifest));
 
-const httpsAgent = new https.Agent({
-	keepAlive: true
-});
+const md5 = (buffer) => crypto.createHash('md5').update(new Uint8Array(buffer)).digest('hex');
 
-const usedFiles = new Set();
+const allCompressedFiles = [];
 
 const downloadAsset = async (asset) => {
   const md5ext = asset.md5ext;
-  if (usedFiles.has(md5ext)) {
-    return;
+  if (!/^[0-9a-f]+\.[a-z]+$/gi.test(md5ext)) {
+    throw new Error(`invalid md5ext: ${md5ext}`);
   }
-  usedFiles.add(md5ext);
-  const path = pathUtil.join(libraryFiles, md5ext);
-  if (fs.existsSync(path)) {
-    console.log(`Already exists: ${md5ext}`);
+
+  const compressedName = `${md5ext}.br`;
+  allCompressedFiles.push(compressedName);
+  const compressedPath = pathUtil.join(libraryFiles, compressedName);
+  if (fs.existsSync(compressedPath)) {
+    console.log(`Already downloaded: ${md5ext}`);
     return;
   }
 
   console.log(`Downloading: ${md5ext}`);
-  const response = await fetch(`https://assets.scratch.mit.edu/${md5ext}`, {
-    agent: httpsAgent
-  });
-  const arrayBuffer = await response.buffer();
+  const response = await fetch(`https://assets.scratch.mit.edu/${md5ext}`);
+  const uncompressed = await response.buffer();
 
   const expectedHash = asset.assetId;
-  const hash = crypto.createHash('md5').update(new Uint8Array(arrayBuffer)).digest("hex")
+  const hash = md5(uncompressed);
   if (hash !== expectedHash) {
     throw new Error(`${md5ext}: Hash mismatch: expected ${expectedHash} but found ${hash}`);
   }
 
-  await writeFile(path, Buffer.from(arrayBuffer));
+  const compressed = await compress(uncompressed);
+  await writeFile(compressedPath, Buffer.from(compressed));
 };
 
 const limiter = new Limiter({
@@ -101,7 +105,7 @@ console.time('Download assets');
 
 limiter.onDone(() => {
   for (const file of fs.readdirSync(libraryFiles)) {
-    if (!usedFiles.has(file)) {
+    if (!allCompressedFiles.includes(file)) {
       console.warn(`Extraneous: ${file}`);
     }
   }

diff --git a/scripts/lib.js b/scripts/lib.js
@@ -1,6 +1,12 @@
+const https = require('https');
 const fetch = require('node-fetch');
 
-const persistentFetch = async (url, opts) => {
+const httpsAgent = new https.Agent({
+  keepAlive: true
+});
+
+const persistentFetch = async (url, opts = {}) => {
+  opts.agent = httpsAgent;
   let err;
   for (let i = 0; i < 3; i++) {
     try {

diff --git a/src/main/index.js b/src/main/index.js
@@ -13,6 +13,7 @@ import * as store from './store';
 import './crash';
 import parseArgs from './parse-args';
 import {isDevelopment, isMac, isLinux, staticDir} from './environment';
+import './library-files';
 
 const readFile = util.promisify(fs.readFile);
 const writeFile = util.promisify(fs.writeFile);

diff --git a/src/main/library-files.js b/src/main/library-files.js
@@ -0,0 +1,60 @@
+import {app, protocol} from 'electron';
+import pathUtil from 'path';
+import fs from 'fs';
+import {promisify} from 'util';
+import {brotliDecompress} from 'zlib';
+import {staticDir} from './environment';
+
+const readFile = promisify(fs.readFile);
+const decompress = promisify(brotliDecompress);
+
+protocol.registerSchemesAsPrivileged([
+  {
+    scheme: 'tw-library-files',
+    privileges: {
+      supportFetchAPI: true
+    }
+  }
+]);
+
+const mimeTypes = new Map();
+mimeTypes.set('wav', 'audio/wav');
+mimeTypes.set('svg', 'image/svg+xml');
+mimeTypes.set('png', 'image/png');
+
+const decompressAsset = async (md5ext) => {
+  const extension = md5ext.split('.')[1];
+  if (!mimeTypes.has(extension)) {
+    throw new Error('Unknown extension: ' + extension);
+  }
+  const baseDirectory = pathUtil.join(staticDir, 'library-files/');
+  const compressedFile = pathUtil.join(baseDirectory, `${md5ext}.br`);
+  if (!compressedFile.startsWith(baseDirectory)) {
+    throw new Error('Path traversal');
+  }
+  const compressedData = await readFile(compressedFile);
+  const decompressed = await decompress(compressedData);
+  return {
+    data: decompressed,
+    type: mimeTypes.get(extension)
+  };
+};
+
+app.whenReady().then(() => {
+  protocol.registerBufferProtocol('tw-library-files', (request, callback) => {
+    const md5ext = new URL(request.url).pathname;
+    decompressAsset(md5ext)
+      .then((data) => {
+        callback({
+          data: data.data,
+          mimeType: data.type
+        });
+      })
+      .catch((err) => {
+        console.error('Cannot read library file', err);
+        callback({
+          statusCode: 404
+        });
+      });
+  });
+});
diff --git a/webpack.renderer.js b/webpack.renderer.js
@@ -72,10 +72,6 @@ module.exports = defaultConfig => {
                 {
                     from: 'node_modules/scratch-blocks/media',
                     to: 'static/blocks-media'
-                },
-                {
-                    from: libraryFilesFolder,
-                    to: 'library-files'
                 }
             ])
         ],