From 33e68472337d285260ca6e66f12f92bf879cfdfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vojt=C4=9Bch=20Jel=C3=ADnek?= Date: Sat, 18 May 2019 16:43:45 +0200 Subject: [PATCH] Speed webpack compilation (#6751) * speed up webpack compilation * compile webpack before running server * add explaining comment * add quicker compile to tests + formatting * add typechecking * dont specify files in tests * add explaining comment * use prod mode in backend test * set DEV_MODE to true * remove trailing space --- core/tests/karma.conf.ts | 19 +++++- package-lock.json | 129 +++++++++++++++++++++++++++++++++++ package.json | 3 + scripts/run_backend_tests.sh | 2 +- scripts/start.sh | 5 +- webpack.config.ts | 2 + webpack.dev.config.ts | 29 ++++++-- webpack.prod.config.ts | 24 +++++-- 8 files changed, 197 insertions(+), 16 deletions(-) diff --git a/core/tests/karma.conf.ts b/core/tests/karma.conf.ts index 50d898ea7558..8bc7cba42b7a 100644 --- a/core/tests/karma.conf.ts +++ b/core/tests/karma.conf.ts @@ -1,4 +1,6 @@ var argv = require('yargs').argv; +var ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); +var path = require('path'); var generatedJs = 'third_party/generated/js/third_party.js'; if (argv.prodEnv) { generatedJs = ( @@ -151,9 +153,22 @@ module.exports = function(config) { module: { rules: [{ test: /\.ts$/, - use: 'ts-loader', + use: [ + 'cache-loader', + 'thread-loader', + { + loader: 'ts-loader', + options: { + // this is needed for thread-loader to work correctly + happyPackMode: true + } + } + ] }] - } + }, + plugins: [ + new ForkTsCheckerWebpackPlugin({ checkSyntacticErrors: true }) + ] } }); }; diff --git a/package-lock.json b/package-lock.json index cfb099cc8afa..e34c47cef0ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1063,6 +1063,65 @@ "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==", "dev": true }, + "babel-code-frame": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", + "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=", + "dev": true, + "requires": { + "chalk": "^1.1.3", + "esutils": "^2.0.2", + "js-tokens": "^3.0.2" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "js-tokens": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", + "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=", + "dev": true + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true + } + } + }, "babel-eslint": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.0.1.tgz", @@ -1495,6 +1554,12 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true }, + "buffer-json": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-json/-/buffer-json-2.0.0.tgz", + "integrity": "sha512-+jjPFVqyfF1esi9fvfUs3NqM0pH1ziZ36VP4hmA/y/Ssfo/5w5xHKfTw9BwQjoJ1w/oVtpLomqwUHKdefGyuHw==", + "dev": true + }, "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", @@ -1593,6 +1658,28 @@ "unset-value": "^1.0.0" } }, + "cache-loader": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/cache-loader/-/cache-loader-3.0.1.tgz", + "integrity": "sha512-HzJIvGiGqYsFUrMjAJNDbVZoG7qQA+vy9AIoKs7s9DscNfki0I589mf2w6/tW+kkFH3zyiknoWV5Jdynu6b/zw==", + "dev": true, + "requires": { + "buffer-json": "^2.0.0", + "find-cache-dir": "^2.1.0", + "loader-utils": "^1.2.3", + "mkdirp": "^0.5.1", + "neo-async": "^2.6.1", + "schema-utils": "^1.0.0" + }, + "dependencies": { + "neo-async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", + "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==", + "dev": true + } + } + }, "call-me-maybe": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz", @@ -3877,6 +3964,22 @@ "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", "dev": true }, + "fork-ts-checker-webpack-plugin": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-1.3.3.tgz", + "integrity": "sha512-XrSkBRxE3MfCSFfMSmbEI/LvCYV9UPUWN1RDfF6r5jXnrhR5qzqxEHxjCwQlnN/uH7IlSfnaVtD3FLXQBznX5w==", + "dev": true, + "requires": { + "babel-code-frame": "^6.22.0", + "chalk": "^2.4.1", + "chokidar": "^2.0.4", + "micromatch": "^3.1.10", + "minimatch": "^3.0.4", + "semver": "^5.6.0", + "tapable": "^1.0.0", + "worker-rpc": "^0.1.0" + } + }, "form-data": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", @@ -6823,6 +6926,12 @@ "integrity": "sha512-gdUU1Fwj5ep4kplwcmftruWofEFt6lfpkkr3h860CXbAB9c3hGb55EOL2ali0Td5oebvW0E1+3Sr+Ur7XfKpRA==", "dev": true }, + "microevent.ts": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/microevent.ts/-/microevent.ts-0.1.1.tgz", + "integrity": "sha512-jo1OfR4TaEwd5HOrt5+tAZ9mqT4jmpNAusXtyfNzqVm9uiSYFZlKM1wYL4oU7azZW/PxQW53wM0S6OR1JHNa2g==", + "dev": true + }, "micromatch": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", @@ -10143,6 +10252,17 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "thread-loader": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/thread-loader/-/thread-loader-2.1.2.tgz", + "integrity": "sha512-7xpuc9Ifg6WU+QYw/8uUqNdRwMD+N5gjwHKMqETrs96Qn+7BHwECpt2Brzr4HFlf4IAkZsayNhmGdbkBsTJ//w==", + "dev": true, + "requires": { + "loader-runner": "^2.3.1", + "loader-utils": "^1.1.0", + "neo-async": "^2.6.0" + } + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -11401,6 +11521,15 @@ "errno": "~0.1.7" } }, + "worker-rpc": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/worker-rpc/-/worker-rpc-0.1.1.tgz", + "integrity": "sha512-P1WjMrUB3qgJNI9jfmpZ/htmBEjFh//6l/5y8SD9hg1Ef5zTTVVoRjTrTEzPrNBQvmhMxkoTsjOXN10GWU7aCg==", + "dev": true, + "requires": { + "microevent.ts": "~0.1.1" + } + }, "wrap-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", diff --git a/package.json b/package.json index b6121c7e5871..aaec7acbd789 100644 --- a/package.json +++ b/package.json @@ -40,12 +40,14 @@ "ajv": "^6.10.0", "babel-eslint": "^10.0.1", "browserstack-local": "^1.3.7", + "cache-loader": "^3.0.1", "clean-webpack-plugin": "^2.0.1", "dotenv": "^7.0.0", "enhanced-resolve": "^4.1.0", "eslint": "^5.16.0", "eslint-plugin-angular": "^4.0.0", "eslint-plugin-html": "^5.0.3", + "fork-ts-checker-webpack-plugin": "^1.3.3", "gulp": "^4.0.1", "gulp-concat": "^2.6.1", "htmllint": "^0.8.0", @@ -65,6 +67,7 @@ "protractor-screenshot-reporter": "0.0.5", "stylelint": "^10.0.1", "stylelint-config-standard": "^18.3.0", + "thread-loader": "^2.1.2", "ts-loader": "^5.4.4", "typescript": "^3.4.5", "uglify-js": "^3.5.8", diff --git a/scripts/run_backend_tests.sh b/scripts/run_backend_tests.sh index ac681f9413d8..0eece12d1966 100755 --- a/scripts/run_backend_tests.sh +++ b/scripts/run_backend_tests.sh @@ -90,7 +90,7 @@ echo "Compiling typescript..." $NODE_MODULE_DIR/typescript/bin/tsc --project . echo "Compiling webpack..." -$NODE_MODULE_DIR/webpack/bin/webpack.js --config webpack.dev.config.ts +$NODE_MODULE_DIR/webpack/bin/webpack.js --config webpack.prod.config.ts $PYTHON_CMD scripts/build.py $PYTHON_CMD scripts/backend_tests.py $@ diff --git a/scripts/start.sh b/scripts/start.sh index df5960d95a1e..4e7c5d11042e 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -100,15 +100,18 @@ rm assets/constants.js.bak # Set up a local dev instance. # TODO(sll): do this in a new shell. -echo Starting GAE development server # To turn emailing on, add the option '--enable_sendmail=yes' and change the relevant # settings in feconf.py. Be careful with this -- you do not want to spam people # accidentally! if ! [[ "$FORCE_PROD_MODE" == "True" ]]; then ($NODE_PATH/bin/node $NODE_MODULE_DIR/gulp/bin/gulp.js watch)& + # In prod mode webpack is launched through scripts/build.py + echo Compiling webpack... + $NODE_MODULE_DIR/webpack/bin/webpack.js --config webpack.dev.config.ts ($NODE_MODULE_DIR/webpack/bin/webpack.js --config webpack.dev.config.ts --watch)& fi +echo Starting GAE development server (python $GOOGLE_APP_ENGINE_HOME/dev_appserver.py $CLEAR_DATASTORE_ARG $ENABLE_CONSOLE_ARG --admin_host 0.0.0.0 --admin_port 8000 --host 0.0.0.0 --port 8181 --skip_sdk_update_check true app.yaml)& # Wait for the servers to come up. diff --git a/webpack.config.ts b/webpack.config.ts index 3c0e8f0de4db..b14b306ff59f 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -13,6 +13,7 @@ // limitations under the License. var CleanWebpackPlugin = require('clean-webpack-plugin'); +var ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); var HtmlWebpackPlugin = require('html-webpack-plugin'); var htmlMinifyConfig = { @@ -352,5 +353,6 @@ module.exports = { minify: htmlMinifyConfig, inject: false }), + new ForkTsCheckerWebpackPlugin({ checkSyntacticErrors: true }) ] }; diff --git a/webpack.dev.config.ts b/webpack.dev.config.ts index de6d414750e3..2d202ae2cf9c 100644 --- a/webpack.dev.config.ts +++ b/webpack.dev.config.ts @@ -26,12 +26,29 @@ module.exports = { entry: commonWebpackConfig.entries, plugins: commonWebpackConfig.plugins, module: { - rules: [ - { - test: /\.ts$/, - use: 'ts-loader', - } - ] + rules: [{ + test: /\.ts$/, + include: [ + path.resolve(__dirname, 'core/templates/dev/head'), + path.resolve(__dirname, 'typings') + ], + use: [ + 'cache-loader', + { + loader: 'thread-loader', + options: { + poolTimeout: Infinity, + } + }, + { + loader: 'ts-loader', + options: { + // this is needed for thread-loader to work correctly + happyPackMode: true + } + } + ] + }] }, output: { filename: '[name].bundle.js', diff --git a/webpack.prod.config.ts b/webpack.prod.config.ts index 5529b2580dbb..7c268bca3f33 100644 --- a/webpack.prod.config.ts +++ b/webpack.prod.config.ts @@ -26,12 +26,24 @@ module.exports = { entry: commonWebpackConfig.entries, plugins: commonWebpackConfig.plugins, module: { - rules: [ - { - test: /\.ts$/, - use: 'ts-loader', - } - ] + rules: [{ + test: /\.ts$/, + include: [ + path.resolve(__dirname, 'core/templates/dev/head'), + path.resolve(__dirname, 'typings') + ], + use: [ + 'cache-loader', + 'thread-loader', + { + loader: 'ts-loader', + options: { + // this is needed for thread-loader to work correctly + happyPackMode: true + } + } + ] + }] }, output: { filename: '[name].[contenthash].bundle.js',