feat: default to goma enabled, add support for cache-only goma (elect…

…ron#66) * fix: put goma.gn in the args * feat: default to goma enabled, add support for cache-only goma
kjvenalainen · Jan 30, 2020 · 635d114 · 635d114
1 parent d1e94ec
commit 635d114
Show file tree

Hide file tree

Showing 12 changed files with 190 additions and 120 deletions.
diff --git a/README.md b/README.md
@@ -254,7 +254,7 @@ e test --runners=main
 | e show exe        | The path of the built Electron executable                      |
 | e show root       | The path of the root directory from `e init --root`.           |
 | e show src [name] | The path of the named (default: electron) source dir           |
-| e show stats      | SCCache build statistics                                       |
+| e show stats      | Build statistics                                               |
 
 Example usage:
 

diff --git a/example-configs/evm.base.yml b/example-configs/evm.base.yml
@@ -8,10 +8,7 @@ origin:
   node: https_or_ssh_url
 gen:
   args:
-    # path to sccache for faster builds (https://github.com/mozilla/sccache)
-    - cc_wrapper="path/to/your/developer/folder/electron/src/electron/external_binaries/sccache"
+    # path to goma for faster builds (https://notgoma.com)
+    - import("path/to/your/developer/folder/build-tools/third_party/goma")
 env:
   GIT_CACHE_PATH: /Users/use_name/.git_cache
-  SCCACHE_BUCKET: electronjs-sccache-ci
-  SCCACHE_CACHE_SIZE: 40G
-  SCCACHE_TWO_TIER: true
diff --git a/package.json b/package.json
@@ -18,6 +18,7 @@
     "chalk": "^2.4.1",
     "command-exists": "^1.2.8",
     "commander": "^3.0.2",
+    "cross-zip": "^3.0.0",
     "got": "^10.2.2",
     "hasha": "^5.1.0",
     "js-yaml": "^3.13.1",
@@ -35,8 +36,12 @@
     "rimraf": "^3.0.0"
   },
   "lint-staged": {
-    "*.js": ["prettier --write"],
-    "e": ["prettier --write"]
+    "*.js": [
+      "prettier --write"
+    ],
+    "e": [
+      "prettier --write"
+    ]
   },
   "husky": {
     "hooks": {

diff --git a/src/e b/src/e
@@ -105,7 +105,8 @@ program
     }
     let cwd;
     if (args[0] === 'goma_ctl' || args[0] === 'goma_auth') {
-      cwd = goma.dir(evmConfig.current().root);
+      goma.downloadAndPrepare();
+      cwd = goma.dir;
       args[0] = `${args[0]}.py`;
       args.unshift('python');
     }

diff --git a/src/e-build.js b/src/e-build.js
@@ -8,7 +8,6 @@ const program = require('commander');
 const evmConfig = require('./evm-config');
 const { fatal } = require('./utils/logging');
 const depot = require('./utils/depot-tools');
-const sccache = require('./utils/sccache');
 const goma = require('./utils/goma');
 
 function runGNGen(config) {
@@ -28,20 +27,21 @@ function ensureGNGen(config) {
 }
 
 function runNinja(config, target, ninjaArgs) {
-  if (config.goma) {
-    if (goma.exists(config.root)) {
+  if (config.goma !== 'none') {
+    if (config.goma === 'cluster') {
       const authenticated = goma.isAuthenticated(config.root);
       if (!authenticated) {
-        throw new Error('Goma not authenticated.');
-      } else {
-        goma.ensure(config.root);
+        throw new Error(
+          `Goma not authenticated. Either sign in with ${color.cmd(
+            'e d goma_auth login',
+          )} or change your config.goma value to 'cache-only'`,
+        );
       }
     }
+    goma.ensure();
     if (!ninjaArgs.includes('-j') && !ninjaArgs.find(arg => /^-j[0-9]+$/.test(arg.trim()))) {
       ninjaArgs.push('-j', process.platform === 'darwin' ? 50 : 200);
     }
-  } else {
-    sccache.ensure(config);
   }
 
   depot.ensure(config);

diff --git a/src/e-init.js b/src/e-init.js
@@ -10,7 +10,6 @@ const evmConfig = require('./evm-config');
 const { color, fatal } = require('./utils/logging');
 const { resolvePath, ensureDir } = require('./utils/paths');
 const goma = require('./utils/goma');
-const sccache = require('./utils/sccache');
 const depot = require('./utils/depot-tools');
 
 function getVSReleaseLine() {
@@ -27,12 +26,8 @@ function createConfig(options) {
   // build the `gn gen` args
   const gn_args = [`import("//electron/build/args/${options.import}.gn")`];
 
-  if (options.useGoma) {
-    if (goma.exists(root)) {
-      gn_args.push('import("//electron/build/args/goma.gn")');
-    }
-  } else {
-    gn_args.push(`cc_wrapper="${sccache.exec(root)}"`);
+  if (options.goma !== 'none') {
+    gn_args.push(`import("${goma.gnFilePath}")`);
   }
 
   if (options.asan) gn_args.push('is_asan=true');
@@ -47,18 +42,8 @@ function createConfig(options) {
     platform_env.GYP_MSVS_VERSION = getVSReleaseLine();
   }
 
-  // sccache-specific environment variables
-  const sccacheEnv = {
-    SCCACHE_BUCKET: process.env.SCCACHE_BUCKET || 'electronjs-sccache-ci',
-    SCCACHE_CACHE_SIZE: process.env.SCCACHE_CACHE_SIZE || '20G',
-    SCCACHE_DIR: process.env.SCCACHE_DIR
-      ? resolvePath(process.env.SCCACHE_DIR)
-      : path.resolve(homedir, '.sccache'),
-    SCCACHE_TWO_TIER: process.env.SCCACHE_TWO_TIER || 'true',
-  };
-
   return {
-    goma: options.useGoma,
+    goma: options.goma,
     root,
     origin: {
       electron: options.useHttps
@@ -78,7 +63,6 @@ function createConfig(options) {
         ? resolvePath(process.env.GIT_CACHE_PATH)
         : path.resolve(homedir, '.git_cache'),
       ...platform_env,
-      ...(!options.useGoma && sccacheEnv),
     },
   };
 }
@@ -145,9 +129,9 @@ program
   .option('--lsan', `When building, enable clang's leak sanitizer`, false)
   .option('--bootstrap', 'Run `e sync` and `e build` after creating the build config.')
   .option(
-    '--use-goma',
-    `Use Electron's custom deployment of Goma (only available to maintainers).`,
-    false,
+    '--goma <target>',
+    `Use Electron's custom deployment of Goma.  Can be "cache-only", "cluster" or "none".  The "cluster" mode is only available to maintainers`,
+    'cache-only',
   )
   .option(
     '--use-https',
@@ -173,6 +157,15 @@ try {
     throw Error(`Build config ${color.config(name)} already exists! (${color.path(filename)})`);
   }
 
+  // Make sure the goma options are valid
+  if (!['none', 'cache-only', 'cluster'].includes(options.goma)) {
+    throw new Error(
+      `Config property ${color.config('goma')} must be one of ${color.config(
+        'cache-only',
+      )} or ${color.config('cluster')} but you provided ${color.config(options.goma)}`,
+    );
+  }
+
   // save the new config
   const config = createConfig(options);
   ensureRoot(config);
@@ -186,11 +179,13 @@ try {
 
   // (maybe) run sync to ensure external binaries are downloaded
   if (program.bootstrap) {
-    childProcess.execFileSync('node', [e, 'sync', '-v'], opts);
+    childProcess.execFileSync('node', [e, 'sync', '-v', '--ignore_locks'], opts);
   }
 
   // maybe authenticate with Goma
-  if (config.goma) goma.auth(config.root);
+  if (config.goma === 'cluster') {
+    goma.auth();
+  }
 
   // (maybe) build Electron
   if (program.bootstrap) {

diff --git a/src/e-show.js b/src/e-show.js
@@ -6,7 +6,7 @@ const program = require('commander');
 
 const evmConfig = require('./evm-config');
 const { color, fatal } = require('./utils/logging');
-const sccache = require('./utils/sccache');
+const goma = require('./utils/goma');
 
 function gitStatus(config) {
   const exec = 'git';
@@ -152,13 +152,12 @@ program
 
 program
   .command('stats')
-  .description('sccache build statistics')
+  .description('build statistics')
   .action(() => {
     try {
       const config = evmConfig.current();
-      const exec = sccache.exec(config.root);
-      const options = { env: config.env, stdio: 'inherit' };
-      childProcess.execFileSync(exec, ['--show-stats'], options);
+      const options = { env: { ...process.env, ...config.env }, stdio: 'inherit', cwd: goma.dir };
+      childProcess.execFileSync('python', ['goma_ctl.py', 'stat'], options);
     } catch (e) {
       fatal(e);
     }

diff --git a/src/evm-config.js b/src/evm-config.js
@@ -5,6 +5,7 @@ const path = require('path');
 const yml = require('js-yaml');
 const { color } = require('./utils/logging');
 const { ensureDir } = require('./utils/paths');
+const goma = require('./utils/goma');
 
 const configRoot = process.env.EVM_CONFIG || path.resolve(__dirname, '..', 'configs');
 const currentFile = path.resolve(configRoot, 'evm-current.txt');
@@ -103,8 +104,52 @@ function loadConfigFileRaw(name) {
   return maybeExtendConfig(yml.safeLoad(configContents));
 }
 
+function sanitizeConfig(config) {
+  if (!['none', 'cluster', 'cache-only'].includes(config.goma)) {
+    config.goma = 'cache-only';
+    console.warn(
+      `${color.warn} Your evm config ${color.config(
+        currentName(),
+      )} does not define the ${color.config(
+        'goma',
+      )} property as one of "none", "cluster" or "cache-only", we are defaulting to to ${color.config(
+        'cache-only',
+      )} for you`,
+    );
+  }
+  if (
+    config.goma !== 'none' &&
+    (!config.gen || !config.gen.args || !config.gen.args.find(arg => arg.includes(goma.gnFilePath)))
+  ) {
+    config.gen.args.push(`import("${goma.gnFilePath}")`);
+    console.warn(
+      `${color.warn} Your evm config ${color.config(
+        currentName(),
+      )} has goma enabled but did not include the goma gn file "${color.path(
+        goma.gnFilePath,
+      )}" in the gen args. We've put it there for you`,
+    );
+  }
+  if (
+    config.goma !== 'none' &&
+    config.gen &&
+    config.gen.args &&
+    config.gen.args.find(arg => arg.includes('cc_wrapper'))
+  ) {
+    config.gen.args = config.gen.args.filter(arg => !arg.includes('cc_wrapper'));
+    console.warn(
+      `${color.warn} Your evm config ${color.config(
+        currentName(),
+      )} has goma enabled but also defines a ${color.config(
+        'cc_wrapper',
+      )} argument, we have removed it for you`,
+    );
+  }
+  return config;
+}
+
 module.exports = {
-  current: () => loadConfigFileRaw(currentName()),
+  current: () => sanitizeConfig(loadConfigFileRaw(currentName())),
   currentName,
   execOf,
   names,