Auto merge of rust-lang#38008 - bluss:rustbuild-benches, r=alexcrichton

Add rustbuild command `bench`

Add command bench to rustbuild, so that `./x.py bench <path>` can compile and run benchmarks.

`./x.py bench --stage 1 src/libcollections` and `./x.py bench --stage 1 src/libstd` should both compile well. Just `./x.py bench` runs all benchmarks for the libstd crates.

Fixes rust-lang#37897

src/bootstrap/check.rs

@@ -15,6 +15,7 @@
 
 use std::collections::HashSet;
 use std::env;
+use std::fmt;
 use std::fs;
 use std::path::{PathBuf, Path};
 use std::process::Command;
@@ -26,6 +27,34 @@ use util::{self, dylib_path, dylib_path_var};
 
 const ADB_TEST_DIR: &'static str = "/data/tmp";
 
+/// The two modes of the test runner; tests or benchmarks.
+#[derive(Copy, Clone)]
+pub enum TestKind {
+    /// Run `cargo test`
+    Test,
+    /// Run `cargo bench`
+    Bench,
+}
+
+impl TestKind {
+    // Return the cargo subcommand for this test kind
+    fn subcommand(self) -> &'static str {
+        match self {
+            TestKind::Test => "test",
+            TestKind::Bench => "bench",
+        }
+    }
+}
+
+impl fmt::Display for TestKind {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str(match *self {
+            TestKind::Test => "Testing",
+            TestKind::Bench => "Benchmarking",
+        })
+    }
+}
+
 /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
 ///
 /// This tool in `src/tools` will verify the validity of all our links in the
@@ -278,6 +307,7 @@ pub fn krate(build: &Build,
              compiler: &Compiler,
              target: &str,
              mode: Mode,
+             test_kind: TestKind,
              krate: Option<&str>) {
     let (name, path, features, root) = match mode {
         Mode::Libstd => {
@@ -291,15 +321,15 @@ pub fn krate(build: &Build,
         }
         _ => panic!("can only test libraries"),
     };
-    println!("Testing {} stage{} ({} -> {})", name, compiler.stage,
+    println!("{} {} stage{} ({} -> {})", test_kind, name, compiler.stage,
              compiler.host, target);
 
     // Build up the base `cargo test` command.
     //
     // Pass in some standard flags then iterate over the graph we've discovered
     // in `cargo metadata` with the maps above and figure out what `-p`
     // arguments need to get passed.
-    let mut cargo = build.cargo(compiler, mode, target, "test");
+    let mut cargo = build.cargo(compiler, mode, target, test_kind.subcommand());
     cargo.arg("--manifest-path")
          .arg(build.src.join(path).join("Cargo.toml"))
          .arg("--features").arg(features);

src/bootstrap/flags.rs

@@ -49,6 +49,10 @@ pub enum Subcommand {
         paths: Vec<PathBuf>,
         test_args: Vec<String>,
     },
+    Bench {
+        paths: Vec<PathBuf>,
+        test_args: Vec<String>,
+    },
     Clean,
     Dist {
         install: bool,
@@ -141,6 +145,7 @@ Arguments:
                    command == "dist" ||
                    command == "doc" ||
                    command == "test" ||
+                   command == "bench" ||
                    command == "clean"  {
                     println!("Available invocations:");
                     if args.iter().any(|a| a == "-v") {
@@ -163,6 +168,7 @@ println!("\
 Subcommands:
     build       Compile either the compiler or libraries
     test        Build and run some test suites
+    bench       Build and run some benchmarks
     doc         Build documentation
     clean       Clean out build directories
     dist        Build and/or install distribution artifacts
@@ -210,6 +216,14 @@ To learn more about a subcommand, run `./x.py <command> -h`
                     test_args: m.opt_strs("test-args"),
                 }
             }
+            "bench" => {
+                opts.optmulti("", "test-args", "extra arguments", "ARGS");
+                m = parse(&opts);
+                Subcommand::Bench {
+                    paths: remaining_as_path(&m),
+                    test_args: m.opt_strs("test-args"),
+                }
+            }
             "clean" => {
                 m = parse(&opts);
                 if m.free.len() > 0 {
@@ -259,7 +273,8 @@ To learn more about a subcommand, run `./x.py <command> -h`
 impl Subcommand {
     pub fn test_args(&self) -> Vec<&str> {
         match *self {
-            Subcommand::Test { ref test_args, .. } => {
+            Subcommand::Test { ref test_args, .. } |
+            Subcommand::Bench { ref test_args, .. } => {
                 test_args.iter().flat_map(|s| s.split_whitespace()).collect()
             }
             _ => Vec::new(),

src/bootstrap/lib.rs

@@ -141,6 +141,7 @@ struct Crate {
     doc_step: String,
     build_step: String,
     test_step: String,
+    bench_step: String,
 }
 
 /// The various "modes" of invoking Cargo.
@@ -457,7 +458,8 @@ impl Build {
         if self.config.verbose || self.flags.verbose {
             cargo.arg("-v");
         }
-        if self.config.rust_optimize {
+        // FIXME: cargo bench does not accept `--release`
+        if self.config.rust_optimize && cmd != "bench" {
             cargo.arg("--release");
         }
         if self.config.vendor {

src/bootstrap/metadata.rs

@@ -70,6 +70,7 @@ fn build_krate(build: &mut Build, krate: &str) {
                 build_step: format!("build-crate-{}", package.name),
                 doc_step: format!("doc-crate-{}", package.name),
                 test_step: format!("test-crate-{}", package.name),
+                bench_step: format!("bench-crate-{}", package.name),
                 name: package.name,
                 deps: Vec::new(),
                 path: path,

src/bootstrap/step.rs

@@ -11,7 +11,7 @@
 use std::collections::{HashMap, HashSet};
 use std::mem;
 
-use check;
+use check::{self, TestKind};
 use compile;
 use dist;
 use doc;
@@ -268,37 +268,55 @@ pub fn build_rules(build: &Build) -> Rules {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("libtest"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
-                                        Mode::Libstd, Some(&krate.name)));
+                                        Mode::Libstd, TestKind::Test,
+                                        Some(&krate.name)));
     }
     rules.test("check-std-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
          .default(true)
-         .run(move |s| check::krate(build, &s.compiler(), s.target, Mode::Libstd,
-                               None));
+         .run(move |s| check::krate(build, &s.compiler(), s.target,
+                                    Mode::Libstd, TestKind::Test, None));
+
+    // std benchmarks
+    for (krate, path, _default) in krates("std_shim") {
+        rules.bench(&krate.bench_step, path)
+             .dep(|s| s.name("libtest"))
+             .run(move |s| check::krate(build, &s.compiler(), s.target,
+                                        Mode::Libstd, TestKind::Bench,
+                                        Some(&krate.name)));
+    }
+    rules.bench("bench-std-all", "path/to/nowhere")
+         .dep(|s| s.name("libtest"))
+         .default(true)
+         .run(move |s| check::krate(build, &s.compiler(), s.target,
+                                    Mode::Libstd, TestKind::Bench, None));
+
     for (krate, path, _default) in krates("test_shim") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("libtest"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
-                                        Mode::Libtest, Some(&krate.name)));
+                                        Mode::Libtest, TestKind::Test,
+                                        Some(&krate.name)));
     }
     rules.test("check-test-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
          .default(true)
-         .run(move |s| check::krate(build, &s.compiler(), s.target, Mode::Libtest,
-                               None));
+         .run(move |s| check::krate(build, &s.compiler(), s.target,
+                                    Mode::Libtest, TestKind::Test, None));
     for (krate, path, _default) in krates("rustc-main") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("librustc"))
              .host(true)
              .run(move |s| check::krate(build, &s.compiler(), s.target,
-                                        Mode::Librustc, Some(&krate.name)));
+                                        Mode::Librustc, TestKind::Test,
+                                        Some(&krate.name)));
     }
     rules.test("check-rustc-all", "path/to/nowhere")
          .dep(|s| s.name("librustc"))
          .default(true)
          .host(true)
-         .run(move |s| check::krate(build, &s.compiler(), s.target, Mode::Librustc,
-                               None));
+         .run(move |s| check::krate(build, &s.compiler(), s.target,
+                                    Mode::Librustc, TestKind::Test, None));
 
     rules.test("check-linkchecker", "src/tools/linkchecker")
          .dep(|s| s.name("tool-linkchecker"))
@@ -449,6 +467,7 @@ struct Rule<'a> {
 enum Kind {
     Build,
     Test,
+    Bench,
     Dist,
     Doc,
 }
@@ -538,6 +557,11 @@ impl<'a> Rules<'a> {
         self.rule(name, path, Kind::Test)
     }
 
+    fn bench<'b>(&'b mut self, name: &'a str, path: &'a str)
+                -> RuleBuilder<'a, 'b> {
+        self.rule(name, path, Kind::Bench)
+    }
+
     fn doc<'b>(&'b mut self, name: &'a str, path: &'a str)
                -> RuleBuilder<'a, 'b> {
         self.rule(name, path, Kind::Doc)
@@ -583,6 +607,7 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
             "build" => Kind::Build,
             "doc" => Kind::Doc,
             "test" => Kind::Test,
+            "bench" => Kind::Bench,
             "dist" => Kind::Dist,
             _ => return,
         };
@@ -606,6 +631,7 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
             Subcommand::Build { ref paths } => (Kind::Build, &paths[..]),
             Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]),
             Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]),
+            Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]),
             Subcommand::Dist { install } => {
                 if install {
                     return vec![self.sbuild.name("install")]

src/libcollections/Cargo.toml

@@ -15,3 +15,7 @@ rustc_unicode = { path = "../librustc_unicode" }
 [[test]]
 name = "collectionstest"
 path = "../libcollectionstest/lib.rs"
+
+[[bench]]
+name = "collectionstest"
+path = "../libcollectionstest/lib.rs"

src/libcompiler_builtins/Cargo.toml

@@ -8,6 +8,7 @@ version = "0.0.0"
 name = "compiler_builtins"
 path = "lib.rs"
 test = false
+bench = false
 
 [dependencies]
 core = { path = "../libcore" }

src/libcore/Cargo.toml

@@ -7,7 +7,12 @@ version = "0.0.0"
 name = "core"
 path = "lib.rs"
 test = false
+bench = false
 
 [[test]]
 name = "coretest"
 path = "../libcoretest/lib.rs"
+
+[[bench]]
+name = "coretest"
+path = "../libcoretest/lib.rs"

src/libpanic_abort/Cargo.toml

@@ -6,6 +6,7 @@ version = "0.0.0"
 [lib]
 path = "lib.rs"
 test = false
+bench = false
 
 [dependencies]
 core = { path = "../libcore" }

src/libpanic_unwind/Cargo.toml

@@ -6,6 +6,7 @@ version = "0.0.0"
 [lib]
 path = "lib.rs"
 test = false
+bench = false
 
 [dependencies]
 alloc = { path = "../liballoc" }

src/librustc_unicode/Cargo.toml

@@ -7,6 +7,7 @@ version = "0.0.0"
 name = "rustc_unicode"
 path = "lib.rs"
 test = false
+bench = false
 
 [dependencies]
 core = { path = "../libcore" }

src/libunwind/Cargo.toml

@@ -8,6 +8,7 @@ build = "build.rs"
 name = "unwind"
 path = "lib.rs"
 test = false
+bench = false
 
 [dependencies]
 core = { path = "../libcore" }

src/rustc/libc_shim/Cargo.toml

@@ -16,6 +16,7 @@ build = "build.rs"
 name = "libc"
 path = "../../liblibc/src/lib.rs"
 test = false
+bench = false
 
 [dependencies]
 core = { path = "../../libcore" }