backends: Make ninja not build test executables by default

When they're marked `build_by_default : false`

mesonbuild/backend/backends.py

@@ -854,15 +854,17 @@ def replace_outputs(self, args, private_dir, output_list):
         return newargs
 
     def get_build_by_default_targets(self):
+        """Get all build and custom targets that must be built by default."""
         result = OrderedDict()
-        # Get all build and custom targets that must be built by default
         for name, t in self.build.get_targets().items():
             if t.build_by_default:
                 result[name] = t
-        # Get all targets used as test executables and arguments. These must
-        # also be built by default. XXX: Sometime in the future these should be
-        # built only before running tests.
-        for t in self.build.get_tests():
+        return result
+
+    @staticmethod
+    def _get_targets(values):
+        result = OrderedDict()
+        for t in values:
             exe = t.exe
             if hasattr(exe, 'held_object'):
                 exe = exe.held_object
@@ -879,6 +881,14 @@ def get_build_by_default_targets(self):
                 result[dep.get_id()] = dep
         return result
 
+    def get_test_targets(self):
+        """Get all targets used as test executables and arguments."""
+        return self._get_targets(self.build.get_tests())
+
+    def get_benchmark_targets(self):
+        """Get all targets used as test executables and arguments."""
+        return self._get_targets(self.build.get_benchmarks())
+
     @lru_cache(maxsize=None)
     def get_custom_target_provided_by_generated_source(self, generated_source):
         libs = []

mesonbuild/backend/ninjabackend.py

@@ -823,7 +823,15 @@ def generate_tests(self):
             cmd += ['--no-stdsplit']
         if self.environment.coredata.get_builtin_option('errorlogs'):
             cmd += ['--print-errorlogs']
-        elem = NinjaBuildElement(self.all_outputs, 'meson-test', 'CUSTOM_COMMAND', ['all', 'PHONY'])
+        tests = self.get_test_targets()
+
+        tests = [os.path.join(self.get_target_dir(t), t.get_outputs()[0])
+                 for t in self.get_test_targets().values()]
+        elem = NinjaBuildElement(self.all_outputs, 'meson-build-tests', 'CUSTOM_COMMAND', 'PHONY')
+        elem.add_dep(tests)
+        self.add_build(elem)
+
+        elem = NinjaBuildElement(self.all_outputs, 'meson-test', 'CUSTOM_COMMAND', ['meson-build-tests', 'PHONY'])
         elem.add_item('COMMAND', cmd)
         elem.add_item('DESC', 'Running all tests.')
         elem.add_item('pool', 'console')
@@ -835,7 +843,13 @@ def generate_tests(self):
         cmd = self.environment.get_build_command(True) + [
             'test', '--benchmark', '--logbase',
             'benchmarklog', '--num-processes=1', '--no-rebuild']
-        elem = NinjaBuildElement(self.all_outputs, 'meson-benchmark', 'CUSTOM_COMMAND', ['all', 'PHONY'])
+        elem = NinjaBuildElement(self.all_outputs, 'meson-build-benchmarks', 'CUSTOM_COMMAND', 'PHONY')
+        benches = [os.path.join(self.get_target_dir(t), t.get_outputs()[0])
+                   for t in self.get_benchmark_targets().values()]
+        elem.add_dep(benches)
+        self.add_build(elem)
+
+        elem = NinjaBuildElement(self.all_outputs, 'meson-benchmark', 'CUSTOM_COMMAND', ['meson-build-benchmarks', 'PHONY'])
         elem.add_item('COMMAND', cmd)
         elem.add_item('DESC', 'Running benchmark suite.')
         elem.add_item('pool', 'console')

mesonbuild/mtest.py

@@ -486,6 +486,9 @@ def run(self) -> TestRun:
     def _run_cmd(self, cmd: typing.List[str]) -> TestRun:
         starttime = time.time()
 
+        if not os.path.exists(cmd[0]):
+            return TestRun(self.test, self.test_env, TestResult.SKIP, 0, 0, '', 'Test Executable not built.', cmd)
+
         if len(self.test.extra_paths) > 0:
             self.env['PATH'] = os.pathsep.join(self.test.extra_paths + ['']) + self.env['PATH']
             winecmd = []
@@ -977,7 +980,7 @@ def list_tests(th: TestHarness) -> bool:
         print(th.get_pretty_suite(t))
     return not tests
 
-def rebuild_all(wd: str) -> bool:
+def rebuild_all(wd: str, benchmark: bool) -> bool:
     if not (Path(wd) / 'build.ninja').is_file():
         print('Only ninja backend is supported to rebuild tests before running them.')
         return True
@@ -987,7 +990,12 @@ def rebuild_all(wd: str) -> bool:
         print("Can't find ninja, can't rebuild test.")
         return False
 
-    ret = subprocess.run([ninja, '-C', wd]).returncode
+    if benchmark:
+        target = 'meson-build-benchmarks'
+    else:
+        target = 'meson-build-tests'
+
+    ret = subprocess.run([ninja, '-C', wd, target]).returncode
     if ret != 0:
         print('Could not rebuild {}'.format(wd))
         return False
@@ -1021,7 +1029,7 @@ def run(options: argparse.Namespace) -> int:
     options.wd = os.path.abspath(options.wd)
 
     if not options.list and not options.no_rebuild:
-        if not rebuild_all(options.wd):
+        if not rebuild_all(options.wd, options.benchmark):
             # We return 125 here in case the build failed.
             # The reason is that exit code 125 tells `git bisect run` that the current commit should be skipped.
             # Thus users can directly use `meson test` to bisect without needing to handle the does-not-build case separately in a wrapper script.

test cases/common/92 default library/meson.build

@@ -1,10 +1,10 @@
 project('default library', 'cpp')
 
 flib = library('ef', 'ef.cpp')
-exe = executable('eftest', 'eftest.cpp', link_with : flib)
+exe = executable('eftest', 'eftest.cpp', link_with : flib, build_by_default : false)
 test('eftest', exe)
 
 # Same as above, but using build_target()
 flib2 = build_target('ef2', 'ef.cpp', target_type: 'library')
-exe2 = executable('eftest2', 'eftest.cpp', link_with : flib2)
+exe2 = executable('eftest2', 'eftest.cpp', link_with : flib2, build_by_default : false)
 test('eftest2', exe2)

test cases/common/95 benchmark/meson.build

@@ -1,4 +1,4 @@
 project('benchmark', 'c')
 
-delayer = executable('delayer', 'delayer.c', c_args : '-D_GNU_SOURCE')
+delayer = executable('delayer', 'delayer.c', c_args : '-D_GNU_SOURCE', build_by_default : false)
 benchmark('delayer', delayer)