backends: Make ninja not build test executables by default

mesonbuild/backend/backends.py

@@ -854,15 +854,17 @@ def replace_outputs(self, args, private_dir, output_list):
         return newargs
 
     def get_build_by_default_targets(self):
+        """Get all build and custom targets that must be built by default."""
         result = OrderedDict()
-        # Get all build and custom targets that must be built by default
         for name, t in self.build.get_targets().items():
             if t.build_by_default:
                 result[name] = t
-        # Get all targets used as test executables and arguments. These must
-        # also be built by default. XXX: Sometime in the future these should be
-        # built only before running tests.
-        for t in self.build.get_tests():
+        return result
+
+    @staticmethod
+    def _get_targets(values):
+        result = OrderedDict()
+        for t in values:
             exe = t.exe
             if hasattr(exe, 'held_object'):
                 exe = exe.held_object
@@ -879,6 +881,14 @@ def get_build_by_default_targets(self):
                 result[dep.get_id()] = dep
         return result
 
+    def get_test_targets(self):
+        """Get all targets used as test executables and arguments."""
+        return self._get_targets(self.build.get_tests())
+
+    def get_benchmark_targets(self):
+        """Get all targets used as test executables and arguments."""
+        return self._get_targets(self.build.get_benchmarks())
+
     @lru_cache(maxsize=None)
     def get_custom_target_provided_by_generated_source(self, generated_source):
         libs = []

mesonbuild/backend/ninjabackend.py

@@ -823,7 +823,15 @@ def generate_tests(self):
             cmd += ['--no-stdsplit']
         if self.environment.coredata.get_builtin_option('errorlogs'):
             cmd += ['--print-errorlogs']
-        elem = NinjaBuildElement(self.all_outputs, 'meson-test', 'CUSTOM_COMMAND', ['all', 'PHONY'])
+        tests = self.get_test_targets()
+
+        tests = [os.path.join(self.get_target_dir(t), t.get_outputs()[0])
+                 for t in self.get_test_targets().values()]
+        elem = NinjaBuildElement(self.all_outputs, 'meson-build-tests', 'CUSTOM_COMMAND', 'PHONY')
+        elem.add_dep(tests)
+        self.add_build(elem)
+
+        elem = NinjaBuildElement(self.all_outputs, 'meson-test', 'CUSTOM_COMMAND', ['meson-build-tests', 'PHONY'])
         elem.add_item('COMMAND', cmd)
         elem.add_item('DESC', 'Running all tests.')
         elem.add_item('pool', 'console')
@@ -835,7 +843,13 @@ def generate_tests(self):
         cmd = self.environment.get_build_command(True) + [
             'test', '--benchmark', '--logbase',
             'benchmarklog', '--num-processes=1', '--no-rebuild']
-        elem = NinjaBuildElement(self.all_outputs, 'meson-benchmark', 'CUSTOM_COMMAND', ['all', 'PHONY'])
+        elem = NinjaBuildElement(self.all_outputs, 'meson-build-benchmarks', 'CUSTOM_COMMAND', 'PHONY')
+        benches = [os.path.join(self.get_target_dir(t), t.get_outputs()[0])
+                   for t in self.get_benchmark_targets().values()]
+        elem.add_dep(benches)
+        self.add_build(elem)
+
+        elem = NinjaBuildElement(self.all_outputs, 'meson-benchmark', 'CUSTOM_COMMAND', ['meson-build-benchmarks', 'PHONY'])
         elem.add_item('COMMAND', cmd)
         elem.add_item('DESC', 'Running benchmark suite.')
         elem.add_item('pool', 'console')

mesonbuild/mtest.py

@@ -486,6 +486,10 @@ def run(self) -> TestRun:
     def _run_cmd(self, cmd: typing.List[str]) -> TestRun:
         starttime = time.time()
 
+        # This happens when --no-rebuild is passed and test targets have not been built
+        if not os.path.exists(cmd[0]):
+            return TestRun(self.test, self.test_env, TestResult.SKIP, 0, 0, '', 'Test Executable not built.', cmd)
+
         if len(self.test.extra_paths) > 0:
             self.env['PATH'] = os.pathsep.join(self.test.extra_paths + ['']) + self.env['PATH']
             winecmd = []
@@ -977,7 +981,7 @@ def list_tests(th: TestHarness) -> bool:
         print(th.get_pretty_suite(t))
     return not tests
 
-def rebuild_all(wd: str) -> bool:
+def rebuild_all(wd: str, benchmark: bool) -> bool:
     if not (Path(wd) / 'build.ninja').is_file():
         print('Only ninja backend is supported to rebuild tests before running them.')
         return True
@@ -987,7 +991,12 @@ def rebuild_all(wd: str) -> bool:
         print("Can't find ninja, can't rebuild test.")
         return False
 
-    ret = subprocess.run([ninja, '-C', wd]).returncode
+    if benchmark:
+        target = 'meson-build-benchmarks'
+    else:
+        target = 'meson-build-tests'
+
+    ret = subprocess.run([ninja, '-C', wd, target]).returncode
     if ret != 0:
         print('Could not rebuild {}'.format(wd))
         return False
@@ -1021,7 +1030,7 @@ def run(options: argparse.Namespace) -> int:
     options.wd = os.path.abspath(options.wd)
 
     if not options.list and not options.no_rebuild:
-        if not rebuild_all(options.wd):
+        if not rebuild_all(options.wd, options.benchmark):
             # We return 125 here in case the build failed.
             # The reason is that exit code 125 tells `git bisect run` that the current commit should be skipped.
             # Thus users can directly use `meson test` to bisect without needing to handle the does-not-build case separately in a wrapper script.

run_tests.py

@@ -211,6 +211,8 @@ def get_backend_commands(backend, debug=False):
         test_cmd = cmd + ['test', 'benchmark']
         install_cmd = cmd + ['install']
         uninstall_cmd = cmd + ['uninstall']
+        # Because we use no-rebuild we need to build the test and benchmark targets explicitly
+        cmd.extend(['all', 'meson-build-tests', 'meson-build-benchmarks'])
     else:
         raise AssertionError('Unknown backend: {!r}'.format(backend))
     return cmd, clean_cmd, test_cmd, install_cmd, uninstall_cmd

run_unittests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python3ass
 # Copyright 2016-2017 The Meson development team
 
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -3841,6 +3841,20 @@ def test_alias_target(self):
         self.assertPathExists(os.path.join(self.builddir, 'prog' + exe_suffix))
         self.assertPathExists(os.path.join(self.builddir, 'hello.txt'))
 
+    def test_tests_not_default(self):
+        testdir = os.path.join(self.common_test_dir, '1 trivial')
+        self.init(testdir)
+        self.build()
+        self.assertBuildIsNoop()
+        self.run_tests()
+
+    def test_benchmark_not_default(self):
+        testdir = os.path.join(self.common_test_dir, '95 benchmark')
+        self.init(testdir)
+        self.build()
+        self.assertBuildIsNoop()
+
+
 class FailureTests(BasePlatformTests):
     '''
     Tests that test failure conditions. Build files here should be dynamically

test cases/common/1 trivial/meson.build

@@ -19,7 +19,7 @@ if meson.is_cross_build()
   test('native exe in cross build', native_exe)
 endif
 
-exe = executable('trivialprog', sources : sources)
+exe = executable('trivialprog', sources : sources, build_by_default : false)
 
 test('runtest', exe) # This is a comment
 

test cases/common/95 benchmark/meson.build

@@ -1,4 +1,4 @@
 project('benchmark', 'c')
 
-delayer = executable('delayer', 'delayer.c', c_args : '-D_GNU_SOURCE')
+delayer = executable('delayer', 'delayer.c', c_args : '-D_GNU_SOURCE', build_by_default : false)
 benchmark('delayer', delayer)