add --coverage (#212)

add helper config for code coverage report

README.md

@@ -29,7 +29,7 @@ Usage: ```builder.pyz [build|inspect|<action-name>] [spec] [OPTIONS]```
 * ```--build-dir DIR``` - Make a new directory to do all the build work in, instead of using the current directory
 * ```--dump-config``` - Dumps the resultant config after merging all available options. Useful for debugging your project configuration.
 * ```--cmake-extra``` - Extra cmake config arg applied to all projects. e.g ```--cmake-extra=-DBUILD_SHARED_LIBS=ON```. May be specified multiple times.
-
+* ```--coverage``` - Generate the test coverage report and upload it to codecov. Only supported when using cmake and gcc as compiler, error out on other cases.
 
 ### Supported Targets:
 * linux: x86|i686, x64|x86_64, armv6, armv7, arm64|armv8|aarch64|arm64v8

builder/actions/cmake.py

@@ -1,7 +1,6 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-import argparse
 import os
 import re
 import shutil
@@ -101,7 +100,7 @@ def _project_dirs(env, project):
     return source_dir, build_dir, install_dir
 
 
-def _build_project(env, project, cmake_extra, build_tests=False, args_transformer=None):
+def _build_project(env, project, cmake_extra, build_tests=False, args_transformer=None, coverage=False):
     sh = env.shell
     config = project.get_config(env.spec)
     toolchain = env.toolchain
@@ -156,6 +155,19 @@ def _build_project(env, project, cmake_extra, build_tests=False, args_transforme
     # Using a UniqueList seems to solve the problem well enough for now.
     cmake_args += project.cmake_args(env)
     cmake_args += cmake_extra
+    if coverage:
+        if c_path and "gcc" in c_path:
+            # Tell cmake to add coverage related configuration. And make sure GCC is used to compile the project.
+            # CMAKE_C_FLAGS for GCC to enable code coverage information.
+            # COVERAGE_EXTRA_FLAGS="*" is configuration for gcov
+            #   --preserve-paths: to include path information in the report file name
+            #   --source-prefix `pwd`: to exculde the `pwd` from the file name
+            cmake_args += [
+                "-DCMAKE_C_FLAGS=-fprofile-arcs -ftest-coverage",
+                "-DCOVERAGE_EXTRA_FLAGS=--preserve-paths --source-prefix `pwd`"
+            ]
+        else:
+            raise Exception('--coverage only support GCC as compiler. Current compiler is: {}'.format(c_path))
 
     # Allow caller to programmatically tweak the cmake_args,
     # as a last resort in case data merging wasn't working out
@@ -195,16 +207,12 @@ def run(self, env):
         toolchain = env.toolchain
         sh = env.shell
 
-        parser = argparse.ArgumentParser()
-        parser.add_argument('--cmake-extra', action='append', default=[])
-        args = parser.parse_known_args(env.args.args)[0]
-
         for d in (env.build_dir, env.deps_dir, env.install_dir):
             sh.mkdir(d)
 
         # BUILD
         build_tests = self.project.needs_tests(env)
-        _build_project(env, self.project, args.cmake_extra, build_tests, self.args_transformer)
+        _build_project(env, self.project, env.args.cmake_extra, build_tests, self.args_transformer, env.args.coverage)
 
     def __str__(self):
         return 'cmake build {} @ {}'.format(self.project.name, self.project.path)
@@ -234,6 +242,9 @@ def run(self, env):
         ctest = toolchain.ctest_binary()
         sh.exec(*toolchain.shell_env, ctest,
                 "--output-on-failure", working_dir=project_build_dir, check=True)
+        # Try to generate the coverage report. Will be ignored by ctest if no coverage data available.
+        sh.exec(*toolchain.shell_env, ctest,
+                "-T", "coverage", working_dir=project_build_dir, check=True)
 
     def __str__(self):
         return 'ctest {} @ {}'.format(self.project.name, self.project.path)

builder/core/shell.py

@@ -150,7 +150,7 @@ def exec(self, *command, **kwargs):
             result = util.run_command(*command, **kwargs, dryrun=self.dryrun)
         return result
 
-    def get_secret(self, secret_id):
+    def get_secret(self, secret_id, key=None):
         """get string from secretsmanager"""
 
         # NOTE: using AWS CLI instead of boto3 because we know CLI is already
@@ -165,4 +165,8 @@ def get_secret(self, secret_id):
         print('>', subprocess.list2cmdline(cmd))
         result = self.exec(*cmd, check=True, quiet=True)
         secret_value = json.loads(result.output)
-        return secret_value['SecretString']
+        if key is not None:
+            screct_pairs = json.loads(secret_value['SecretString'])
+            return screct_pairs[key]
+        else:
+            return secret_value['SecretString']

builder/main.py

@@ -143,6 +143,9 @@ def parse_args():
                         default='{}-{}'.format(current_os(), current_arch()),
                         choices=data.PLATFORMS.keys())
     parser.add_argument('--variant', type=str, help="Build variant to use instead of default")
+    parser.add_argument('--cmake-extra', action='append', default=[])
+    parser.add_argument('--coverage', action='store_true',
+                        help="Enable test coverage report and upload it the codecov. Only supported when using cmake with gcc as compiler, error out on other cases.")
 
     # hand parse command and spec from within the args given
     command = None
@@ -222,6 +225,19 @@ def parse_args():
     return args, spec
 
 
+def upload_test_coverage(env):
+    try:
+        token = env.shell.get_secret("codecov-token", env.project.name)
+    except:
+        print(f"No token found for {env.project.name}, check https://app.codecov.io/github/awslabs/{env.project.name}/settings for token and add it to codecov-token in secret-manager.", file=sys.stderr)
+        exit()
+    # only works for linux for now
+    env.shell.exec('curl', '-Os', 'https://uploader.codecov.io/latest/linux/codecov', check=True)
+    env.shell.exec('chmod', '+x', 'codecov', check=True)
+    # based on the way generated report, we only upload the report started with `source/`
+    env.shell.exec('./codecov', '-t', token, '-f', 'source#*', check=True)
+
+
 def main():
     args, spec = parse_args()
 
@@ -278,6 +294,9 @@ def main():
     else:
         run_action(args.command, env)
 
+    if args.coverage:
+        upload_test_coverage(env)
+
 
 if __name__ == '__main__':
     main()