Misc updates after trying to run clang-tidy

.gitignore

@@ -3,3 +3,5 @@ __pycache__/
 venv/
 dist/
 *.egg-info/
+*.ini
+*.xml

README.md

@@ -45,8 +45,8 @@ After a process of generating a compile_commands.json, you can run processcdb wi
 
     processcdb --tool clang-tidy
 
-This will try to locate the json file from current working directory and runs the tool, in this case
-clang-tidy, against all files that are compiled and not blacklisted in processcdb comfig file or in
+This will try to locate the json file from current working directory and runs the tool, in this case the 
+clang-tidy tool as defined in the config file, against all files that are compiled and not blacklisted in processcdb config file or in
 tools own configuration file and generates the output to standard output. If you need to run the tool when you
 don't have access to change the current working directory, you can pass `--cdb` and absolute location:
 
@@ -77,7 +77,7 @@ capture the standard output or provide `--config` parameter.
 ## Configuration file
 
 Each tool has a separate section and each section can be configured either in the tool specific section or
-in default. Minimal. single tool configuratio would look something like this:
+in default. The minimal single tool configuration would look something like this:
 
     [clang-tidy]
     binary=C:\llvm-11.0.0\bin\clang-tidy.exe

run_processcdb.bat

@@ -0,0 +1,6 @@
+@echo off
+pushd
+
+D:\fone\branch\external\ego\dev\python3\bin\python.exe src/processcdb --tool clang-tidy --xml --output results.xml --cdb D:\fone\branch\external\ego\dev\clang_tools\compile_commands.json
+
+popd

src/processcdb/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from ._version import get_versions
+from _version import get_versions
 
 __version__ = get_versions()["version"]
 del get_versions

src/processcdb/__main__.py

@@ -3,11 +3,10 @@
 import sys
 import json
 import traceback
-from .cdb_tools import TOOLS
-from .misc import remove_untouched_files, remove_dupes, argument_parser
+from cdb_tools import TOOLS
+from misc import remove_dupes, argument_parser
 from configparser import ConfigParser
-from .logger import LOGGER as log, LOG_LEVELS
-
+from logger import LOGGER as log, LOG_LEVELS
 
 def main():
     args = argument_parser(TOOLS).parse_args()
@@ -39,8 +38,8 @@ def main():
     if args.cdb.is_file():
         cdb = json.loads(args.cdb.read_text())
         if cdb:
-            if args.commit_a is not None:
-                cdb = remove_untouched_files(cdb, (args.commit_a, args.commit_b))
+            #if args.commit_a is not None:
+                #cdb = filterByChangelist(cdb, (args.commit_a, args.commit_b))
 
             if not args.allow_dupes:
                 cdb = remove_dupes(cdb)

src/processcdb/cdb_tools.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 
-from .toolbase import Tool  # noqa: F401
-from .clangtidy import ClangTidy  # noqa: F401
-from .clang import Clang  # noqa: F401
-from .lizard import Lizard  # noqa: F401
-from .cppcheck import CppCheck  # noqa: F401
+from toolbase import Tool  # noqa: F401
+from clangtidy import ClangTidy  # noqa: F401
+from clang import Clang  # noqa: F401
+from lizard import Lizard  # noqa: F401
+from cppcheck import CppCheck  # noqa: F401
 
 TOOLS = {
     "clang-tidy": ClangTidy,

src/processcdb/clang.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
-from .misc import is_any_windows
-from .toolbase import Tool
+from misc import is_any_windows
+from toolbase import Tool
 
 
 class Clang(Tool):
@@ -16,8 +16,10 @@ def execute(self, cdb, args):
         if args.output is not None:
             arguments.extend([f"--output {args.output}"])
 
+        absoluteBinaryPath = Path(self.binary).resolve()
+
         if is_any_windows():
-            arguments.extend([f"--use-analyzer {self.binary}"])
+            arguments.extend([f"--use-analyzer {absoluteBinaryPath}"])
 
-        final_command = f"{self.binary} {' '.join(arguments)}"
+        final_command = f"{absoluteBinaryPath} {' '.join(arguments)}"
         return self.run(final_command)

src/processcdb/clangtidy.py

@@ -11,9 +11,9 @@
 from functools import partial
 import shutil
 import signal
-from .logger import LOGGER as log
-from .toolbase import Tool
-from .tidy_converter import OutputParser
+from logger import LOGGER as log
+from toolbase import Tool
+from tidy_converter import OutputParser
 
 list_of_futures = []
 
@@ -108,9 +108,8 @@ def _generate_cmd_queue(self, cdb, args):
         for compilation_unit in cdb:
             arguments = []
 
-            arguments.extend(self.config.getlist("default_args"))
             directory = Path(compilation_unit["directory"]).absolute()
-            full_command = compilation_unit["command"].split(" ")
+            full_command = compilation_unit["arguments"]
             absolute_filename = directory / compilation_unit["file"]
             compiler = Path(full_command[0]).name.lower()
 
@@ -120,14 +119,16 @@ def _generate_cmd_queue(self, cdb, args):
             arguments = self.convert_includes(arguments)
             arguments.extend(self.includes_as_cli_flags(self.default_includes()))
 
-            extra = "--quiet"
-            if compiler == "cl.exe":
+            default_args = self.config.getlist("default_args")
+            extra = f"--quiet {' '.join(default_args)}"
+            if compiler.endswith("cl.exe"):
                 arguments = list(map(self.convert_arguments, arguments))
                 extra = f"{extra} --extra-arg-before=--driver-mode=cl"
 
             if absolute_filename.is_file():
                 if self.should_scan(absolute_filename, args.file):
-                    tmp_cmd = f"cd {directory} && {self.binary} {extra} {absolute_filename} -- {' '.join(arguments)}"
+                    absoluteBinaryPath = Path(self.binary).resolve()
+                    tmp_cmd = f"cd {directory} && {absoluteBinaryPath} {extra} {absolute_filename} -- {' '.join(arguments)}"
                     command_queue.append(tmp_cmd)
                 else:
                     log.debug(f"File {absolute_filename} is not scanned")

src/processcdb/cppcheck.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from .toolbase import Tool
+from toolbase import Tool
 import os
 import tempfile
 import configparser
@@ -80,7 +80,8 @@ def execute(self, cdb, args):
                     arguments.extend(["--xml", "--xml-version=2"])
 
                 if os.path.isfile(temp_name_includes) and os.path.isfile(temp_name_sources):
-                    tmp_cmd = f"{self.binary} {' '.join(arguments)}"
+                    absoluteBinaryPath = Path(self.binary).resolve()
+                    tmp_cmd = f"{absoluteBinaryPath} {' '.join(arguments)}"
                     if args.output is not None:
                         final_command = f"{tmp_cmd} 2> {args.output}"
                     else:

src/processcdb/lizard.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from .toolbase import Tool
+from toolbase import Tool
 import tempfile
 import os
 

src/processcdb/misc.py

@@ -3,18 +3,10 @@
 import platform
 import subprocess
 from pathlib import Path
-from whatthepatch import parse_patch
-from git import Repo
-from appdirs import AppDirs
 from collections import ChainMap
 import argparse
-from ._version import get_versions
-from .logger import LOGGER as log, LOG_LEVELS  # noqa: F401
-
-__author__ = "Jani Mikkonen"
-__email__ = "[email protected]"
-__version__ = get_versions()["version"]
-
+from _version import get_versions
+from logger import LOGGER as log, LOG_LEVELS  # noqa: F401
 
 def is_windows():
     return "WINDOWS" in platform.system().upper()
@@ -49,37 +41,6 @@ def capture_output(args, captureErr=True, capture_exceptions=True):
     return buff.split("\n")
 
 
-def remove_untouched_files(cdb, commits):
-    def modified_lines(data):
-        return data[0] is None and data[1] is not None
-
-    def line_numbers(data):
-        return data[1]
-
-    patch = None
-    repo = Repo(".", search_parent_directories=True)
-    commits = list(filter(None, commits))
-    if len(commits) == 2:
-        patch = parse_patch(repo.git.diff(commits[0], commits[1]))
-    else:
-        patch = parse_patch(repo.git.diff(f"{commits[0]}^"))
-    new_cdb = []
-    base_dir = Path(repo.git_dir).parent
-    lookup = {}
-    for i in cdb:
-        current = Path(i["directory"]) / i["file"]
-        lookup[current] = i
-
-    for diff in patch:
-        fullname = base_dir / diff.header.new_path
-        fullname = (base_dir / diff.header.new_path).absolute()
-        if fullname in lookup:
-            cdb_entry = lookup[fullname]
-            cdb_entry["changes_rows"] = map(line_numbers, filter(modified_lines, diff.changes))
-            new_cdb.append(cdb_entry)
-    return new_cdb
-
-
 def remove_dupes(cdb):
     seen = []
     new_cdb = []
@@ -93,8 +54,7 @@ def remove_dupes(cdb):
 
 def argument_parser(tools):
     # TODO: Offload tool specific argument to the tool class itself if possible.
-    app_dirs = AppDirs("processcdb", __author__)
-    default_config_file = Path(app_dirs.user_config_dir) / "processcdb.ini"
+    default_config_file = "processcdb.ini"
     parser = argparse.ArgumentParser(description="Static analysis wrapper", epilog=f"Available tools: \n{','.join(tools.keys())}")
     parser.add_argument(
         "--cdb",
@@ -204,6 +164,9 @@ def to_list(value):
 
 
 def to_dict(value):
+    if value == None or value == '':
+        return None
+
     def format(val):
         k,v = val.split("=", 2)
         return {k: v.split(",")}

src/processcdb/toolbase.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 
-from .misc import is_any_windows, capture_output, to_dict, to_list
+from misc import is_any_windows, capture_output, to_dict, to_list
 import multiprocessing
 import subprocess
 import configparser
 import os
-from .logger import LOGGER as log
+from logger import LOGGER as log
 import shutil
 import shlex
 from fnmatch import fnmatch
@@ -129,24 +129,36 @@ def run(self, command_line):
         return subprocess.call(command_line, shell=True)
 
     def include_string(self, include_path, path_matchers):
-        if any([fnmatch(include_path, pattern) for pattern in path_matchers]):
+        if len(path_matchers[0]) > 0 and any([fnmatch(include_path, pattern) for pattern in path_matchers]):
             return f'-isystem "{include_path}"'
         else:
-            return f"-I{include_path}"
+            return f'-I"{include_path}"'
 
     def includes_as_cli_flags(self, includes):
         path_matchers = self.config.getlist("includes_as_system")
         return list(map(lambda i: self.include_string(i, path_matchers), includes))
 
     def convert_includes(self, arguments):
-        def convert(arg, path_matchers):
-            if arg and arg.startswith("-I"):
-                return self.include_string(arg[2:], path_matchers)
-            return arg
-
+        def convert(arg, foundInclude, path_matchers):
+            if foundInclude:
+                foundInclude = False
+                return self.include_string(arg, path_matchers), foundInclude
+            elif arg and arg.startswith("-I"):
+                foundInclude = True
+                return None, foundInclude
+            return arg, foundInclude
+
+        foundInclude = False
         path_matchers = self.config.getlist("includes_as_system")
-        return list(map(lambda arg: convert(arg, path_matchers), arguments))
-
+
+        result = []
+        for arg in arguments:
+            converted = convert(arg, foundInclude, path_matchers)
+            if converted[0] != None:
+                result.append(converted[0])
+            foundInclude = converted[1]
+
+        return result
 
     def filter_arguments(self, args):
         def allowed_argument(arg):
@@ -157,8 +169,9 @@ def allowed_argument(arg):
     def add_additions(self, args):
         new_args = args.copy()
         additions = self.config.getdict("arg_additions")
-        for arg in args:
-            key = arg[1:]
-            if key in additions:
-                new_args.extend(additions[key])
+        if additions != None:
+            for arg in args:
+                key = arg[1:]
+                if key in additions:
+                    new_args.extend(additions[key])
         return new_args