diff --git a/codecov_cli/commands/upload.py b/codecov_cli/commands/upload.py index cf6a4b1b..2e5fc2b6 100644 --- a/codecov_cli/commands/upload.py +++ b/codecov_cli/commands/upload.py @@ -18,6 +18,14 @@ def _turn_env_vars_into_dict(ctx, params, value): _global_upload_options = [ + click.option( + "--type", + "--upload-file-type", + "upload_file_type", + help="The type of file to search for an upload. Uploads coverage files by default", + type=click.Choice(["coverage", "testing"]), + default="coverage", + ), click.option( "--report-code", help="The code of the report. If unsure, leave default", @@ -34,7 +42,7 @@ def _turn_env_vars_into_dict(ctx, params, value): "-s", "--dir", "--coverage-files-search-root-folder", - "coverage_files_search_root_folder", + "files_search_root_folder", help="Folder where to search for coverage files", type=click.Path(path_type=pathlib.Path), default=pathlib.Path.cwd, @@ -43,7 +51,7 @@ def _turn_env_vars_into_dict(ctx, params, value): click.option( "--exclude", "--coverage-files-search-exclude-folder", - "coverage_files_search_exclude_folders", + "files_search_exclude_folders", help="Folders to exclude from search", type=click.Path(path_type=pathlib.Path), multiple=True, @@ -53,7 +61,7 @@ def _turn_env_vars_into_dict(ctx, params, value): "-f", "--file", "--coverage-files-search-direct-file", - "coverage_files_search_explicitly_listed_files", + "files_search_explicitly_listed_files", help="Explicit files to upload. These will be added to the coverage files found for upload. If you wish to only upload the specified files, please consider using --disable-search to disable uploading other files.", type=click.Path(path_type=pathlib.Path), multiple=True, @@ -172,6 +180,7 @@ def global_upload_options(func): def do_upload( ctx: click.Context, commit_sha: str, + upload_file_type: str, report_code: str, build_code: typing.Optional[str], build_url: typing.Optional[str], @@ -180,9 +189,9 @@ def do_upload( flags: typing.List[str], name: typing.Optional[str], network_root_folder: pathlib.Path, - coverage_files_search_root_folder: pathlib.Path, - coverage_files_search_exclude_folders: typing.List[pathlib.Path], - coverage_files_search_explicitly_listed_files: typing.List[pathlib.Path], + files_search_root_folder: pathlib.Path, + files_search_exclude_folders: typing.List[pathlib.Path], + files_search_explicitly_listed_files: typing.List[pathlib.Path], disable_search: bool, disable_file_fixes: bool, token: typing.Optional[uuid.UUID], @@ -205,6 +214,7 @@ def do_upload( "Starting upload processing", extra=dict( extra_log_attributes=dict( + upload_file_type=upload_file_type, commit_sha=commit_sha, report_code=report_code, build_code=build_code, @@ -214,9 +224,9 @@ def do_upload( flags=flags, name=name, network_root_folder=network_root_folder, - coverage_files_search_root_folder=coverage_files_search_root_folder, - coverage_files_search_exclude_folders=coverage_files_search_exclude_folders, - coverage_files_search_explicitly_listed_files=coverage_files_search_explicitly_listed_files, + files_search_root_folder=files_search_root_folder, + files_search_exclude_folders=files_search_exclude_folders, + files_search_explicitly_listed_files=files_search_explicitly_listed_files, plugin_names=plugin_names, token=token, branch=branch, @@ -235,6 +245,7 @@ def do_upload( versioning_system, ci_adapter, commit_sha=commit_sha, + upload_file_type=upload_file_type, report_code=report_code, build_code=build_code, build_url=build_url, @@ -243,13 +254,9 @@ def do_upload( flags=flags, name=name, network_root_folder=network_root_folder, - coverage_files_search_root_folder=coverage_files_search_root_folder, - coverage_files_search_exclude_folders=list( - coverage_files_search_exclude_folders - ), - coverage_files_search_explicitly_listed_files=list( - coverage_files_search_explicitly_listed_files - ), + files_search_root_folder=files_search_root_folder, + files_search_exclude_folders=list(files_search_exclude_folders), + files_search_explicitly_listed_files=list(files_search_explicitly_listed_files), plugin_names=plugin_names, token=token, branch=branch, diff --git a/codecov_cli/commands/upload_process.py b/codecov_cli/commands/upload_process.py index 67fd14da..e4d43147 100644 --- a/codecov_cli/commands/upload_process.py +++ b/codecov_cli/commands/upload_process.py @@ -33,9 +33,9 @@ def upload_process( flags: typing.List[str], name: typing.Optional[str], network_root_folder: pathlib.Path, - coverage_files_search_root_folder: pathlib.Path, - coverage_files_search_exclude_folders: typing.List[pathlib.Path], - coverage_files_search_explicitly_listed_files: typing.List[pathlib.Path], + files_search_root_folder: pathlib.Path, + files_search_exclude_folders: typing.List[pathlib.Path], + files_search_explicitly_listed_files: typing.List[pathlib.Path], disable_search: bool, disable_file_fixes: bool, token: typing.Optional[uuid.UUID], @@ -63,9 +63,9 @@ def upload_process( flags=flags, name=name, network_root_folder=network_root_folder, - coverage_files_search_root_folder=coverage_files_search_root_folder, - coverage_files_search_exclude_folders=coverage_files_search_exclude_folders, - coverage_files_search_explicitly_listed_files=coverage_files_search_explicitly_listed_files, + files_search_root_folder=files_search_root_folder, + files_search_exclude_folders=files_search_exclude_folders, + files_search_explicitly_listed_files=files_search_explicitly_listed_files, plugin_names=plugin_names, token=token, branch=branch, @@ -111,9 +111,9 @@ def upload_process( flags=flags, name=name, network_root_folder=network_root_folder, - coverage_files_search_root_folder=coverage_files_search_root_folder, - coverage_files_search_exclude_folders=coverage_files_search_exclude_folders, - coverage_files_search_explicitly_listed_files=coverage_files_search_explicitly_listed_files, + files_search_root_folder=files_search_root_folder, + files_search_exclude_folders=files_search_exclude_folders, + files_search_explicitly_listed_files=files_search_explicitly_listed_files, disable_search=disable_search, token=token, plugin_names=plugin_names, diff --git a/codecov_cli/helpers/folder_searcher.py b/codecov_cli/helpers/folder_searcher.py index 6abbd97b..19e1be46 100644 --- a/codecov_cli/helpers/folder_searcher.py +++ b/codecov_cli/helpers/folder_searcher.py @@ -58,7 +58,7 @@ def search_files( this_is_excluded = functools.partial( _is_excluded, filename_exclude_regex, multipart_exclude_regex ) - for (dirpath, dirnames, filenames) in os.walk(folder_to_search): + for dirpath, dirnames, filenames in os.walk(folder_to_search): dirs_to_remove = set(d for d in dirnames if d in folders_to_ignore) if multipart_exclude_regex is not None: diff --git a/codecov_cli/services/upload/__init__.py b/codecov_cli/services/upload/__init__.py index 959d6ed9..2b575d73 100644 --- a/codecov_cli/services/upload/__init__.py +++ b/codecov_cli/services/upload/__init__.py @@ -10,10 +10,23 @@ from codecov_cli.helpers.request import log_warnings_and_errors_if_any from codecov_cli.helpers.versioning_systems import VersioningSystemInterface from codecov_cli.plugins import select_preparation_plugins -from codecov_cli.services.upload.coverage_file_finder import select_coverage_file_finder +from codecov_cli.services.upload.finders.coverage_file_finder import ( + select_coverage_file_finder, +) from codecov_cli.services.upload.legacy_upload_sender import LegacyUploadSender -from codecov_cli.services.upload.network_finder import select_network_finder -from codecov_cli.services.upload.upload_collector import UploadCollector +from codecov_cli.services.upload.finders.network_finder import select_network_finder +from codecov_cli.services.upload.collectors.legacy_upload_collector import ( + LegacyUploadCollector, +) +from codecov_cli.services.upload.finders.testing_result_file_finder import ( + select_testing_result_file_finder, +) +from codecov_cli.services.upload.collectors.testing_result_upload_collector import ( + TestingResultUploadCollector, +) +from codecov_cli.services.upload.collectors.coverage_upload_collector import ( + CoverageUploadCollector, +) from codecov_cli.services.upload.upload_sender import UploadSender from codecov_cli.services.upload_completion import upload_completion_logic from codecov_cli.types import RequestResult @@ -26,6 +39,7 @@ def do_upload_logic( versioning_system: VersioningSystemInterface, ci_adapter: CIAdapterBase, *, + upload_file_type: str, commit_sha: str, report_code: str, build_code: typing.Optional[str], @@ -35,9 +49,9 @@ def do_upload_logic( flags: typing.List[str], name: typing.Optional[str], network_root_folder: Path, - coverage_files_search_root_folder: Path, - coverage_files_search_exclude_folders: typing.List[Path], - coverage_files_search_explicitly_listed_files: typing.List[Path], + files_search_root_folder: Path, + files_search_exclude_folders: typing.List[Path], + files_search_explicitly_listed_files: typing.List[Path], plugin_names: typing.List[str], token: uuid.UUID, branch: typing.Optional[str], @@ -52,23 +66,32 @@ def do_upload_logic( handle_no_reports_found: bool = False, disable_file_fixes: bool = False, ): - preparation_plugins = select_preparation_plugins(cli_config, plugin_names) - coverage_file_selector = select_coverage_file_finder( - coverage_files_search_root_folder, - coverage_files_search_exclude_folders, - coverage_files_search_explicitly_listed_files, - disable_search, - ) - network_finder = select_network_finder(versioning_system) - collector = UploadCollector( - preparation_plugins, network_finder, coverage_file_selector, disable_file_fixes - ) try: - upload_data = collector.generate_upload_data() + if upload_file_type == "coverage": + upload_data = prepare_coverage_data( + cli_config, + plugin_names, + files_search_root_folder, + files_search_exclude_folders, + files_search_explicitly_listed_files, + disable_search, + versioning_system, + use_legacy_uploader, + disable_file_fixes, + env_vars, + ) + else: + upload_data = prepare_testing_result_data( + files_search_root_folder, + files_search_exclude_folders, + files_search_explicitly_listed_files, + disable_search, + ) except click.ClickException as exp: if handle_no_reports_found: logger.info( - "No coverage reports found. Triggering notificaions without uploading." + "No reports found. Triggering notificaions without uploading.", + extra={"upload_type": upload_file_type}, ) upload_completion_logic( commit_sha=commit_sha, @@ -82,15 +105,17 @@ def do_upload_logic( error=None, warnings=None, status_code=200, - text="No coverage reports found. Triggering notificaions without uploading.", + text=f"No {upload_file_type} reports found. Triggering notificaions without uploading.", ) else: raise exp + if use_legacy_uploader: sender = LegacyUploadSender() else: sender = UploadSender() logger.debug(f"Selected uploader to use: {type(sender)}") + ci_service = ( ci_adapter.get_fallback_value(FallbackFieldEnum.service) if ci_adapter is not None @@ -99,6 +124,7 @@ def do_upload_logic( if not dry_run: sending_result = sender.send_upload_data( + upload_file_type, upload_data, commit_sha, token, @@ -126,3 +152,62 @@ def do_upload_logic( ) log_warnings_and_errors_if_any(sending_result, "Upload", fail_on_error) return sending_result + + +def prepare_testing_result_data( + search_root_folder, + search_exclude_folders, + search_explicit, + disable_search, +): + testing_result_file_finder = select_testing_result_file_finder( + search_root_folder, + search_exclude_folders, + search_explicit, + disable_search, + ) + + collector = TestingResultUploadCollector(testing_result_file_finder) + + return collector.generate_payload_data() + + +def prepare_coverage_data( + cli_config, + plugin_names, + search_root_folder, + search_exclude_folders, + search_explicit, + disable_search, + versioning_system, + use_legacy, + disable_file_fixes, + env_vars, +): + coverage_file_selector = select_coverage_file_finder( + search_root_folder, + search_exclude_folders, + search_explicit, + disable_search, + ) + network_finder = select_network_finder(versioning_system) + + if use_legacy: + Collector = LegacyUploadCollector + else: + Collector = CoverageUploadCollector + + collector = Collector( + network_finder, + coverage_file_selector, + disable_file_fixes, + env_vars, + ) + + preparation_plugins = select_preparation_plugins(cli_config, plugin_names) + + for prep in preparation_plugins: + logger.debug(f"Running preparation plugin: {type(prep)}") + prep.run_preparation(collector) + + return collector.generate_upload_data() diff --git a/codecov_cli/services/upload/upload_collector.py b/codecov_cli/services/upload/collectors/coverage_upload_collector.py similarity index 52% rename from codecov_cli/services/upload/upload_collector.py rename to codecov_cli/services/upload/collectors/coverage_upload_collector.py index d30e036e..b562a081 100644 --- a/codecov_cli/services/upload/upload_collector.py +++ b/codecov_cli/services/upload/collectors/coverage_upload_collector.py @@ -1,19 +1,23 @@ +import base64 +import json import logging import pathlib import re import typing -import uuid +import zlib from collections import namedtuple from fnmatch import fnmatch +from typing import Any, Dict import click -from codecov_cli.services.upload.coverage_file_finder import CoverageFileFinder -from codecov_cli.services.upload.network_finder import NetworkFinder +from codecov_cli.services.upload.finders.network_finder import NetworkFinder +from codecov_cli.services.upload.finders.coverage_file_finder import CoverageFileFinder from codecov_cli.types import ( - PreparationPluginInterface, UploadCollectionResult, + UploadCollectionResultFile, UploadCollectionResultFileFixer, + PreparationPluginInterface, ) logger = logging.getLogger("codecovcli") @@ -23,21 +27,21 @@ ) -class UploadCollector(object): +class CoverageUploadCollector(object): def __init__( self, - preparation_plugins: typing.List[PreparationPluginInterface], network_finder: NetworkFinder, coverage_file_finder: CoverageFileFinder, disable_file_fixes: bool = False, + env_vars: typing.Dict[str, str] = None, ): - self.preparation_plugins = preparation_plugins self.network_finder = network_finder self.coverage_file_finder = coverage_file_finder self.disable_file_fixes = disable_file_fixes + self.env_vars = env_vars def _produce_file_fixes_for_network( - self, network: typing.List[str] + self, network: typing.List[UploadCollectionResultFile] ) -> typing.List[UploadCollectionResultFileFixer]: if not network or self.disable_file_fixes: return [] @@ -96,39 +100,41 @@ def _produce_file_fixes_for_network( result = [] for filename in network: for glob, fix_patterns in file_regex_patterns.items(): - if fnmatch(filename, glob): + if fnmatch(filename.get_filename().decode(), glob): result.append(self._get_file_fixes(filename, fix_patterns)) break return result def _get_file_fixes( - self, filename: str, fix_patterns_to_apply: fix_patterns_to_apply + self, + file: UploadCollectionResultFile, + fix_patterns_to_apply: fix_patterns_to_apply, ) -> UploadCollectionResultFileFixer: - path = pathlib.Path(filename) + path = pathlib.Path(file.get_filename().decode()) fixed_lines_without_reason = set() fixed_lines_with_reason = set() eof = None try: - with open(filename, "r") as f: - for lineno, line_content in enumerate(f): - if any( - pattern.match(line_content) - for pattern in fix_patterns_to_apply.with_reason - ): - fixed_lines_with_reason.add((lineno + 1, line_content)) - elif any( - pattern.match(line_content) - for pattern in fix_patterns_to_apply.without_reason - ): - fixed_lines_without_reason.add(lineno + 1) - - if fix_patterns_to_apply.eof: - eof = lineno + 1 + file_content = file.get_string().splitlines(keepends=True) + for lineno, line_content in enumerate(file_content): + if any( + pattern.match(line_content) + for pattern in fix_patterns_to_apply.with_reason + ): + fixed_lines_with_reason.add((lineno + 1, line_content)) + elif any( + pattern.match(line_content) + for pattern in fix_patterns_to_apply.without_reason + ): + fixed_lines_without_reason.add(lineno + 1) + + if fix_patterns_to_apply.eof: + eof = lineno + 1 except UnicodeDecodeError as err: logger.warning( - f"There was an issue decoding: {filename}, file fixes were not applied to this file.", + f"There was an issue decoding: {file.get_filename().decode()}, file fixes were not applied to this file.", extra=dict( encoding=err.encoding, reason=err.reason, @@ -139,13 +145,10 @@ def _get_file_fixes( path, fixed_lines_without_reason, fixed_lines_with_reason, eof ) - def generate_upload_data(self) -> UploadCollectionResult: - for prep in self.preparation_plugins: - logger.debug(f"Running preparation plugin: {type(prep)}") - prep.run_preparation(self) + def generate_upload_data(self) -> bytes: logger.debug("Collecting relevant files") network = self.network_finder.find_files() - coverage_files = self.coverage_file_finder.find_coverage_files() + coverage_files = self.coverage_file_finder.find_files() logger.info(f"Found {len(coverage_files)} coverage files to upload") if not coverage_files: raise click.ClickException( @@ -156,8 +159,76 @@ def generate_upload_data(self) -> UploadCollectionResult: ) for file in coverage_files: logger.info(f"> {file}") - return UploadCollectionResult( + collection_result = UploadCollectionResult( network=network, coverage_files=coverage_files, file_fixes=self._produce_file_fixes_for_network(network), ) + + return self._generate_payload(collection_result, self.env_vars) + + def _generate_payload( + self, upload_data: UploadCollectionResult, env_vars: typing.Dict[str, str] + ) -> bytes: + network_files = upload_data.network + payload = { + "report_fixes": { + "format": "legacy", + "value": self._get_file_fixers(upload_data), + }, + "network_files": [file.get_filename().decode() for file in network_files] + if network_files is not None + else [], + "coverage_files": self._get_coverage_files(upload_data), + "metadata": {}, + } + + json_data = json.dumps(payload) + return json_data.encode() + + def _get_file_fixers( + self, upload_data: UploadCollectionResult + ) -> Dict[str, Dict[str, Any]]: + """ + Returns file/path fixes in the following format: + + { + {path}: { + "eof": int(eof_line), + "lines": {set_of_lines}, + }, + } + """ + file_fixers = {} + for file_fixer in upload_data.file_fixes: + fixed_lines_with_reason = set( + [fixer[0] for fixer in file_fixer.fixed_lines_with_reason] + ) + total_fixed_lines = list( + file_fixer.fixed_lines_without_reason.union(fixed_lines_with_reason) + ) + file_fixers[str(file_fixer.path)] = { + "eof": file_fixer.eof, + "lines": total_fixed_lines, + } + + return file_fixers + + def _get_coverage_files(self, upload_data: UploadCollectionResult): + return [self._format_coverage_file(file) for file in upload_data.coverage_files] + + def _format_coverage_file(self, file: UploadCollectionResultFile): + format, formatted_content = self._get_format_info(file) + return { + "filename": file.get_filename().decode(), + "format": format, + "data": formatted_content, + "labels": "", + } + + def _get_format_info(self, file: UploadCollectionResultFile): + format = "base64+compressed" + formatted_content = ( + base64.b64encode(zlib.compress((file.get_content()))) + ).decode() + return format, formatted_content diff --git a/codecov_cli/services/upload/collectors/legacy_upload_collector.py b/codecov_cli/services/upload/collectors/legacy_upload_collector.py new file mode 100644 index 00000000..98852526 --- /dev/null +++ b/codecov_cli/services/upload/collectors/legacy_upload_collector.py @@ -0,0 +1,65 @@ +import logging +import typing +from collections import namedtuple +from fnmatch import fnmatch +from typing import Any, Dict + + +from codecov_cli.types import ( + UploadCollectionResult, + UploadCollectionResultFile, +) +from codecov_cli.services.upload.collectors.coverage_upload_collector import ( + CoverageUploadCollector, +) + +logger = logging.getLogger("codecovcli") + +fix_patterns_to_apply = namedtuple( + "fix_patterns_to_apply", ["without_reason", "with_reason", "eof"] +) + + +class LegacyUploadCollector(CoverageUploadCollector): + def _generate_payload( + self, upload_data: UploadCollectionResult, env_vars: typing.Dict[str, str] + ) -> bytes: + env_vars_section = self._generate_env_vars_section(env_vars) + network_section = self._generate_network_section(upload_data) + coverage_files_section = self._generate_coverage_files_section(upload_data) + + return b"".join([env_vars_section, network_section, coverage_files_section]) + + def _generate_env_vars_section(self, env_vars) -> bytes: + filtered_env_vars = { + key: value for key, value in env_vars.items() if value is not None + } + + if not filtered_env_vars: + return b"" + + env_vars_section = "".join( + f"{env_var}={value}\n" for env_var, value in filtered_env_vars.items() + ) + return env_vars_section.encode() + b"<<<<<< ENV\n" + + def _generate_network_section(self, upload_data: UploadCollectionResult) -> bytes: + network_files = upload_data.network + + if not network_files: + return b"" + + network_files_section = "".join(file + "\n" for file in network_files) + return network_files_section.encode() + b"<<<<<< network\n" + + def _generate_coverage_files_section(self, upload_data: UploadCollectionResult): + return b"".join( + self._format_coverage_file(file) for file in upload_data.coverage_files + ) + + def _format_coverage_file(self, file: UploadCollectionResultFile) -> bytes: + header = b"# path=" + file.get_filename() + b"\n" + file_content = file.get_content() + b"\n" + file_end = b"<<<<<< EOF\n" + + return header + file_content + file_end diff --git a/codecov_cli/services/upload/collectors/testing_result_upload_collector.py b/codecov_cli/services/upload/collectors/testing_result_upload_collector.py new file mode 100644 index 00000000..c72dcda2 --- /dev/null +++ b/codecov_cli/services/upload/collectors/testing_result_upload_collector.py @@ -0,0 +1,40 @@ +import base64 +import json +import zlib + +from codecov_cli.services.upload.finders.testing_result_file_finder import ( + TestingResultFileFinder, +) +from codecov_cli.types import UploadCollectionResultFile + + +class TestingResultUploadCollector: + def __init__(self, file_finder: TestingResultFileFinder) -> None: + self.file_finder = file_finder + + def generate_payload_data(self) -> bytes: + testing_result_files = self.file_finder.find_files() + + return json.dumps( + { + "testing_result_files": [ + self._format_file(file) for file in testing_result_files + ] + } + ).encode() + + def _format_file(self, file: UploadCollectionResultFile): + format_info, formatted_content = self._get_format_info(file) + return { + "filename": file.get_filename().decode(), + "format": format_info, + "data": formatted_content, + "labels": "", + } + + def _get_format_info(self, file: UploadCollectionResultFile): + format_info = "base64+compressed" + formatted_content = ( + base64.b64encode(zlib.compress((file.get_content()))) + ).decode() + return format_info, formatted_content diff --git a/codecov_cli/services/upload/finders/coverage_file_finder.py b/codecov_cli/services/upload/finders/coverage_file_finder.py new file mode 100644 index 00000000..97b26351 --- /dev/null +++ b/codecov_cli/services/upload/finders/coverage_file_finder.py @@ -0,0 +1,41 @@ +import logging +from pathlib import Path + +from codecov_cli.services.upload.finders.result_file_finder import ResultFileFinder + +logger = logging.getLogger("codecovcli") + +coverage_files_patterns = [ + "*.clover", + "*.codecov.*", + "*.gcov", + "*.lcov", + "*.lst", + "*coverage*.*", + "*Jacoco*.xml", + "clover.xml", + "cobertura.xml", + "codecov-result.json", + "codecov.*", + "cover.out", + "coverage-final.json", + "excoveralls.json", + "gcov.info", + "jacoco*.xml", + "lcov.dat", + "pylcov.dat", + "lcov.info", + "luacov.report.out", + "naxsi.info", + "nosetests.xml", + "report.xml", + "test_cov.xml", +] + + +class CoverageFileFinder(ResultFileFinder): + file_patterns = coverage_files_patterns + + +def select_coverage_file_finder(*args, **kwargs): + return CoverageFileFinder(*args, **kwargs) diff --git a/codecov_cli/services/upload/network_finder.py b/codecov_cli/services/upload/finders/network_finder.py similarity index 66% rename from codecov_cli/services/upload/network_finder.py rename to codecov_cli/services/upload/finders/network_finder.py index 3ccfb463..786054ef 100644 --- a/codecov_cli/services/upload/network_finder.py +++ b/codecov_cli/services/upload/finders/network_finder.py @@ -2,6 +2,7 @@ import typing from codecov_cli.helpers.versioning_systems import VersioningSystemInterface +from codecov_cli.types import UploadCollectionResultFile class NetworkFinder(object): @@ -13,8 +14,11 @@ def find_files( network_root: typing.Optional[pathlib.Path] = None, network_filter=None, network_adjuster=None, - ) -> typing.List[str]: - return self.versioning_system.list_relevant_files(network_root) + ) -> typing.List[UploadCollectionResultFile]: + return [ + UploadCollectionResultFile(pathlib.Path(file)) + for file in self.versioning_system.list_relevant_files(network_root) + ] def select_network_finder(versioning_system: VersioningSystemInterface): diff --git a/codecov_cli/services/upload/coverage_file_finder.py b/codecov_cli/services/upload/finders/result_file_finder.py similarity index 73% rename from codecov_cli/services/upload/coverage_file_finder.py rename to codecov_cli/services/upload/finders/result_file_finder.py index 555df796..094d430e 100644 --- a/codecov_cli/services/upload/coverage_file_finder.py +++ b/codecov_cli/services/upload/finders/result_file_finder.py @@ -8,35 +8,8 @@ logger = logging.getLogger("codecovcli") -coverage_files_patterns = [ - "*.clover", - "*.codecov.*", - "*.gcov", - "*.lcov", - "*.lst", - "*coverage*.*", - "*Jacoco*.xml", - "clover.xml", - "cobertura.xml", - "codecov-result.json", - "codecov.*", - "cover.out", - "coverage-final.json", - "excoveralls.json", - "gcov.info", - "jacoco*.xml", - "lcov.dat", - "pylcov.dat", - "lcov.info", - "luacov.report.out", - "naxsi.info", - "nosetests.xml", - "report.xml", - "test_cov.xml", -] - -coverage_files_excluded_patterns = [ +excluded_patterns = [ "*.am", "*.bash", "*.bat", @@ -170,7 +143,9 @@ ] -class CoverageFileFinder(object): +class ResultFileFinder(object): + file_patterns = None + def __init__( self, project_root: Path = None, @@ -183,36 +158,32 @@ def __init__( self.explicitly_listed_files = explicitly_listed_files or None self.disable_search = disable_search - def find_coverage_files(self) -> typing.List[UploadCollectionResultFile]: - regex_patterns_to_exclude = globs_to_regex(coverage_files_excluded_patterns) - coverage_files_paths = [] - user_coverage_files_paths = [] + def find_files(self) -> typing.List[UploadCollectionResultFile]: + regex_patterns_to_exclude = globs_to_regex(excluded_patterns) + files_paths = [] + user_files_paths = [] if self.explicitly_listed_files: - user_coverage_files_paths = self.get_user_specified_coverage_files( - regex_patterns_to_exclude - ) + user_files_paths = self.get_user_specified_files(regex_patterns_to_exclude) if not self.disable_search: - regex_patterns_to_include = globs_to_regex(coverage_files_patterns) - coverage_files_paths = search_files( + regex_patterns_to_include = globs_to_regex(self.file_patterns) + files_paths = search_files( self.project_root, default_folders_to_ignore + self.folders_to_ignore, filename_include_regex=regex_patterns_to_include, filename_exclude_regex=regex_patterns_to_exclude, ) result_files = [ - UploadCollectionResultFile(path) - for path in coverage_files_paths - if coverage_files_paths + UploadCollectionResultFile(path) for path in files_paths if files_paths ] user_result_files = [ UploadCollectionResultFile(path) - for path in user_coverage_files_paths - if user_coverage_files_paths + for path in user_files_paths + if user_files_paths ] return list(set(result_files + user_result_files)) - def get_user_specified_coverage_files(self, regex_patterns_to_exclude): + def get_user_specified_files(self, regex_patterns_to_exclude): user_filenames_to_include = [] files_excluded_but_user_includes = [] for file in self.explicitly_listed_files: @@ -230,7 +201,7 @@ def get_user_specified_coverage_files(self, regex_patterns_to_exclude): multipart_include_regex = globs_to_regex( [str(path.resolve()) for path in self.explicitly_listed_files] ) - user_coverage_files_paths = list( + user_files_paths = list( search_files( self.project_root, default_folders_to_ignore + self.folders_to_ignore, @@ -241,7 +212,7 @@ def get_user_specified_coverage_files(self, regex_patterns_to_exclude): ) not_found_files = [] for filepath in self.explicitly_listed_files: - if filepath.resolve() not in user_coverage_files_paths: + if filepath.resolve() not in user_files_paths: not_found_files.append(filepath) if not_found_files: @@ -250,15 +221,4 @@ def get_user_specified_coverage_files(self, regex_patterns_to_exclude): extra=dict(extra_log_attributes=dict(not_found_files=not_found_files)), ) - return user_coverage_files_paths - - -def select_coverage_file_finder( - root_folder_to_search, folders_to_ignore, explicitly_listed_files, disable_search -): - return CoverageFileFinder( - root_folder_to_search, - folders_to_ignore, - explicitly_listed_files, - disable_search, - ) + return user_files_paths diff --git a/codecov_cli/services/upload/finders/testing_result_file_finder.py b/codecov_cli/services/upload/finders/testing_result_file_finder.py new file mode 100644 index 00000000..59738184 --- /dev/null +++ b/codecov_cli/services/upload/finders/testing_result_file_finder.py @@ -0,0 +1,15 @@ +import logging + +from codecov_cli.services.upload.finders.result_file_finder import ResultFileFinder + +logger = logging.getLogger("codecovcli") + +testing_result_patterns = ["**junit.xml"] + + +class TestingResultFileFinder(ResultFileFinder): + file_patterns = testing_result_patterns + + +def select_testing_result_file_finder(*args, **kwargs): + return TestingResultFileFinder(*args, **kwargs) diff --git a/codecov_cli/services/upload/legacy_upload_sender.py b/codecov_cli/services/upload/legacy_upload_sender.py index 99ff6429..98baea85 100644 --- a/codecov_cli/services/upload/legacy_upload_sender.py +++ b/codecov_cli/services/upload/legacy_upload_sender.py @@ -37,7 +37,8 @@ class UploadSendingResult(object): class LegacyUploadSender(object): def send_upload_data( self, - upload_data: UploadCollectionResult, + upload_file_type: str, + upload_data: bytes, commit_sha: str, token: uuid.UUID, env_vars: typing.Dict[str, str], @@ -54,7 +55,6 @@ def send_upload_data( git_service: typing.Optional[str] = None, enterprise_url: typing.Optional[str] = None, ) -> UploadSendingResult: - params = { "package": f"codecov-cli/{codecov_cli_version}", "commit": commit_sha, @@ -83,49 +83,5 @@ def send_upload_data( return resp result_url, put_url = resp.text.split("\n") - reports_payload = self._generate_payload(upload_data, env_vars) - resp = send_put_request(put_url, data=reports_payload) + resp = send_put_request(put_url, data=upload_data) return resp - - def _generate_payload( - self, upload_data: UploadCollectionResult, env_vars: typing.Dict[str, str] - ) -> bytes: - env_vars_section = self._generate_env_vars_section(env_vars) - network_section = self._generate_network_section(upload_data) - coverage_files_section = self._generate_coverage_files_section(upload_data) - - return b"".join([env_vars_section, network_section, coverage_files_section]) - - def _generate_env_vars_section(self, env_vars) -> bytes: - filtered_env_vars = { - key: value for key, value in env_vars.items() if value is not None - } - - if not filtered_env_vars: - return b"" - - env_vars_section = "".join( - f"{env_var}={value}\n" for env_var, value in filtered_env_vars.items() - ) - return env_vars_section.encode() + b"<<<<<< ENV\n" - - def _generate_network_section(self, upload_data: UploadCollectionResult) -> bytes: - network_files = upload_data.network - - if not network_files: - return b"" - - network_files_section = "".join(file + "\n" for file in network_files) - return network_files_section.encode() + b"<<<<<< network\n" - - def _generate_coverage_files_section(self, upload_data: UploadCollectionResult): - return b"".join( - self._format_coverage_file(file) for file in upload_data.coverage_files - ) - - def _format_coverage_file(self, file: UploadCollectionResultFile) -> bytes: - header = b"# path=" + file.get_filename() + b"\n" - file_content = file.get_content() + b"\n" - file_end = b"<<<<<< EOF\n" - - return header + file_content + file_end diff --git a/codecov_cli/services/upload/upload_sender.py b/codecov_cli/services/upload/upload_sender.py index 447aa8e4..d67c9d5e 100644 --- a/codecov_cli/services/upload/upload_sender.py +++ b/codecov_cli/services/upload/upload_sender.py @@ -26,7 +26,8 @@ class UploadSender(object): def send_upload_data( self, - upload_data: UploadCollectionResult, + upload_file_type: str, + upload_data: bytes, commit_sha: str, token: uuid.UUID, env_vars: typing.Dict[str, str], @@ -44,6 +45,7 @@ def send_upload_data( enterprise_url: typing.Optional[str] = None, ) -> RequestResult: data = { + "upload_file_type": upload_file_type, "ci_url": build_url, "flags": flags, "env": env_vars, @@ -57,8 +59,6 @@ def send_upload_data( encoded_slug = encode_slug(slug) upload_url = enterprise_url or CODECOV_API_URL url = f"{upload_url}/upload/{git_service}/{encoded_slug}/commits/{commit_sha}/reports/{report_code}/uploads" - # Data that goes to storage - reports_payload = self._generate_payload(upload_data, env_vars) logger.debug("Sending upload request to Codecov") resp_from_codecov = send_post_request( @@ -79,69 +79,5 @@ def send_upload_data( ) put_url = resp_json_obj["raw_upload_location"] logger.debug("Sending upload to storage") - resp_from_storage = send_put_request(put_url, data=reports_payload) + resp_from_storage = send_put_request(put_url, data=upload_data) return resp_from_storage - - def _generate_payload( - self, upload_data: UploadCollectionResult, env_vars: typing.Dict[str, str] - ) -> bytes: - network_files = upload_data.network - payload = { - "report_fixes": { - "format": "legacy", - "value": self._get_file_fixers(upload_data), - }, - "network_files": network_files if network_files is not None else [], - "coverage_files": self._get_coverage_files(upload_data), - "metadata": {}, - } - - json_data = json.dumps(payload) - return json_data.encode() - - def _get_file_fixers( - self, upload_data: UploadCollectionResult - ) -> Dict[str, Dict[str, Any]]: - """ - Returns file/path fixes in the following format: - - { - {path}: { - "eof": int(eof_line), - "lines": {set_of_lines}, - }, - } - """ - file_fixers = {} - for file_fixer in upload_data.file_fixes: - fixed_lines_with_reason = set( - [fixer[0] for fixer in file_fixer.fixed_lines_with_reason] - ) - total_fixed_lines = list( - file_fixer.fixed_lines_without_reason.union(fixed_lines_with_reason) - ) - file_fixers[str(file_fixer.path)] = { - "eof": file_fixer.eof, - "lines": total_fixed_lines, - } - - return file_fixers - - def _get_coverage_files(self, upload_data: UploadCollectionResult): - return [self._format_coverage_file(file) for file in upload_data.coverage_files] - - def _format_coverage_file(self, file: UploadCollectionResultFile): - format, formatted_content = self._get_format_info(file) - return { - "filename": file.get_filename().decode(), - "format": format, - "data": formatted_content, - "labels": "", - } - - def _get_format_info(self, file: UploadCollectionResultFile): - format = "base64+compressed" - formatted_content = ( - base64.b64encode(zlib.compress((file.get_content()))) - ).decode() - return format, formatted_content diff --git a/codecov_cli/types.py b/codecov_cli/types.py index 95f9f759..772b36d9 100644 --- a/codecov_cli/types.py +++ b/codecov_cli/types.py @@ -14,6 +14,10 @@ def get_content(self) -> bytes: with open(self.path, "rb") as f: return f.read() + def get_string(self) -> bytes: + with open(self.path, "r") as f: + return f.read() + def __repr__(self) -> str: return str(self.path) @@ -39,7 +43,7 @@ class UploadCollectionResultFileFixer(object): @dataclass class UploadCollectionResult(object): __slots__ = ["network", "coverage_files", "file_fixes"] - network: typing.List[str] + network: typing.List[UploadCollectionResultFile] coverage_files: typing.List[UploadCollectionResultFile] file_fixes: typing.List[UploadCollectionResultFileFixer] diff --git a/tests/commands/test_invoke_upload.py b/tests/commands/test_invoke_upload.py index 360b172b..7bbcac43 100644 --- a/tests/commands/test_invoke_upload.py +++ b/tests/commands/test_invoke_upload.py @@ -2,7 +2,10 @@ from codecov_cli.fallbacks import FallbackFieldEnum from codecov_cli.main import cli -from codecov_cli.services.upload import UploadCollector, UploadSender +from codecov_cli.services.upload import UploadSender +from codecov_cli.services.upload.collectors.coverage_upload_collector import ( + CoverageUploadCollector, +) from codecov_cli.types import RequestError, RequestResult from tests.factory import FakeProvider, FakeVersioningSystem from tests.test_helpers import parse_outstreams_into_log_lines @@ -32,7 +35,9 @@ def test_upload_raise_Z_option(mocker): upload_sender = mocker.patch.object( UploadSender, "send_upload_data", return_value=result ) - upload_collector = mocker.patch.object(UploadCollector, "generate_upload_data") + upload_collector = mocker.patch.object( + CoverageUploadCollector, "generate_upload_data" + ) fake_ci_provider = FakeProvider({FallbackFieldEnum.commit_sha: None}) mocker.patch("codecov_cli.main.get_ci_adapter", return_value=fake_ci_provider) diff --git a/tests/helpers/test_legacy_upload_sender.py b/tests/helpers/test_legacy_upload_sender.py index 405f87e0..7c3009af 100644 --- a/tests/helpers/test_legacy_upload_sender.py +++ b/tests/helpers/test_legacy_upload_sender.py @@ -7,6 +7,9 @@ from codecov_cli import __version__ as codecov_cli_version from codecov_cli.services.upload.legacy_upload_sender import LegacyUploadSender +from codecov_cli.services.upload.collectors.legacy_upload_collector import ( + LegacyUploadCollector, +) from codecov_cli.types import UploadCollectionResult from tests.data import reports_examples @@ -187,7 +190,9 @@ def test_generate_env_vars_section(self): env_vars = {"var1": "value1", "var2": "value2", "var3": None, "abc": "valbc"} actual_lines = ( - LegacyUploadSender()._generate_env_vars_section(env_vars).split(b"\n") + LegacyUploadCollector(None, None, None, None) + ._generate_env_vars_section(env_vars) + .split(b"\n") ) assert terminator in actual_lines @@ -208,7 +213,12 @@ def test_generate_env_vars_section(self): def test_generate_env_vars_section_empty_result(self): env_vars = {"var1": None} - assert LegacyUploadSender()._generate_env_vars_section(env_vars) == b"" + assert ( + LegacyUploadCollector(None, None, None, None)._generate_env_vars_section( + env_vars + ) + == b"" + ) def test_generate_network_section(self): network_files = [ @@ -237,9 +247,9 @@ def test_generate_network_section(self): upload_data = UploadCollectionResult(network_files, [], []) - actual_network_section = LegacyUploadSender()._generate_network_section( - upload_data - ) + actual_network_section = LegacyUploadCollector( + None, None, None, None + )._generate_network_section(upload_data) assert [line.strip() for line in expected_network_section.split(b"\n")] == [ line for line in actual_network_section.split(b"\n") @@ -247,7 +257,7 @@ def test_generate_network_section(self): def test_generate_network_section_empty_result(self): assert ( - LegacyUploadSender()._generate_network_section( + LegacyUploadCollector(None, None, None, None)._generate_network_section( UploadCollectionResult([], [], []) ) == b"" @@ -270,9 +280,9 @@ def test_format_coverage_file(self, mocker): fake_result_file.get_content.return_value = coverage_file_seperated[1][ : -len(b"\n<<<<<< EOF\n") ] - actual_coverage_file_section = LegacyUploadSender()._format_coverage_file( - fake_result_file - ) + actual_coverage_file_section = LegacyUploadCollector( + None, None, None, None + )._format_coverage_file(fake_result_file) assert ( actual_coverage_file_section @@ -280,9 +290,8 @@ def test_format_coverage_file(self, mocker): ) def test_generate_coverage_files_section(self, mocker): - mocker.patch( - "codecov_cli.services.upload.LegacyUploadSender._format_coverage_file", + "codecov_cli.services.upload.collectors.legacy_upload_collector.LegacyUploadCollector._format_coverage_file", side_effect=lambda file_bytes: file_bytes, ) @@ -293,7 +302,9 @@ def test_generate_coverage_files_section(self, mocker): reports_examples.coverage_file_section_simple, ] - actual_section = LegacyUploadSender()._generate_coverage_files_section( + actual_section = LegacyUploadCollector( + None, None, None, None + )._generate_coverage_files_section( UploadCollectionResult([], coverage_files, []) ) @@ -303,18 +314,20 @@ def test_generate_coverage_files_section(self, mocker): def test_generate_payload_overall(self, mocker): mocker.patch( - "codecov_cli.services.upload.LegacyUploadSender._generate_env_vars_section", + "codecov_cli.services.upload.collectors.legacy_upload_collector.LegacyUploadCollector._generate_env_vars_section", return_value=reports_examples.env_section, ) mocker.patch( - "codecov_cli.services.upload.LegacyUploadSender._generate_network_section", + "codecov_cli.services.upload.collectors.legacy_upload_collector.LegacyUploadCollector._generate_network_section", return_value=reports_examples.network_section, ) mocker.patch( - "codecov_cli.services.upload.LegacyUploadSender._generate_coverage_files_section", + "codecov_cli.services.upload.collectors.legacy_upload_collector.LegacyUploadCollector._generate_coverage_files_section", return_value=reports_examples.coverage_file_section_simple, ) - actual_report = LegacyUploadSender()._generate_payload(None, None) + actual_report = LegacyUploadCollector(None, None, None, None)._generate_payload( + None, None + ) assert actual_report == reports_examples.env_network_coverage_sections diff --git a/tests/helpers/test_network_finder.py b/tests/helpers/test_network_finder.py index b7881812..ca2889ed 100644 --- a/tests/helpers/test_network_finder.py +++ b/tests/helpers/test_network_finder.py @@ -1,16 +1,19 @@ +from pathlib import Path from unittest.mock import MagicMock import pytest -from codecov_cli.services.upload.network_finder import NetworkFinder +from codecov_cli.services.upload.finders.network_finder import NetworkFinder +from codecov_cli.types import UploadCollectionResultFile def test_find_files(mocker, tmp_path): - expected_filenames = ["a.txt", "b.txt"] mocked_vs = MagicMock() mocked_vs.list_relevant_files.return_value = expected_filenames - assert NetworkFinder(mocked_vs).find_files(tmp_path) == expected_filenames + assert NetworkFinder(mocked_vs).find_files(tmp_path) == [ + UploadCollectionResultFile(Path(filename)) for filename in expected_filenames + ] mocked_vs.list_relevant_files.assert_called_with(tmp_path) diff --git a/tests/helpers/test_upload_sender.py b/tests/helpers/test_upload_sender.py index 75876695..d1e51cc1 100644 --- a/tests/helpers/test_upload_sender.py +++ b/tests/helpers/test_upload_sender.py @@ -9,6 +9,9 @@ from codecov_cli import __version__ as codecov_cli_version from codecov_cli.helpers.encoder import encode_slug from codecov_cli.services.upload.upload_sender import UploadSender +from codecov_cli.services.upload.collectors.coverage_upload_collector import ( + CoverageUploadCollector, +) from codecov_cli.types import UploadCollectionResult, UploadCollectionResultFileFixer from tests.data import reports_examples @@ -233,7 +236,9 @@ def test_upload_sender_http_error_with_invalid_sha( class TestPayloadGeneration(object): def test_generate_payload_overall(self, mocked_coverage_file): - actual_report = UploadSender()._generate_payload( + actual_report = CoverageUploadCollector( + None, None, None, None + )._generate_payload( get_fake_upload_collection_result(mocked_coverage_file), None ) expected_report = { @@ -300,9 +305,9 @@ def test_generate_payload_overall(self, mocked_coverage_file): assert actual_report == json.dumps(expected_report).encode() def test_generate_empty_payload_overall(self): - actual_report = UploadSender()._generate_payload( - UploadCollectionResult([], [], []), None - ) + actual_report = CoverageUploadCollector( + None, None, None, None + )._generate_payload(UploadCollectionResult([], [], []), None) expected_report = { "report_fixes": { "format": "legacy", @@ -315,9 +320,9 @@ def test_generate_empty_payload_overall(self): assert actual_report == json.dumps(expected_report).encode() def test_formatting_file_coverage_info(self, mocker, mocked_coverage_file): - format, formatted_content = UploadSender()._get_format_info( - mocked_coverage_file - ) + format, formatted_content = CoverageUploadCollector( + None, None, None, None + )._get_format_info(mocked_coverage_file) assert format == "base64+compressed" assert ( formatted_content @@ -326,12 +331,12 @@ def test_formatting_file_coverage_info(self, mocker, mocked_coverage_file): def test_coverage_file_format(self, mocker, mocked_coverage_file): mocker.patch( - "codecov_cli.services.upload.upload_sender.UploadSender._get_format_info", + "codecov_cli.services.upload.collectors.coverage_upload_collector.CoverageUploadCollector._get_format_info", return_value=("base64+compressed", "encoded_file_data"), ) - json_formatted_coverage_file = UploadSender()._format_coverage_file( - mocked_coverage_file - ) + json_formatted_coverage_file = CoverageUploadCollector( + None, None, None, None + )._format_coverage_file(mocked_coverage_file) print(json_formatted_coverage_file["data"]) assert json_formatted_coverage_file == { "filename": mocked_coverage_file.get_filename().decode(), diff --git a/tests/services/upload/test_coverage_file_finder.py b/tests/services/upload/test_coverage_file_finder.py index dd83a8a4..87277c04 100644 --- a/tests/services/upload/test_coverage_file_finder.py +++ b/tests/services/upload/test_coverage_file_finder.py @@ -2,17 +2,17 @@ import unittest from pathlib import Path -from codecov_cli.services.upload.coverage_file_finder import CoverageFileFinder +from codecov_cli.services.upload.finders.coverage_file_finder import CoverageFileFinder from codecov_cli.types import UploadCollectionResultFile class TestCoverageFileFinder(object): - def test_find_coverage_files_mocked_search_files(self, mocker): + def test_find_files_mocked_search_files(self, mocker): mocker.patch( - "codecov_cli.services.upload.coverage_file_finder.search_files", + "codecov_cli.services.upload.finders.result_file_finder.search_files", return_value=[], ) - assert CoverageFileFinder().find_coverage_files() == [] + assert CoverageFileFinder().find_files() == [] coverage_files_paths = [ Path("a/b.txt"), @@ -20,7 +20,7 @@ def test_find_coverage_files_mocked_search_files(self, mocker): ] mocker.patch( - "codecov_cli.services.upload.coverage_file_finder.search_files", + "codecov_cli.services.upload.finders.result_file_finder.search_files", return_value=coverage_files_paths, ) @@ -32,12 +32,12 @@ def test_find_coverage_files_mocked_search_files(self, mocker): expected_paths = sorted([file.get_filename() for file in expected]) actual_paths = sorted( - [file.get_filename() for file in CoverageFileFinder().find_coverage_files()] + [file.get_filename() for file in CoverageFileFinder().find_files()] ) assert expected_paths == actual_paths - def test_find_coverage_files(self, tmp_path): + def test_find_files(self, tmp_path): (tmp_path / "sub").mkdir() (tmp_path / "sub" / "subsub").mkdir() (tmp_path / "node_modules").mkdir() @@ -87,12 +87,12 @@ def test_find_coverage_files(self, tmp_path): expected = { UploadCollectionResultFile((tmp_path / file)) for file in should_find } - actual = set(CoverageFileFinder(tmp_path).find_coverage_files()) + actual = set(CoverageFileFinder(tmp_path).find_files()) assert actual == expected extra = tmp_path / "sub" / "nosetests.xml" extra.touch() - actual = set(CoverageFileFinder(tmp_path).find_coverage_files()) + actual = set(CoverageFileFinder(tmp_path).find_files()) assert actual - expected == {UploadCollectionResultFile(extra)} @@ -116,7 +116,7 @@ def setUp(self): def tearDown(self): self.temp_dir.cleanup() # Clean up the temporary directory - def test_find_coverage_files_with_existing_files(self): + def test_find_files_with_existing_files(self): # Create some sample coverage files coverage_files = [ self.project_root / "coverage.xml", @@ -128,10 +128,7 @@ def test_find_coverage_files_with_existing_files(self): file.touch() result = sorted( - [ - file.get_filename() - for file in self.coverage_file_finder.find_coverage_files() - ] + [file.get_filename() for file in self.coverage_file_finder.find_files()] ) expected = [ UploadCollectionResultFile(Path(f"{self.project_root}/coverage.xml")), @@ -142,11 +139,11 @@ def test_find_coverage_files_with_existing_files(self): expected_paths = sorted([file.get_filename() for file in expected]) self.assertEqual(result, expected_paths) - def test_find_coverage_files_with_no_files(self): - result = self.coverage_file_finder.find_coverage_files() + def test_find_files_with_no_files(self): + result = self.coverage_file_finder.find_files() self.assertEqual(result, []) - def test_find_coverage_files_with_disabled_search(self): + def test_find_files_with_disabled_search(self): # Create some sample coverage files print("project root", self.project_root) coverage_files = [ @@ -163,10 +160,7 @@ def test_find_coverage_files_with_disabled_search(self): self.coverage_file_finder.disable_search = True result = sorted( - [ - file.get_filename() - for file in self.coverage_file_finder.find_coverage_files() - ] + [file.get_filename() for file in self.coverage_file_finder.find_files()] ) expected = [ @@ -179,7 +173,7 @@ def test_find_coverage_files_with_disabled_search(self): self.assertEqual(result, expected_paths) - def test_find_coverage_files_with_user_specified_files(self): + def test_find_files_with_user_specified_files(self): # Create some sample coverage files coverage_files = [ self.project_root / "coverage.xml", @@ -192,10 +186,7 @@ def test_find_coverage_files_with_user_specified_files(self): file.touch() result = sorted( - [ - file.get_filename() - for file in self.coverage_file_finder.find_coverage_files() - ] + [file.get_filename() for file in self.coverage_file_finder.find_files()] ) expected = [ @@ -211,7 +202,7 @@ def test_find_coverage_files_with_user_specified_files(self): expected_paths = sorted([file.get_filename() for file in expected]) self.assertEqual(result, expected_paths) - def test_find_coverage_files_with_user_specified_files_not_found(self): + def test_find_files_with_user_specified_files_not_found(self): # Create some sample coverage files coverage_files = [ self.project_root / "coverage.xml", @@ -227,10 +218,7 @@ def test_find_coverage_files_with_user_specified_files_not_found(self): ) result = sorted( - [ - file.get_filename() - for file in self.coverage_file_finder.find_coverage_files() - ] + [file.get_filename() for file in self.coverage_file_finder.find_files()] ) expected = [ diff --git a/tests/services/upload/test_testing_result_file_finder.py b/tests/services/upload/test_testing_result_file_finder.py new file mode 100644 index 00000000..11a3c273 --- /dev/null +++ b/tests/services/upload/test_testing_result_file_finder.py @@ -0,0 +1,221 @@ +import tempfile +import unittest +from pathlib import Path + +from codecov_cli.services.upload.finders.testing_result_file_finder import ( + TestingResultFileFinder, +) +from codecov_cli.types import UploadCollectionResultFile + + +class TestTestingResultFileFinder(object): + def test_find_files_mocked_search_files(self, mocker): + mocker.patch( + "codecov_cli.services.upload.finders.result_file_finder.search_files", + return_value=[], + ) + assert TestingResultFileFinder().find_files() == [] + + coverage_files_paths = [ + Path("a/junit.xml"), + ] + + mocker.patch( + "codecov_cli.services.upload.finders.result_file_finder.search_files", + return_value=coverage_files_paths, + ) + + expected = [ + UploadCollectionResultFile(Path("a/junit.xml")), + ] + + expected_paths = sorted([file.get_filename() for file in expected]) + + actual_paths = sorted( + [file.get_filename() for file in TestingResultFileFinder().find_files()] + ) + + assert expected_paths == actual_paths + + def test_find_files(self, tmp_path): + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "subsub").mkdir() + (tmp_path / "node_modules").mkdir() + + should_find = ["junit.xml", "sub/subsub/junit.xml"] + + should_ignore = [ + "abc.codecov.exe", + "sub/abc.codecov.exe", + "codecov.exe", + "__pycache__", + "sub/subsub/__pycache__", + ".gitignore", + "a.sql", + "a.csv", + ".abc-coveragerc", + ".coverage-xyz", + "sub/scoverage.measurements.xyz", + "sub/test_abcd_coverage.txt", + "test-result-ff-codecoverage.json", + "node_modules/abc-coverage.cov", + "node_modules/junit.xml", + ] + + for filename in should_find: + (tmp_path / filename).touch() + + for filename in should_ignore: + (tmp_path / filename).touch() + + expected = { + UploadCollectionResultFile((tmp_path / file)) for file in should_find + } + actual = set(TestingResultFileFinder(tmp_path).find_files()) + assert actual == expected + + extra = tmp_path / "sub" / "junit.xml" + extra.touch() + actual = set(TestingResultFileFinder(tmp_path).find_files()) + assert actual - expected == {UploadCollectionResultFile(extra)} + + +class TestTestingResultFileFinderUserInput(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() # Create a temporary directory + self.project_root = Path(self.temp_dir.name) + self.folders_to_ignore = [] + self.explicitly_listed_files = [ + self.project_root / "subdirectory" / "junit.xml", + ] + self.disable_search = False + self.testing_result_file_finder = TestingResultFileFinder( + self.project_root, + self.folders_to_ignore, + self.explicitly_listed_files, + self.disable_search, + ) + + def tearDown(self): + self.temp_dir.cleanup() # Clean up the temporary directory + + def test_find_files_with_existing_files(self): + # Create some sample coverage files + coverage_files = [ + self.project_root / "subdirectory" / "junit.xml", + self.project_root / "other_file.txt", + ] + (self.project_root / "subdirectory").mkdir() + for file in coverage_files: + file.touch() + + result = sorted( + [ + file.get_filename() + for file in self.testing_result_file_finder.find_files() + ] + ) + expected = [ + UploadCollectionResultFile( + Path(f"{self.project_root}/subdirectory/junit.xml") + ), + ] + expected_paths = sorted([file.get_filename() for file in expected]) + self.assertEqual(result, expected_paths) + + def test_find_files_with_no_files(self): + result = self.testing_result_file_finder.find_files() + self.assertEqual(result, []) + + def test_find_files_with_disabled_search(self): + # Create some sample coverage files + print("project root", self.project_root) + coverage_files = [ + self.project_root / "subdirectory" / "junit.xml", + self.project_root / "junit.xml", + ] + (self.project_root / "subdirectory").mkdir() + for file in coverage_files: + file.touch() + + # Disable search + self.testing_result_file_finder.disable_search = True + + result = sorted( + [ + file.get_filename() + for file in self.testing_result_file_finder.find_files() + ] + ) + + expected = [ + UploadCollectionResultFile( + Path(f"{self.project_root}/subdirectory/junit.xml") + ), + ] + expected_paths = sorted([file.get_filename() for file in expected]) + + self.assertEqual(result, expected_paths) + + def test_find_files_with_user_specified_files(self): + # Create some sample coverage files + coverage_files = [ + self.project_root / "subdirectory" / "junit.xml", + self.project_root / "extra.xml", + self.project_root / "junit.xml", + ] + (self.project_root / "subdirectory").mkdir() + for file in coverage_files: + file.touch() + + self.testing_result_file_finder.explicitly_listed_files.append( + self.project_root / "extra.xml" + ) + + result = sorted( + [ + file.get_filename() + for file in self.testing_result_file_finder.find_files() + ] + ) + + expected = [ + UploadCollectionResultFile( + Path(f"{self.project_root}/subdirectory/junit.xml") + ), + UploadCollectionResultFile(Path(f"{self.project_root}/junit.xml")), + UploadCollectionResultFile(Path(f"{self.project_root}/extra.xml")), + ] + expected_paths = sorted([file.get_filename() for file in expected]) + self.assertEqual(result, expected_paths) + + def test_find_files_with_user_specified_files_not_found(self): + # Create some sample coverage files + coverage_files = [ + self.project_root / "junit.xml", + self.project_root / "subdirectory" / "junit.xml", + ] + (self.project_root / "subdirectory").mkdir() + for file in coverage_files: + file.touch() + + # Add a non-existent file to explicitly_listed_files + self.testing_result_file_finder.explicitly_listed_files.append( + self.project_root / "non_existent.xml" + ) + + result = sorted( + [ + file.get_filename() + for file in self.testing_result_file_finder.find_files() + ] + ) + + expected = [ + UploadCollectionResultFile(Path(f"{self.project_root}/junit.xml")), + UploadCollectionResultFile( + Path(f"{self.project_root}/subdirectory/junit.xml") + ), + ] + expected_paths = sorted([file.get_filename() for file in expected]) + self.assertEqual(result, expected_paths) diff --git a/tests/services/upload/test_upload_collector.py b/tests/services/upload/test_upload_collector.py index 15279275..0d0d9c09 100644 --- a/tests/services/upload/test_upload_collector.py +++ b/tests/services/upload/test_upload_collector.py @@ -1,15 +1,20 @@ from pathlib import Path from unittest.mock import patch -from codecov_cli.services.upload.upload_collector import UploadCollector +from codecov_cli.services.upload.collectors.coverage_upload_collector import ( + CoverageUploadCollector, +) + +from codecov_cli.types import UploadCollectionResultFile def test_fix_kt_files(): kt_file = Path("tests/data/files_to_fix_examples/sample.kt") + result_file = UploadCollectionResultFile(kt_file) - col = UploadCollector(None, None, None) + col = CoverageUploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(kt_file)]) + fixes = col._produce_file_fixes_for_network([result_file]) assert len(fixes) == 1 fixes_for_kt_file = fixes[0] @@ -26,10 +31,11 @@ def test_fix_kt_files(): def test_fix_go_files(): go_file = Path("tests/data/files_to_fix_examples/sample.go") + result_file = UploadCollectionResultFile(go_file) - col = UploadCollector(None, None, None) + col = CoverageUploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(go_file)]) + fixes = col._produce_file_fixes_for_network([result_file]) assert len(fixes) == 1 fixes_for_go_file = fixes[0] @@ -48,14 +54,15 @@ def test_fix_go_files(): ) -@patch("codecov_cli.services.upload.upload_collector.open") +@patch("codecov_cli.types.open") def test_fix_bad_encoding_files(mock_open): mock_open.side_effect = UnicodeDecodeError("", bytes(), 0, 0, "") go_file = Path("tests/data/files_to_fix_examples/bad_encoding.go") + result_file = UploadCollectionResultFile(go_file) - col = UploadCollector(None, None, None) + col = CoverageUploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(go_file)]) + fixes = col._produce_file_fixes_for_network([result_file]) assert len(fixes) == 1 fixes_for_go_file = fixes[0] assert fixes_for_go_file.eof is None @@ -65,10 +72,11 @@ def test_fix_bad_encoding_files(mock_open): def test_fix_php_files(): php_file = Path("tests/data/files_to_fix_examples/sample.php") + result_file = UploadCollectionResultFile(php_file) - col = UploadCollector(None, None, None) + col = CoverageUploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(php_file)]) + fixes = col._produce_file_fixes_for_network([result_file]) assert len(fixes) == 1 fixes_for_php_file = fixes[0] @@ -80,10 +88,11 @@ def test_fix_php_files(): def test_fix_for_cpp_swift_vala(tmp_path): cpp_file = Path("tests/data/files_to_fix_examples/sample.cpp") + result_file = UploadCollectionResultFile(cpp_file) - col = UploadCollector(None, None, None) + col = CoverageUploadCollector(None, None, None) - fixes = col._produce_file_fixes_for_network([str(cpp_file)]) + fixes = col._produce_file_fixes_for_network([result_file]) assert len(fixes) == 1 fixes_for_cpp_file = fixes[0] @@ -103,7 +112,7 @@ def test_fix_for_cpp_swift_vala(tmp_path): def test_fix_when_disabled_fixes(tmp_path): cpp_file = Path("tests/data/files_to_fix_examples/sample.cpp") - col = UploadCollector(None, None, None, True) + col = CoverageUploadCollector(None, None, True, None) fixes = col._produce_file_fixes_for_network([str(cpp_file)]) diff --git a/tests/services/upload/test_upload_service.py b/tests/services/upload/test_upload_service.py index dfb6c4da..f65e6dbf 100644 --- a/tests/services/upload/test_upload_service.py +++ b/tests/services/upload/test_upload_service.py @@ -4,7 +4,6 @@ from codecov_cli.services.upload import ( LegacyUploadSender, - UploadCollector, UploadSender, do_upload_logic, ) @@ -12,6 +11,12 @@ UploadSendingResult, UploadSendingResultWarning, ) +from codecov_cli.services.upload.collectors.legacy_upload_collector import ( + LegacyUploadCollector, +) +from codecov_cli.services.upload.collectors.coverage_upload_collector import ( + CoverageUploadCollector, +) from codecov_cli.types import RequestResult from tests.test_helpers import parse_outstreams_into_log_lines @@ -27,7 +32,7 @@ def test_do_upload_logic_happy_path_legacy_uploader(mocker): "codecov_cli.services.upload.select_network_finder" ) mock_generate_upload_data = mocker.patch.object( - UploadCollector, "generate_upload_data" + LegacyUploadCollector, "generate_upload_data" ) mock_send_upload_data = mocker.patch.object( LegacyUploadSender, @@ -56,9 +61,9 @@ def test_do_upload_logic_happy_path_legacy_uploader(mocker): flags=None, name="name", network_root_folder=None, - coverage_files_search_root_folder=None, - coverage_files_search_exclude_folders=None, - coverage_files_search_explicitly_listed_files=None, + files_search_root_folder=None, + files_search_exclude_folders=None, + files_search_explicitly_listed_files=None, plugin_names=["first_plugin", "another", "forth"], token="token", branch="branch", @@ -113,7 +118,7 @@ def test_do_upload_logic_happy_path(mocker): "codecov_cli.services.upload.select_network_finder" ) mock_generate_upload_data = mocker.patch.object( - UploadCollector, "generate_upload_data" + CoverageUploadCollector, "generate_upload_data" ) mock_send_upload_data = mocker.patch.object( UploadSender, @@ -142,9 +147,9 @@ def test_do_upload_logic_happy_path(mocker): flags=None, name="name", network_root_folder=None, - coverage_files_search_root_folder=None, - coverage_files_search_exclude_folders=None, - coverage_files_search_explicitly_listed_files=None, + files_search_root_folder=None, + files_search_exclude_folders=None, + files_search_explicitly_listed_files=None, plugin_names=["first_plugin", "another", "forth"], token="token", branch="branch", @@ -198,7 +203,7 @@ def test_do_upload_logic_dry_run(mocker): "codecov_cli.services.upload.select_network_finder" ) mock_generate_upload_data = mocker.patch.object( - UploadCollector, "generate_upload_data" + CoverageUploadCollector, "generate_upload_data" ) mock_send_upload_data = mocker.patch.object( LegacyUploadSender, @@ -223,9 +228,9 @@ def test_do_upload_logic_dry_run(mocker): flags=None, name="name", network_root_folder=None, - coverage_files_search_root_folder=None, - coverage_files_search_exclude_folders=None, - coverage_files_search_explicitly_listed_files=None, + files_search_root_folder=None, + files_search_exclude_folders=None, + files_search_explicitly_listed_files=None, plugin_names=["first_plugin", "another", "forth"], token="token", branch="branch", @@ -259,7 +264,7 @@ def test_do_upload_logic_verbose(mocker, use_verbose_option): mocker.patch("codecov_cli.services.upload.select_preparation_plugins") mocker.patch("codecov_cli.services.upload.select_coverage_file_finder") mocker.patch("codecov_cli.services.upload.select_network_finder") - mocker.patch.object(UploadCollector, "generate_upload_data") + mocker.patch.object(LegacyUploadCollector, "generate_upload_data") mocker.patch.object( LegacyUploadSender, "send_upload_data", @@ -283,9 +288,9 @@ def test_do_upload_logic_verbose(mocker, use_verbose_option): flags=None, name="name", network_root_folder=None, - coverage_files_search_root_folder=None, - coverage_files_search_exclude_folders=None, - coverage_files_search_explicitly_listed_files=None, + files_search_root_folder=None, + files_search_exclude_folders=None, + files_search_explicitly_listed_files=None, plugin_names=["first_plugin", "another", "forth"], token="token", branch="branch", @@ -332,7 +337,7 @@ def side_effect(*args, **kwargs): raise click.ClickException("error") mock_generate_upload_data = mocker.patch( - "codecov_cli.services.upload.UploadCollector.generate_upload_data", + "codecov_cli.services.upload.collectors.coverage_upload_collector.CoverageUploadCollector.generate_upload_data", side_effect=side_effect, ) mock_upload_completion_call = mocker.patch( @@ -357,9 +362,9 @@ def side_effect(*args, **kwargs): flags=None, name="name", network_root_folder=None, - coverage_files_search_root_folder=None, - coverage_files_search_exclude_folders=None, - coverage_files_search_explicitly_listed_files=None, + files_search_root_folder=None, + files_search_exclude_folders=None, + files_search_explicitly_listed_files=None, plugin_names=["first_plugin", "another", "forth"], token="token", branch="branch", @@ -415,7 +420,7 @@ def side_effect(*args, **kwargs): ) mock_generate_upload_data = mocker.patch( - "codecov_cli.services.upload.UploadCollector.generate_upload_data", + "codecov_cli.services.upload.collectors.coverage_upload_collector.CoverageUploadCollector.generate_upload_data", side_effect=side_effect, ) cli_config = {} @@ -437,9 +442,9 @@ def side_effect(*args, **kwargs): flags=None, name="name", network_root_folder=None, - coverage_files_search_root_folder=None, - coverage_files_search_exclude_folders=None, - coverage_files_search_explicitly_listed_files=None, + files_search_root_folder=None, + files_search_exclude_folders=None, + files_search_explicitly_listed_files=None, plugin_names=["first_plugin", "another", "forth"], token="token", branch="branch",