diff --git a/.github/workflows/kai-demo-evaluator-linux.yml b/.github/workflows/kai-demo-evaluator-linux.yml index b827c208..10e7f9a1 100644 --- a/.github/workflows/kai-demo-evaluator-linux.yml +++ b/.github/workflows/kai-demo-evaluator-linux.yml @@ -1,4 +1,4 @@ -name: Run demo with kai client and evaluate results +name: Run demo with kai client and evaluate results (Linux) on: schedule: diff --git a/kai-client-ci/files.py b/kai-client-ci/files.py index 4362c94a..3f93e807 100644 --- a/kai-client-ci/files.py +++ b/kai-client-ci/files.py @@ -4,6 +4,8 @@ import zipfile import json import os +from pathlib import Path + import git import requests @@ -103,6 +105,26 @@ def set_executable_permissions(file_path: str): logger.error(f'Failed to set executable permissions for {file_path}: {e}') +import os +import zipfile +import requests + +def remove_dependency_from_requirements(requirements_path: Path, dependency: str): + """ + Removes a specific dependency from the requirements.txt file. + This prevents unnecessary installations of unused dependencies (Mostly to avoid errors on Windows). + """ + if not os.path.exists(requirements_path): + return + + with open(requirements_path, 'r') as file: + lines = file.readlines() + + filtered_lines = [line for line in lines if not line.strip().startswith(dependency)] + + with open(requirements_path, 'w') as file: + file.writelines(filtered_lines) + def on_rmtree_error(func, path, exc_info): """" Error handler for ``shutil.rmtree``. diff --git a/kai-client-ci/kai_handler.py b/kai-client-ci/kai_handler.py index 947b1bef..d22d9743 100755 --- a/kai-client-ci/kai_handler.py +++ b/kai-client-ci/kai_handler.py @@ -8,7 +8,7 @@ from consts import KAI_FILES_FOLDER, KAI_FOLDER from files import unzip_file, download_file, rename_extracted_folder, set_executable_permissions, copy_file, \ - clone_repository, on_rmtree_error + clone_repository, on_rmtree_error, remove_dependency_from_requirements from logger import get_logger from utils import is_windows @@ -19,13 +19,13 @@ def download_kai_release(): api_response = requests.get("https://api.github.com/repos/konveyor/kai/releases/latest") if api_response.status_code != 200: logger.error(f"Failed to fetch the latest release API. Status code: {api_response.status_code}") - return + raise Exception("Failed to fetch the latest release") release_data = api_response.json() assets = release_data.get("assets", []) if not assets: logger.error("No assets found in the latest release") - return + raise Exception("No assets found in the latest release") system = platform.system().lower() @@ -35,7 +35,7 @@ def download_kai_release(): rpc_server_name = "kai-rpc-server.linux-x86_64.zip" else: logger.error("Unsupported operating system") - return + raise ValueError("Unsupported operating system") rpc_server_url = next((asset["browser_download_url"] for asset in assets if asset["name"] == rpc_server_name), None) @@ -49,6 +49,7 @@ def download_kai_release(): download_file(source_code_url, file_path) unzip_file(file_path, KAI_FILES_FOLDER) rename_extracted_folder(KAI_FILES_FOLDER, "kai") + remove_dependency_from_requirements(Path(f"{KAI_FOLDER}/requirements.txt"), "imgui") logger.info(f"Downloading RPC server binary from {rpc_server_url}") @@ -56,7 +57,7 @@ def download_kai_release(): download_file(rpc_server_url, file_path) unzip_file(file_path, os.path.join(KAI_FILES_FOLDER, "kai", "example", "analysis")) - if system == "linux": + if not is_windows(): set_executable_permissions(os.path.join(KAI_FILES_FOLDER, "kai", "example", "analysis", "kai-rpc-server")) set_executable_permissions(os.path.join(KAI_FILES_FOLDER, "kai", "example", "analysis", "kai-analyzer-rpc")) @@ -67,20 +68,20 @@ def setup_kai_external_files(): for file_name in files_to_move: source_path = os.path.join('./fixtures', file_name) - target_path = os.path.join('./kai_files/kai/example/analysis', file_name) + target_path = os.path.join(f"{KAI_FOLDER}/example/analysis", file_name) copy_file(source_path, target_path) - copy_file('./fixtures/config.toml', './kai_files/kai/example/') + copy_file('./fixtures/config.toml', f"{KAI_FOLDER}/example/") clone_repository('rulesets', 'https://github.com/konveyor/rulesets.git', 'main') os.rename('data/rulesets/default/generated/', 'kai_files/kai/example/analysis/rulesets') shutil.rmtree('data/rulesets/', onerror=on_rmtree_error) clone_repository('coolstore', 'https://github.com/konveyor-ecosystem/coolstore', 'main') - os.rename('data/coolstore', 'kai_files/kai/example/coolstore') + os.rename('data/coolstore', f"{KAI_FOLDER}/example/coolstore") -def setup_kai_dependencies(): +def setup_kai_dependencies() -> None: venv_folder = os.path.join(KAI_FOLDER, "venv") if is_windows(): @@ -97,7 +98,7 @@ def setup_kai_dependencies(): subprocess.run([pip_executable, "install", "-r", os.path.join(KAI_FOLDER, "requirements.txt")], check=True) -def run_demo(): +def run_demo() -> None: python_venv_executable = get_python_venv_executable() demo_rel_path = Path(f"../../../{python_venv_executable}") cwd = os.path.join(KAI_FOLDER, "example") @@ -116,10 +117,11 @@ def run_demo(): logger.info(f"run_demo.py script executed successfully:\n{result.stdout}") else: logger.error(f"run_demo.py failed with return code {result.returncode}: \n{result.stderr}") + raise Exception(f"run_demo.py failed") def get_python_venv_executable() -> str: """ - :return: python venv executable path which is kai_files/kai + :return: python venv executable path inside kai_files/kai """ venv_folder = os.path.join(KAI_FOLDER, "venv") diff --git a/kai-client-ci/main.py b/kai-client-ci/main.py index f14c1040..4328ca04 100644 --- a/kai-client-ci/main.py +++ b/kai-client-ci/main.py @@ -28,6 +28,9 @@ kai_handler.run_demo() demo_end = time.time() + if count_modified_files(COOLSTORE_FOLDER) == 0: + raise Exception('No modified files found') + kai_eval_handler.download_kai_eval() kai_eval_handler.parse_kai_logs() @@ -62,7 +65,7 @@ try: zip_name = datetime.now().strftime('%Y-%m-%d--%H-%M') zip_path = zip_folder('data', zip_name, 'output') - report_data_url = s3_handler.upload(zip_path, zip_path) + report_data_url = s3_handler.upload(zip_path, zip_path.replace("\\", "/")) logger.info(f'Run data uploaded to {report_data_url}') s3_handler.delete('report.json')