From ffe2ee5b83794972e892dfc7b92fb6de818511aa Mon Sep 17 00:00:00 2001 From: Krazer Date: Mon, 30 Dec 2024 13:38:03 -0600 Subject: [PATCH 1/8] add command io --- .github/workflows/exe-release.yml | 64 ++++++ .github/workflows/release.yml | 3 - aider/args.py | 6 + aider/command_io.py | 344 ++++++++++++++++++++++++++++++ aider/main.py | 64 +++--- scripts/build.py | 167 +++++++++++++++ scripts/build_config.json | 8 + scripts/local_build.ps1 | 47 ++++ scripts/local_build.sh | 46 ++++ 9 files changed, 721 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/exe-release.yml create mode 100644 aider/command_io.py create mode 100755 scripts/build.py create mode 100644 scripts/build_config.json create mode 100644 scripts/local_build.ps1 create mode 100755 scripts/local_build.sh diff --git a/.github/workflows/exe-release.yml b/.github/workflows/exe-release.yml new file mode 100644 index 00000000000..ac74623bc34 --- /dev/null +++ b/.github/workflows/exe-release.yml @@ -0,0 +1,64 @@ +name: Exe Release + +on: + workflow_dispatch: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+_cio' + branches: + - 'commandio' + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Setup Environment (Linux/macOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python scripts/local_build.py + + - name: Setup Environment (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + python -m pip install --upgrade pip + ./scripts/local_build.ps1 + + - name: Build Release + run: python scripts/build.py + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: release-${{ runner.os }} + path: dist/* + + create-release: + needs: build + runs-on: ubuntu-latest + steps: + - name: Download all artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + files: artifacts/**/* + generate_release_notes: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ade95897ab6..0b1a84a67fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,9 +2,6 @@ name: PyPI Release on: workflow_dispatch: - push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' jobs: build_and_publish: diff --git a/aider/args.py b/aider/args.py index 2ba98853dae..ab8d93d995e 100644 --- a/aider/args.py +++ b/aider/args.py @@ -776,6 +776,12 @@ def get_parser(default_config_files, git_root): " or home directory)" ), ) + group.add_argument( + "--commandio", + action="store_true", + help="Run aider using commands via stdin/stdout", + default=False, + ) # This is a duplicate of the argument in the preparser and is a no-op by this time of # argument parsing, but it's here so that the help is displayed as expected. group.add_argument( diff --git a/aider/command_io.py b/aider/command_io.py new file mode 100644 index 00000000000..8148b1b732b --- /dev/null +++ b/aider/command_io.py @@ -0,0 +1,344 @@ +import sys +import json +import select +import re + +from aider.io import InputOutput +from aider.io import AutoCompleter +from aider.mdstream import MarkdownStream +from prompt_toolkit.document import Document + +class CommandMarkdownStream(MarkdownStream): + def __init__(self): + super().__init__() + self.last_position = 0 + + def update(self, text, final=False): + new_text = text[self.last_position:] + if new_text or final: + msg = { + "cmd": "assistant-stream", + "value": new_text, + "position": self.last_position, + "final": final + } + print(json.dumps(msg), flush=True) + self.last_position = len(text) + return + +class CommandIO(InputOutput): + def __init__( + self, + yes=False, + input_history_file=None, + chat_history_file=None, + encoding="utf-8", + dry_run=False, + llm_history_file=None + ): + super().__init__( + input_history_file=input_history_file, + chat_history_file=chat_history_file, + encoding=encoding, + dry_run=dry_run, + llm_history_file=llm_history_file, + ) + + self.edit_format:str = "whole" + self.yes = yes + self.input_buffer = "" + self.input_decoder = json.JSONDecoder() + self.updated_rel_fnames = None + self.updated_abs_read_only_fnames = None + + def set_edit_format(self, edit_format): + self.edit_format = edit_format + + def update_files(self, rel_fnames, abs_read_only_fnames): + if(rel_fnames == self.updated_rel_fnames and abs_read_only_fnames == self.updated_abs_read_only_fnames): + return + + msg = { + 'added': list(rel_fnames), + 'added_readonly': list(abs_read_only_fnames) + } + print(f"update_files: {msg}") + self.send_message('files', msg, False) + + self.updated_rel_fnames = rel_fnames + self.updated_abs_read_only_fnames = abs_read_only_fnames + + def get_input( + self, + root, + rel_fnames, + addable_rel_fnames, + commands, + abs_read_only_fnames=None, + edit_format=None + ): + self.update_files(rel_fnames, abs_read_only_fnames) + + obj = self.get_command() + + completer_instance = AutoCompleter( + root, + rel_fnames, + addable_rel_fnames, + commands, + self.encoding, + abs_read_only_fnames=abs_read_only_fnames, + ) + + if obj: + send, inp = self.run_command(obj, commands, completer_instance) + + if send: + return inp + + return "" + + def get_command(self, wait = True): + need_input = False + + while True: + try: + input_chunk = sys.stdin.readline() + +# print(f"read: {input_chunk}", flush=True) + if not input_chunk and need_input: + if wait: + select.select([sys.stdin], [], [], 1) + else: + return None + + if input_chunk: + self.input_buffer += input_chunk + + while self.input_buffer: + try: + obj, idx = self.input_decoder.raw_decode(self.input_buffer) + self.input_buffer = self.input_buffer[idx:].lstrip() + return obj + + except json.JSONDecodeError as e: + # If JSON is not complete, break + print(f"json not complete: {e.msg}", flush=True) + need_input = True + self.input_buffer.clear() + break + + except KeyboardInterrupt: + break + + return "" + + def run_command(self, obj, commands, completer_instance): + cmd_list = commands.get_commands() + + cmd = obj.get('cmd') + + if cmd in cmd_list: + return True, f"{cmd} {obj.get('value')}" + elif cmd == 'interactive': + text = obj.get('value', '') + cursor_position = len(text) + + document = Document(text, cursor_position=cursor_position) + completions = list(completer_instance.get_completions(document, None)) + + suggestions = [] + for completion in completions: + suggestion = { + 'text': completion.text, + 'display': completion.display or completion.text, + 'start_position': completion.start_position, + 'style': completion.style, + 'selected_style': completion.selected_style + } + suggestions.append(suggestion) + + self.send_message('auto_complete', suggestions, False) + return False, "" + elif cmd == 'user': + return True, obj.get('value') + return False, "" + + def user_input(self, inp, log_only=True): + self.send_message("user", inp) + return + + def ai_output(self, content): + hist = "\n" + content.strip() + "\n\n" + self.append_chat_history(hist) +# self.send_message("ai", content) + + def confirm_ask( + self, + question, + default="y", + subject=None, + explicit_yes_required=False, + group=None, + allow_never=False, + ): + self.num_user_asks += 1 + + question_id = (question, subject) + + if question_id in self.never_prompts: + return False + + if group and not group.show_group: + group = None + if group: + allow_never = True + + valid_responses = ["yes", "no"] + options = " (Y)es/(N)o" + if group: + if not explicit_yes_required: + options += "/(A)ll" + valid_responses.append("all") + options += "/(S)kip all" + valid_responses.append("skip") + if allow_never: + options += "/(D)on't ask again" + valid_responses.append("don't") + + msg = { + "cmd": "prompt", + "value": question, + "default": default, + "subject": subject, + "explicit_yes_required": explicit_yes_required, + "group": valid_responses, + "allow_never": allow_never + } + print(json.dumps(msg), flush=True) + + obj = self.get_command() + + cmd = obj.get('cmd') + res = "no" + + if cmd == "prompt_response": + res = obj.get('value') + + hist = f"{question.strip()} {res.strip()}" + self.append_chat_history(hist, linebreak=True, blockquote=True) + + return res.strip().lower().startswith("y") + + def prompt_ask(self, question, default="", subject=None): + res = self.confirm_ask(question, default) + + def _tool_message(self, type, message="", strip=True): + if message.strip(): + if "\n" in message: + for line in message.splitlines(): + self.append_chat_history(line, linebreak=True, blockquote=True, strip=strip) + else: + hist = message.strip() if strip else message + self.append_chat_history(hist, linebreak=True, blockquote=True) + + if not message: + return + + self.send_message(type, message) + + def tool_error(self, message="", strip=True): + self.num_error_outputs += 1 + self._tool_message("error", message, strip) + + def tool_warning(self, message="", strip=True): + self._tool_message("warning", message, strip) + + def send_message(self, type, message, escape=True): + if escape: + message = json.dumps(message)[1:-1] + msg = { + "cmd": type, + "value": message + } + print(json.dumps(msg), flush=True) + + def parse_tokens_cost(self, message): + # Match the tokens pattern + tokens_pattern = r'(\d+) sent, (\d+) received' + tokens_match = re.search(tokens_pattern, message) + + # Match the cost pattern + cost_pattern = r'\$(\d+\.\d+) message, \$(\d+\.\d+) session' + cost_match = re.search(cost_pattern, message) + + if tokens_match and cost_match: + sent = int(tokens_match.group(1)) + received = int(tokens_match.group(2)) + cost = float(cost_match.group(1)) + cost_session = float(cost_match.group(2)) + + return { + 'sent': sent, + 'received': received, + 'cost': cost, + 'cost_session': cost_session + } + + return None + + def check_for_info(self, message): + pattern = r"Aider v(?P\d+)\.(?P\d+)\.(?P\d+)(?:\.(?P[^\s]+))?" + match = re.search(pattern, message) + + if match: + version_info = match.groupdict() + self.send_message("version", version_info, False) + return True + + output_parse_map = [ + ("Main model:", "model", None), + ("Weak model:", "weak_model", None), + ("Git repo:", "repo", None), + ("Repo-map:", "repo_map", None), + ("Tokens:", "tokens", self.parse_tokens_cost) + ] + + for prefix, response_prefix, parser in output_parse_map: + if message.startswith(prefix): + remainder = message[len(prefix):].strip() + value = parser(remainder) if parser else remainder + self.send_message(response_prefix, value, False if parser else True) + return True + return False + + def tool_output(self, *messages, log_only=False, bold=False): + message=" ".join(messages) + + if not message: + return + + if messages: + hist = message + hist = f"{hist.strip()}" + self.append_chat_history(hist, linebreak=True, blockquote=True) + + if self.check_for_info(message): + return + + self.send_message("output", message) + + def get_assistant_mdstream(self): + mdStream = CommandMarkdownStream() + return mdStream + + def assistant_output(self, message, pretty=None): + if not message: + return + self.send_message("assistant", message) + + def print(self, message=""): + if not message: + return + + self.send_message("print", message) diff --git a/aider/main.py b/aider/main.py index ee55261a292..0a037da9b20 100644 --- a/aider/main.py +++ b/aider/main.py @@ -28,6 +28,7 @@ from aider.format_settings import format_settings, scrub_sensitive_info from aider.history import ChatSummary from aider.io import InputOutput +from aider.command_io import CommandIO from aider.llm import litellm # noqa: F401; properly init litellm on launch from aider.models import ModelSettings from aider.repo import ANY_GIT_ERROR, GitRepo @@ -512,30 +513,40 @@ def main(argv=None, input=None, output=None, force_git_root=None, return_coder=F editing_mode = EditingMode.VI if args.vim else EditingMode.EMACS def get_io(pretty): - return InputOutput( - pretty, - args.yes_always, - args.input_history_file, - args.chat_history_file, - input=input, - output=output, - user_input_color=args.user_input_color, - tool_output_color=args.tool_output_color, - tool_warning_color=args.tool_warning_color, - tool_error_color=args.tool_error_color, - completion_menu_color=args.completion_menu_color, - completion_menu_bg_color=args.completion_menu_bg_color, - completion_menu_current_color=args.completion_menu_current_color, - completion_menu_current_bg_color=args.completion_menu_current_bg_color, - assistant_output_color=args.assistant_output_color, - code_theme=args.code_theme, - dry_run=args.dry_run, - encoding=args.encoding, - llm_history_file=args.llm_history_file, - editingmode=editing_mode, - fancy_input=args.fancy_input, - multiline_mode=args.multiline, - ) + if args.commandio: + return CommandIO( + args.yes_always, + args.input_history_file, + args.chat_history_file, + encoding=args.encoding, + dry_run=args.dry_run, + llm_history_file=args.llm_history_file, + ) + else: + return InputOutput( + pretty, + args.yes_always, + args.input_history_file, + args.chat_history_file, + input=input, + output=output, + user_input_color=args.user_input_color, + tool_output_color=args.tool_output_color, + tool_warning_color=args.tool_warning_color, + tool_error_color=args.tool_error_color, + completion_menu_color=args.completion_menu_color, + completion_menu_bg_color=args.completion_menu_bg_color, + completion_menu_current_color=args.completion_menu_current_color, + completion_menu_current_bg_color=args.completion_menu_current_bg_color, + assistant_output_color=args.assistant_output_color, + code_theme=args.code_theme, + dry_run=args.dry_run, + encoding=args.encoding, + llm_history_file=args.llm_history_file, + editingmode=editing_mode, + fancy_input=args.fancy_input, + multiline_mode=args.multiline, + ) io = get_io(args.pretty) try: @@ -737,7 +748,10 @@ def get_io(pretty): editor_model=args.editor_model, editor_edit_format=args.editor_edit_format, ) - + if args.commandio: + if isinstance(io, CommandIO): + io.set_edit_format(main_model.edit_format) + if args.copy_paste and args.edit_format is None: if main_model.edit_format in ("diff", "whole"): main_model.edit_format = "editor-" + main_model.edit_format diff --git a/scripts/build.py b/scripts/build.py new file mode 100755 index 00000000000..0068966c62c --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +import json +import os +import platform +import subprocess +import sys +from pathlib import Path + +def read_config(): + config_path = Path("scripts/build_config.json") + if not config_path.exists(): + print("Error: build_config.json not found") + sys.exit(1) + + with open(config_path) as f: + config = json.load(f) + + # Add repo_path to config + config['repo_path'] = Path("external/aider") + return config + +def setup_repo(config): + # Create external directory if it doesn't exist + os.makedirs(config['repo_path'].parent, exist_ok=True) + + repo_info = config['aider_repo'] + + # Clone if needed + if not config['repo_path'].exists(): + print("Cloning aider repository...") + subprocess.run(["git", "clone", repo_info['url'], config['repo_path']], check=True) + + # Setup correct branch/commit + original_dir = os.getcwd() + os.chdir(config['repo_path']) + + subprocess.run(["git", "fetch"], check=True) + subprocess.run(["git", "checkout", repo_info['branch']], check=True) + + if repo_info['commit'] != repo_info['branch']: + print("Checking out specific commit...") + subprocess.run(["git", "checkout", "-q", repo_info['commit']], check=True) + + os.chdir(original_dir) + +def analyze_imports(config): + from importlib.metadata import distributions + import importlib + + # Read requirements.txt and clean up entries + requirements = set() + with open('requirements.txt', 'r') as f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + # Split on any whitespace and take first part + pkg = line.split()[0] + # Remove version specifiers + pkg = pkg.split('==')[0].split('>=')[0].split('<=')[0] + requirements.add(pkg) + + # Build package to module mapping + pkg_to_module = {} + for dist in distributions(): + print(f"Checking package {dist.name}...") + if dist.name in requirements: + try: + # Try to import the module with the same name as the package + module_name = dist.name.replace('-', '_') # Common replacement + importlib.import_module(module_name) + if module_name != dist.name: + print(f"Package {dist.name} has different module name: {module_name}") + pkg_to_module[dist.name] = module_name + except ImportError as e: + # If import fails, try to parse the actual module name from the error + error_msg = str(e) + if "No module named" in error_msg: + print(f"Package {dist.name} has no module name: {error_msg}") + + # Run PyInstaller with debug imports + debug_output = subprocess.run( + f"pyinstaller --debug=imports aider/run.py", + shell=True, + capture_output=True, + text=True + ).stdout + + # Parse debug output to find missing imports + imported_modules = set() + for line in debug_output.split('\n'): + if 'LOADER: Import' in line: + module = line.split()[2].split('.')[0] + if module: # Ensure we have a valid module name + imported_modules.add(module) + + # Compare with requirements, using module names for PyInstaller + missing_imports = set() + for pkg in requirements: + if pkg in pkg_to_module: + # Use the mapped module name if we found one + module_name = pkg_to_module[pkg] + if module_name not in imported_modules: + missing_imports.add(module_name) + elif pkg not in imported_modules and not pkg.startswith('#'): + # Default to package name with hyphens replaced by underscores + module_name = pkg.replace('-', '_') + missing_imports.add(module_name) + + return missing_imports + +def build_project(config): + original_dir = os.getcwd() + os.chdir(config['repo_path']) + + # Create output directory + output_dir = Path("external/bin") + os.makedirs(f"../../{output_dir}", exist_ok=True) + + # Install dependencies + print("Installing dependencies...") + subprocess.run("pip install -r requirements.txt", shell=True, check=True) + subprocess.run("pip install pyinstaller", shell=True, check=True) + + default_imports = [ + 'aider', + 'aider.resources' + ] + + # Analyze imports after dependencies are installed + missing_imports = analyze_imports(config) + + # Combine default and analyzed imports + all_imports = default_imports + list(missing_imports) + hidden_imports = ' '.join(f'--hidden-import={imp}' for imp in all_imports) + + # Determine executable name based on OS + if sys.platform.startswith('win'): + exe_name = "aider.exe" + else: + exe_name = "aider" + + # Run build command + print(f"Building executable for {sys.platform}...") + arch = os.uname().machine if hasattr(os, 'uname') else platform.machine() + output_path = f"../../{output_dir}/{sys.platform}-{arch}" + os.makedirs(output_path, exist_ok=True) + + build_command = ( + f"pyinstaller --onefile --hidden-import=aider {hidden_imports} " + f"--collect-all aider aider/run.py " + f"--name {exe_name} --distpath {output_path}" + ) + + print(f"Build command: {build_command}") + subprocess.run(build_command, shell=True, check=True) + + os.chdir(original_dir) + +def main(): + config = read_config() + setup_repo(config) + build_project(config) + print("Build complete! Executable should be in out/bin/") + +if __name__ == "__main__": + main() diff --git a/scripts/build_config.json b/scripts/build_config.json new file mode 100644 index 00000000000..c0514a9d358 --- /dev/null +++ b/scripts/build_config.json @@ -0,0 +1,8 @@ +{ + "aider_repo": { + "url": "https://github.com/caseymcc/aider.git", + "branch": "commandio", + "commit": "977387f" + }, + "python_version": "3.12" +} diff --git a/scripts/local_build.ps1 b/scripts/local_build.ps1 new file mode 100644 index 00000000000..7fb4fbc1eaa --- /dev/null +++ b/scripts/local_build.ps1 @@ -0,0 +1,47 @@ +# Exit on error +$ErrorActionPreference = "Stop" + +$CONFIG_FILE = "scripts/build_config.json" + +if (-not (Test-Path $CONFIG_FILE)) { + Write-Error "Error: build_config.json not found" + exit 1 +} + +# Read config file for Python version +$config = Get-Content $CONFIG_FILE | ConvertFrom-Json + +# Check if pyenv-win is installed +$pyenvPath = "$env:USERPROFILE\.pyenv\pyenv-win" +if (-not (Test-Path $pyenvPath)) { + Write-Host "Installing pyenv-win..." + Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/pyenv-win/pyenv-win/master/pyenv-win/install-pyenv-win.ps1" -OutFile "./install-pyenv-win.ps1" + & ./install-pyenv-win.ps1 + Remove-Item ./install-pyenv-win.ps1 + + # Set up environment variables + $env:PYENV = "$env:USERPROFILE\.pyenv\pyenv-win" + $env:Path = "$env:PYENV\bin;$env:PYENV\shims;$env:Path" +} + +# Get Python version from config +$pythonVersion = $config.python_version + +# Install Python version if not present +$installedVersions = & pyenv versions +if ($installedVersions -notcontains $pythonVersion) { + Write-Host "Installing Python $pythonVersion..." + & pyenv install $pythonVersion +} + +# Set local Python version and create venv +Write-Host "Setting up Python virtual environment..." +& pyenv local $pythonVersion +python -m venv venv +. .\venv\Scripts\Activate.ps1 + +# Run the Python build script +python scripts/build.py + +# Deactivate virtual environment +deactivate diff --git a/scripts/local_build.sh b/scripts/local_build.sh new file mode 100755 index 00000000000..50b4d7cc886 --- /dev/null +++ b/scripts/local_build.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Exit on error +set -e + +CONFIG_FILE="scripts/build_config.json" + +if [ ! -f "$CONFIG_FILE" ]; then + echo "Error: build_config.json not found" + exit 1 +fi + +# Setup pyenv +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" + +# Install pyenv if not present +if [ ! -d "$PYENV_ROOT" ]; then + echo "Installing pyenv..." + curl https://pyenv.run | bash +fi + +# Initialize pyenv +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +# Get Python version from config +PYTHON_VERSION=$(jq -r '.python_version' "$CONFIG_FILE") + +# Install Python version if not present +if ! pyenv versions | grep -q $PYTHON_VERSION; then + echo "Installing Python $PYTHON_VERSION..." + pyenv install $PYTHON_VERSION +fi + +# Set local Python version and create venv +echo "Setting up Python virtual environment..." +pyenv local $PYTHON_VERSION +python -m venv venv +source venv/bin/activate + +# Run the Python build script +python scripts/build.py + +# Deactivate virtual environment +deactivate From b07072e66a94c5fb41e595fda8b032f7f059c111 Mon Sep 17 00:00:00 2001 From: Krazer Date: Tue, 31 Dec 2024 12:40:25 -0600 Subject: [PATCH 2/8] add linux exe --- .github/workflows/exe-release.yml | 48 ++++++++++++++++++------- aider/main.py | 2 +- scripts/Dockerfile.cio | 33 ++++++++++++++++++ scripts/build.py | 51 ++++----------------------- scripts/docker_version.py | 14 ++++++++ scripts/local_build.sh | 58 +++++++++++-------------------- 6 files changed, 111 insertions(+), 95 deletions(-) create mode 100644 scripts/Dockerfile.cio create mode 100755 scripts/docker_version.py diff --git a/.github/workflows/exe-release.yml b/.github/workflows/exe-release.yml index ac74623bc34..73ab70fbdbd 100644 --- a/.github/workflows/exe-release.yml +++ b/.github/workflows/exe-release.yml @@ -11,6 +11,7 @@ on: jobs: build: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} @@ -21,26 +22,47 @@ jobs: with: fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 with: - python-version: '3.10' + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Environment (Linux/macOS) - if: runner.os != 'Windows' + - name: Get Docker version + id: docker-version run: | - python -m pip install --upgrade pip - python scripts/local_build.py + echo "version=$(python scripts/docker_version.py)" >> $GITHUB_OUTPUT - - name: Setup Environment (Windows) - if: runner.os == 'Windows' - shell: pwsh + - name: Check if Docker image exists + id: check-image run: | - python -m pip install --upgrade pip - ./scripts/local_build.ps1 + if docker manifest inspect ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }} > /dev/null 2>&1; then + echo "image_exists=true" >> $GITHUB_OUTPUT + else + echo "image_exists=false" >> $GITHUB_OUTPUT + fi + + - name: Build and push Docker image + if: steps.check-image.outputs.image_exists != 'true' + uses: docker/build-push-action@v5 + with: + context: . + file: scripts/Dockerfile.cio + push: true + tags: ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Build Release - run: python scripts/build.py + run: | + docker run --rm \ + -v ${{ github.workspace }}:/repo \ + -v ${{ github.workspace }}/dist:/repo/dist \ + ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }} - name: Upload Artifacts uses: actions/upload-artifact@v3 diff --git a/aider/main.py b/aider/main.py index 0a037da9b20..e9285e4b5a4 100644 --- a/aider/main.py +++ b/aider/main.py @@ -36,7 +36,7 @@ from aider.versioncheck import check_version, install_from_main_branch, install_upgrade from aider.watch import FileWatcher -from .dump import dump # noqa: F401 +from aider.dump import dump # noqa: F401 def check_config_files_for_yes(config_files): diff --git a/scripts/Dockerfile.cio b/scripts/Dockerfile.cio new file mode 100644 index 00000000000..a3ef6793f6f --- /dev/null +++ b/scripts/Dockerfile.cio @@ -0,0 +1,33 @@ +FROM ubuntu:20.04 +LABEL version="0.1.0" + +WORKDIR /app + +# Install system dependencies +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y \ + git \ + build-essential \ + portaudio19-dev \ + python3.12 \ + python3.12-dev \ + curl \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 \ + && rm -rf /var/lib/apt/lists/* + +# Create python command symlink +RUN ln -s /usr/bin/python3.12 /usr/bin/python + +# Copy requirements file +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt pyinstaller + +WORKDIR /repo + +# Set entrypoint to build script +ENTRYPOINT ["python", "scripts/build.py"] diff --git a/scripts/build.py b/scripts/build.py index 0068966c62c..e77b32285de 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -20,30 +20,6 @@ def read_config(): config['repo_path'] = Path("external/aider") return config -def setup_repo(config): - # Create external directory if it doesn't exist - os.makedirs(config['repo_path'].parent, exist_ok=True) - - repo_info = config['aider_repo'] - - # Clone if needed - if not config['repo_path'].exists(): - print("Cloning aider repository...") - subprocess.run(["git", "clone", repo_info['url'], config['repo_path']], check=True) - - # Setup correct branch/commit - original_dir = os.getcwd() - os.chdir(config['repo_path']) - - subprocess.run(["git", "fetch"], check=True) - subprocess.run(["git", "checkout", repo_info['branch']], check=True) - - if repo_info['commit'] != repo_info['branch']: - print("Checking out specific commit...") - subprocess.run(["git", "checkout", "-q", repo_info['commit']], check=True) - - os.chdir(original_dir) - def analyze_imports(config): from importlib.metadata import distributions import importlib @@ -80,7 +56,7 @@ def analyze_imports(config): # Run PyInstaller with debug imports debug_output = subprocess.run( - f"pyinstaller --debug=imports aider/run.py", + f"pyinstaller --debug=imports aider/main.py", shell=True, capture_output=True, text=True @@ -110,21 +86,14 @@ def analyze_imports(config): return missing_imports def build_project(config): - original_dir = os.getcwd() - os.chdir(config['repo_path']) - # Create output directory - output_dir = Path("external/bin") - os.makedirs(f"../../{output_dir}", exist_ok=True) - - # Install dependencies - print("Installing dependencies...") - subprocess.run("pip install -r requirements.txt", shell=True, check=True) - subprocess.run("pip install pyinstaller", shell=True, check=True) + output_dir = Path("dist") + os.makedirs(output_dir, exist_ok=True) default_imports = [ 'aider', - 'aider.resources' + 'aider.resources', + 'importlib_resources' ] # Analyze imports after dependencies are installed @@ -143,25 +112,19 @@ def build_project(config): # Run build command print(f"Building executable for {sys.platform}...") arch = os.uname().machine if hasattr(os, 'uname') else platform.machine() - output_path = f"../../{output_dir}/{sys.platform}-{arch}" - os.makedirs(output_path, exist_ok=True) build_command = ( f"pyinstaller --onefile --hidden-import=aider {hidden_imports} " - f"--collect-all aider aider/run.py " - f"--name {exe_name} --distpath {output_path}" + f"--collect-all aider aider/main.py " + f"--name {exe_name}" ) print(f"Build command: {build_command}") subprocess.run(build_command, shell=True, check=True) - - os.chdir(original_dir) def main(): config = read_config() - setup_repo(config) build_project(config) - print("Build complete! Executable should be in out/bin/") if __name__ == "__main__": main() diff --git a/scripts/docker_version.py b/scripts/docker_version.py new file mode 100755 index 00000000000..5769f5eba78 --- /dev/null +++ b/scripts/docker_version.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +def get_docker_version(): + """Extract version from Dockerfile.""" + with open("scripts/Dockerfile.cio", "r") as f: + for line in f: + if "LABEL version=" in line: + return line.split("=")[1].strip().strip('"') + return None + +if __name__ == "__main__": + version = get_docker_version() + if version: + print(version) diff --git a/scripts/local_build.sh b/scripts/local_build.sh index 50b4d7cc886..ac680f91e7a 100755 --- a/scripts/local_build.sh +++ b/scripts/local_build.sh @@ -3,44 +3,28 @@ # Exit on error set -e -CONFIG_FILE="scripts/build_config.json" - -if [ ! -f "$CONFIG_FILE" ]; then - echo "Error: build_config.json not found" - exit 1 -fi - -# Setup pyenv -export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$PATH" - -# Install pyenv if not present -if [ ! -d "$PYENV_ROOT" ]; then - echo "Installing pyenv..." - curl https://pyenv.run | bash -fi - -# Initialize pyenv -eval "$(pyenv init -)" -eval "$(pyenv virtualenv-init -)" - -# Get Python version from config -PYTHON_VERSION=$(jq -r '.python_version' "$CONFIG_FILE") - -# Install Python version if not present -if ! pyenv versions | grep -q $PYTHON_VERSION; then - echo "Installing Python $PYTHON_VERSION..." - pyenv install $PYTHON_VERSION +# Get version from Dockerfile +DOCKER_VERSION=$(python3 scripts/docker_version.py) +DOCKER_IMAGE="ghcr.io/caseymcc/aider/builder:${DOCKER_VERSION}" + +echo "Checking for Docker image: ${DOCKER_IMAGE}" + +# Try to pull the image from GitHub Container Registry +if docker pull ${DOCKER_IMAGE} >/dev/null 2>&1; then + echo "Found existing Docker image, using it for build" +else + echo "Building Docker image locally..." + docker build -t ${DOCKER_IMAGE} -f scripts/Dockerfile.cio . fi -# Set local Python version and create venv -echo "Setting up Python virtual environment..." -pyenv local $PYTHON_VERSION -python -m venv venv -source venv/bin/activate +# Create dist directory if it doesn't exist +mkdir -p dist -# Run the Python build script -python scripts/build.py +# Run the build in Docker +echo "Running build in Docker container..." +docker run --rm \ + -v "$(pwd):/repo" \ + -v "$(pwd)/dist:/repo/dist" \ + ${DOCKER_IMAGE} -# Deactivate virtual environment -deactivate +echo "Build complete! Executable should be in dist/" From 2cce2056750cbb242ea2f7d381504cee9d9e2a0b Mon Sep 17 00:00:00 2001 From: "Krazer (aider)" Date: Tue, 31 Dec 2024 12:43:21 -0600 Subject: [PATCH 3/8] feat: Add Docker-based Windows build support with Dockerfile and updated build script --- scripts/Dockerfile.windows.cio | 30 +++++++++++++++++ scripts/local_build.ps1 | 59 +++++++++++++--------------------- 2 files changed, 52 insertions(+), 37 deletions(-) create mode 100644 scripts/Dockerfile.windows.cio diff --git a/scripts/Dockerfile.windows.cio b/scripts/Dockerfile.windows.cio new file mode 100644 index 00000000000..11eb28d4a64 --- /dev/null +++ b/scripts/Dockerfile.windows.cio @@ -0,0 +1,30 @@ +# escape=` +FROM mcr.microsoft.com/windows/servercore:ltsc2019 + +# Set working directory +WORKDIR C:/app + +# Install Chocolatey +RUN powershell -Command ` + Set-ExecutionPolicy Bypass -Scope Process -Force; ` + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; ` + iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) + +# Install Python and Git +RUN choco install -y python --version=3.12.0 git + +# Refresh environment variables +RUN refreshenv + +# Add Python to PATH +RUN setx /M PATH "%PATH%;C:\Python312;C:\Python312\Scripts" + +# Install pip requirements +COPY requirements.txt . +RUN python -m pip install --no-cache-dir -r requirements.txt pyinstaller + +# Set working directory for the build +WORKDIR C:/repo + +# Set entrypoint +ENTRYPOINT ["python", "scripts/build.py"] diff --git a/scripts/local_build.ps1 b/scripts/local_build.ps1 index 7fb4fbc1eaa..3922038f7e8 100644 --- a/scripts/local_build.ps1 +++ b/scripts/local_build.ps1 @@ -1,47 +1,32 @@ # Exit on error $ErrorActionPreference = "Stop" -$CONFIG_FILE = "scripts/build_config.json" +# Get version from Dockerfile +$DOCKER_VERSION = python scripts/docker_version.py +$DOCKER_IMAGE = "ghcr.io/caseymcc/aider/builder:${DOCKER_VERSION}" -if (-not (Test-Path $CONFIG_FILE)) { - Write-Error "Error: build_config.json not found" - exit 1 -} - -# Read config file for Python version -$config = Get-Content $CONFIG_FILE | ConvertFrom-Json +Write-Host "Checking for Docker image: ${DOCKER_IMAGE}" -# Check if pyenv-win is installed -$pyenvPath = "$env:USERPROFILE\.pyenv\pyenv-win" -if (-not (Test-Path $pyenvPath)) { - Write-Host "Installing pyenv-win..." - Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/pyenv-win/pyenv-win/master/pyenv-win/install-pyenv-win.ps1" -OutFile "./install-pyenv-win.ps1" - & ./install-pyenv-win.ps1 - Remove-Item ./install-pyenv-win.ps1 - - # Set up environment variables - $env:PYENV = "$env:USERPROFILE\.pyenv\pyenv-win" - $env:Path = "$env:PYENV\bin;$env:PYENV\shims;$env:Path" +# Try to pull the image from GitHub Container Registry +try { + docker pull ${DOCKER_IMAGE} 2>$null + Write-Host "Found existing Docker image, using it for build" } - -# Get Python version from config -$pythonVersion = $config.python_version - -# Install Python version if not present -$installedVersions = & pyenv versions -if ($installedVersions -notcontains $pythonVersion) { - Write-Host "Installing Python $pythonVersion..." - & pyenv install $pythonVersion +catch { + Write-Host "Building Docker image locally..." + docker build -t ${DOCKER_IMAGE} -f scripts/Dockerfile.windows.cio . } -# Set local Python version and create venv -Write-Host "Setting up Python virtual environment..." -& pyenv local $pythonVersion -python -m venv venv -. .\venv\Scripts\Activate.ps1 +# Create dist directory if it doesn't exist +if (-not (Test-Path dist)) { + New-Item -ItemType Directory -Path dist +} -# Run the Python build script -python scripts/build.py +# Run the build in Docker +Write-Host "Running build in Docker container..." +docker run --rm ` + -v "${PWD}:/repo" ` + -v "${PWD}/dist:/repo/dist" ` + ${DOCKER_IMAGE} -# Deactivate virtual environment -deactivate +Write-Host "Build complete! Executable should be in dist/" From fb4a305fa632b660fa2ff07493f388e1a2dd80d1 Mon Sep 17 00:00:00 2001 From: "Krazer (aider)" Date: Tue, 31 Dec 2024 12:52:01 -0600 Subject: [PATCH 4/8] ci: Use Windows-specific Dockerfile when running on Windows runner --- .github/workflows/exe-release.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/exe-release.yml b/.github/workflows/exe-release.yml index 73ab70fbdbd..15d58808f22 100644 --- a/.github/workflows/exe-release.yml +++ b/.github/workflows/exe-release.yml @@ -51,14 +51,20 @@ jobs: uses: docker/build-push-action@v5 with: context: . - file: scripts/Dockerfile.cio + file: ${{ runner.os == 'Windows' && 'scripts/Dockerfile.windows.cio' || 'scripts/Dockerfile.cio' }} push: true tags: ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }} cache-from: type=gha cache-to: type=gha,mode=max - name: Build Release + shell: bash run: | + if [ "${{ runner.os }}" = "Windows" ]; then + DOCKERFILE="scripts/Dockerfile.windows.cio" + else + DOCKERFILE="scripts/Dockerfile.cio" + fi docker run --rm \ -v ${{ github.workspace }}:/repo \ -v ${{ github.workspace }}/dist:/repo/dist \ From 3fb14837af1372c3ac99feab48fdef32d6f459d3 Mon Sep 17 00:00:00 2001 From: "Krazer (aider)" Date: Tue, 31 Dec 2024 12:55:19 -0600 Subject: [PATCH 5/8] feat: Add OS-specific Docker image tags for multi-platform builds --- .github/workflows/exe-release.yml | 4 ++-- scripts/Dockerfile.windows.cio | 1 + scripts/docker_version.py | 16 ++++++++++++++-- scripts/local_build.ps1 | 2 +- scripts/local_build.sh | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/exe-release.yml b/.github/workflows/exe-release.yml index 15d58808f22..3e239e9bcbb 100644 --- a/.github/workflows/exe-release.yml +++ b/.github/workflows/exe-release.yml @@ -53,7 +53,7 @@ jobs: context: . file: ${{ runner.os == 'Windows' && 'scripts/Dockerfile.windows.cio' || 'scripts/Dockerfile.cio' }} push: true - tags: ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }} + tags: ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }}-${{ runner.os }} cache-from: type=gha cache-to: type=gha,mode=max @@ -68,7 +68,7 @@ jobs: docker run --rm \ -v ${{ github.workspace }}:/repo \ -v ${{ github.workspace }}/dist:/repo/dist \ - ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }} + ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }}-${{ runner.os }} - name: Upload Artifacts uses: actions/upload-artifact@v3 diff --git a/scripts/Dockerfile.windows.cio b/scripts/Dockerfile.windows.cio index 11eb28d4a64..8169c131884 100644 --- a/scripts/Dockerfile.windows.cio +++ b/scripts/Dockerfile.windows.cio @@ -1,5 +1,6 @@ # escape=` FROM mcr.microsoft.com/windows/servercore:ltsc2019 +LABEL version="0.1.0" # Set working directory WORKDIR C:/app diff --git a/scripts/docker_version.py b/scripts/docker_version.py index 5769f5eba78..3d9f73f95d6 100755 --- a/scripts/docker_version.py +++ b/scripts/docker_version.py @@ -1,14 +1,26 @@ #!/usr/bin/env python3 +import platform +import sys + def get_docker_version(): """Extract version from Dockerfile.""" - with open("scripts/Dockerfile.cio", "r") as f: + dockerfile = "scripts/Dockerfile.windows.cio" if sys.platform.startswith('win') else "scripts/Dockerfile.cio" + with open(dockerfile, "r") as f: for line in f: if "LABEL version=" in line: return line.split("=")[1].strip().strip('"') return None +def get_os_suffix(): + if sys.platform.startswith('win'): + return "windows" + elif sys.platform.startswith('darwin'): + return "macos" + else: + return "linux" + if __name__ == "__main__": version = get_docker_version() if version: - print(version) + print(f"{version}-{get_os_suffix()}") diff --git a/scripts/local_build.ps1 b/scripts/local_build.ps1 index 3922038f7e8..888dc8a9c7b 100644 --- a/scripts/local_build.ps1 +++ b/scripts/local_build.ps1 @@ -1,7 +1,7 @@ # Exit on error $ErrorActionPreference = "Stop" -# Get version from Dockerfile +# Get version from Dockerfile (includes OS suffix) $DOCKER_VERSION = python scripts/docker_version.py $DOCKER_IMAGE = "ghcr.io/caseymcc/aider/builder:${DOCKER_VERSION}" diff --git a/scripts/local_build.sh b/scripts/local_build.sh index ac680f91e7a..f61ce7ba774 100755 --- a/scripts/local_build.sh +++ b/scripts/local_build.sh @@ -3,7 +3,7 @@ # Exit on error set -e -# Get version from Dockerfile +# Get version from Dockerfile (includes OS suffix) DOCKER_VERSION=$(python3 scripts/docker_version.py) DOCKER_IMAGE="ghcr.io/caseymcc/aider/builder:${DOCKER_VERSION}" From 7089657dff42b1b17a2e7a63dbd312475153a915 Mon Sep 17 00:00:00 2001 From: "Krazer (aider)" Date: Tue, 31 Dec 2024 12:56:17 -0600 Subject: [PATCH 6/8] refactor: Rename Dockerfile.cio to Dockerfile.linux.cio and update references --- .github/workflows/exe-release.yml | 2 +- scripts/docker_version.py | 2 +- scripts/local_build.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/exe-release.yml b/.github/workflows/exe-release.yml index 3e239e9bcbb..f8af4138772 100644 --- a/.github/workflows/exe-release.yml +++ b/.github/workflows/exe-release.yml @@ -51,7 +51,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - file: ${{ runner.os == 'Windows' && 'scripts/Dockerfile.windows.cio' || 'scripts/Dockerfile.cio' }} + file: ${{ runner.os == 'Windows' && 'scripts/Dockerfile.windows.cio' || 'scripts/Dockerfile.linux.cio' }} push: true tags: ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }}-${{ runner.os }} cache-from: type=gha diff --git a/scripts/docker_version.py b/scripts/docker_version.py index 3d9f73f95d6..e11232e9909 100755 --- a/scripts/docker_version.py +++ b/scripts/docker_version.py @@ -5,7 +5,7 @@ def get_docker_version(): """Extract version from Dockerfile.""" - dockerfile = "scripts/Dockerfile.windows.cio" if sys.platform.startswith('win') else "scripts/Dockerfile.cio" + dockerfile = "scripts/Dockerfile.windows.cio" if sys.platform.startswith('win') else "scripts/Dockerfile.linux.cio" with open(dockerfile, "r") as f: for line in f: if "LABEL version=" in line: diff --git a/scripts/local_build.sh b/scripts/local_build.sh index f61ce7ba774..fa1e6ff1cf4 100755 --- a/scripts/local_build.sh +++ b/scripts/local_build.sh @@ -14,7 +14,7 @@ if docker pull ${DOCKER_IMAGE} >/dev/null 2>&1; then echo "Found existing Docker image, using it for build" else echo "Building Docker image locally..." - docker build -t ${DOCKER_IMAGE} -f scripts/Dockerfile.cio . + docker build -t ${DOCKER_IMAGE} -f scripts/Dockerfile.linux.cio . fi # Create dist directory if it doesn't exist From 6dc1d422c2e9093acb2c788fd355d542aa5bb9cf Mon Sep 17 00:00:00 2001 From: Krazer Date: Tue, 31 Dec 2024 12:57:32 -0600 Subject: [PATCH 7/8] rename docker file --- scripts/{Dockerfile.cio => Dockerfile.linux.cio} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{Dockerfile.cio => Dockerfile.linux.cio} (100%) diff --git a/scripts/Dockerfile.cio b/scripts/Dockerfile.linux.cio similarity index 100% rename from scripts/Dockerfile.cio rename to scripts/Dockerfile.linux.cio From e95acd392b4bbeffd0e208dd39aba8129d75c688 Mon Sep 17 00:00:00 2001 From: "Krazer (aider)" Date: Tue, 31 Dec 2024 13:01:32 -0600 Subject: [PATCH 8/8] ci: Fix Docker Buildx setup for Windows in GitHub Actions workflow --- .github/workflows/exe-release.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/exe-release.yml b/.github/workflows/exe-release.yml index f8af4138772..2b7af5b3430 100644 --- a/.github/workflows/exe-release.yml +++ b/.github/workflows/exe-release.yml @@ -23,6 +23,7 @@ jobs: fetch-depth: 0 - name: Set up Docker Buildx + if: runner.os != 'Windows' uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry @@ -36,27 +37,36 @@ jobs: id: docker-version run: | echo "version=$(python scripts/docker_version.py)" >> $GITHUB_OUTPUT + shell: bash - name: Check if Docker image exists id: check-image run: | - if docker manifest inspect ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }} > /dev/null 2>&1; then + if docker manifest inspect ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }}-${{ runner.os }} > /dev/null 2>&1; then echo "image_exists=true" >> $GITHUB_OUTPUT else echo "image_exists=false" >> $GITHUB_OUTPUT fi + shell: bash - - name: Build and push Docker image - if: steps.check-image.outputs.image_exists != 'true' + - name: Build and push Docker image (non-Windows) + if: steps.check-image.outputs.image_exists != 'true' && runner.os != 'Windows' uses: docker/build-push-action@v5 with: context: . - file: ${{ runner.os == 'Windows' && 'scripts/Dockerfile.windows.cio' || 'scripts/Dockerfile.linux.cio' }} + file: scripts/Dockerfile.linux.cio push: true tags: ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }}-${{ runner.os }} cache-from: type=gha cache-to: type=gha,mode=max + - name: Build and push Docker image (Windows) + if: steps.check-image.outputs.image_exists != 'true' && runner.os == 'Windows' + shell: bash + run: | + docker build -t ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }}-${{ runner.os }} -f scripts/Dockerfile.windows.cio . + docker push ghcr.io/${{ github.repository }}/builder:${{ steps.docker-version.outputs.version }}-${{ runner.os }} + - name: Build Release shell: bash run: |