From 750f3dd444e17a378a87614d1aa4850b3efb088a Mon Sep 17 00:00:00 2001 From: Thorsten Beier Date: Wed, 13 Dec 2023 11:34:12 +0100 Subject: [PATCH] pip (#6) --- example_envs/env_with_pip.yaml | 9 +++ jupyterlite_xeus/_pip.py | 100 +++++++++++++++++++++++++++ jupyterlite_xeus/add_on.py | 18 +---- jupyterlite_xeus/constants.py | 10 +++ jupyterlite_xeus/create_conda_env.py | 55 +++++++++++++-- 5 files changed, 171 insertions(+), 21 deletions(-) create mode 100644 example_envs/env_with_pip.yaml create mode 100644 jupyterlite_xeus/_pip.py create mode 100644 jupyterlite_xeus/constants.py diff --git a/example_envs/env_with_pip.yaml b/example_envs/env_with_pip.yaml new file mode 100644 index 0000000..0b2782f --- /dev/null +++ b/example_envs/env_with_pip.yaml @@ -0,0 +1,9 @@ +name: xeus-lite +channels: + - https://repo.mamba.pm/emscripten-forge + - conda-forge +dependencies: + - xeus-python + - xeus-sqlite + - pip: + - python-random-name-generator \ No newline at end of file diff --git a/jupyterlite_xeus/_pip.py b/jupyterlite_xeus/_pip.py new file mode 100644 index 0000000..cc92d4a --- /dev/null +++ b/jupyterlite_xeus/_pip.py @@ -0,0 +1,100 @@ +import sys +import shutil +import os +from subprocess import run as subprocess_run +from tempfile import TemporaryDirectory +from pathlib import Path +import csv +from .constants import PYTHON_VERSION + +def _install_pip_dependencies( + prefix_path, + dependencies, + log=None +): + # Why is this so damn complicated? + # Isn't it easier to download the .whl ourselves? pip is hell + + if log is not None: + log.warning( + """ + Installing pip dependencies. This is very much experimental so use + this feature at your own risks. + Note that you can only install pure-python packages. + pip is being run with the --no-deps option to not pull undesired + system-specific dependencies, so please install your package dependencies + from emscripten-forge or conda-forge. + """ + ) + + # Installing with pip in another prefix that has a different Python version IS NOT POSSIBLE + # So we need to do this whole mess "manually" + pkg_dir = TemporaryDirectory() + + subprocess_run( + [ + sys.executable, + "-m", + "pip", + "install", + *dependencies, + # Install in a tmp directory while we process it + "--target", + pkg_dir.name, + # Specify the right Python version + "--python-version", + PYTHON_VERSION, + # No dependency installed + "--no-deps", + "--no-input", + "--verbose", + ], + check=True, + ) + + # We need to read the RECORD and try to be smart about what goes + # under site-packages and what goes where + packages_dist_info = Path(pkg_dir.name).glob("*.dist-info") + + for package_dist_info in packages_dist_info: + with open(package_dist_info / "RECORD") as record: + record_content = record.read() + record_csv = csv.reader(record_content.splitlines()) + all_files = [_file[0] for _file in record_csv] + + # List of tuples: (path: str, inside_site_packages: bool) + files = [(_file, not _file.startswith("../../")) for _file in all_files] + + # Why? + fixed_record_data = record_content.replace("../../", "../../../") + + # OVERWRITE RECORD file + with open(package_dist_info / "RECORD", "w") as record: + record.write(fixed_record_data) + + non_supported_files = [".so", ".a", ".dylib", ".lib", ".exe.dll"] + + # COPY files under `prefix_path` + for _file, inside_site_packages in files: + path = Path(_file) + + # FAIL if .so / .a / .dylib / .lib / .exe / .dll + if path.suffix in non_supported_files: + raise RuntimeError( + "Cannot install binary PyPI package, only pure Python packages are supported" + ) + + file_path = _file[6:] if not inside_site_packages else _file + install_path = ( + prefix_path + if not inside_site_packages + else prefix_path / "lib" / f"python{PYTHON_VERSION}" / "site-packages" + ) + + src_path = Path(pkg_dir.name) / file_path + dest_path = install_path / file_path + + os.makedirs(dest_path.parent, exist_ok=True) + + shutil.copy(src_path, dest_path) + diff --git a/jupyterlite_xeus/add_on.py b/jupyterlite_xeus/add_on.py index 23a35cb..8bb5c15 100644 --- a/jupyterlite_xeus/add_on.py +++ b/jupyterlite_xeus/add_on.py @@ -17,9 +17,7 @@ from .prefix_bundler import get_prefix_bundler from .create_conda_env import create_conda_env_from_yaml,create_conda_env_from_specs - -EXTENSION_NAME = "xeus" -STATIC_DIR = Path("@jupyterlite") / EXTENSION_NAME / "static" +from .constants import EXTENSION_NAME, STATIC_DIR def get_kernel_binaries(path): @@ -109,13 +107,6 @@ def create_prefix(self): channels=["conda-forge", "https://repo.mamba.pm/emscripten-forge"], ) - - - - - - - def copy_kernels_from_prefix(self): if not os.path.exists(self.prefix) or not os.path.isdir(self.prefix): @@ -147,8 +138,6 @@ def copy_kernels_from_prefix(self): ) - - def copy_kernel(self, kernel_dir, kernel_wasm, kernel_js): kernel_spec = json.loads((kernel_dir / "kernel.json").read_text(**UTF8)) @@ -228,13 +217,8 @@ def copy_jupyterlab_extensions_from_prefix(self, manager): # Find the federated extensions in the emscripten-env and install them prefix = Path(self.prefix) for pkg_json in self.env_extensions(prefix / SHARE_LABEXTENSIONS): - print("pkg_json", pkg_json) yield from self.safe_copy_jupyterlab_extension(pkg_json) - yield from self.register_jupyterlab_extension(manager) - - def register_jupyterlab_extension(self, manager): - jupyterlite_json = manager.output_dir / JUPYTERLITE_JSON lab_extensions_root = manager.output_dir / LAB_EXTENSIONS lab_extensions = self.env_extensions(lab_extensions_root) diff --git a/jupyterlite_xeus/constants.py b/jupyterlite_xeus/constants.py new file mode 100644 index 0000000..d1ddd3c --- /dev/null +++ b/jupyterlite_xeus/constants.py @@ -0,0 +1,10 @@ +from pathlib import Path + + +EXTENSION_NAME = "xeus" +STATIC_DIR = Path("@jupyterlite") / EXTENSION_NAME / "static" + + +PYTHON_MAJOR = 3 +PYTHON_MINOR = 11 +PYTHON_VERSION = f"{PYTHON_MAJOR}.{PYTHON_MINOR}" diff --git a/jupyterlite_xeus/create_conda_env.py b/jupyterlite_xeus/create_conda_env.py index 31203d4..a4461e8 100644 --- a/jupyterlite_xeus/create_conda_env.py +++ b/jupyterlite_xeus/create_conda_env.py @@ -4,6 +4,8 @@ from subprocess import run as subprocess_run import os import yaml + +from ._pip import _install_pip_dependencies try: from mamba.api import create as mamba_create MAMBA_PYTHON_AVAILABLE = True @@ -16,6 +18,26 @@ PLATFORM = "emscripten-wasm32" + +def _extract_specs(env_location, env_data): + + specs = [] + pip_dependencies = [] + + # iterate dependencies + for dependency in env_data.get("dependencies", []): + if isinstance(dependency, str): + specs.append(dependency) + elif isinstance(dependency, dict) and "pip" in dependency: + for pip_dependency in dependency["pip"]: + # If it's a local Python package, make its path relative to the environment file + if (env_location / pip_dependency).is_dir(): + pip_dependencies.append(env_location.parent / pip_dependency).resolve() + else: + pip_dependencies.append(pip_dependency) + + return specs, pip_dependencies + def create_conda_env_from_yaml( env_name, root_prefix, @@ -27,30 +49,55 @@ def create_conda_env_from_yaml( # get the channels channels = yaml_content.get("channels", []) + # get the specs - specs = yaml_content.get("dependencies", []) + specs, pip_dependencies = _extract_specs(env_file.parent, yaml_content) create_conda_env_from_specs( env_name=env_name, root_prefix=root_prefix, specs=specs, channels=channels, + pip_dependencies=pip_dependencies ) +def create_conda_env_from_specs( + env_name, + root_prefix, + specs, + channels, + pip_dependencies=None, +): + _create_conda_env_from_specs_impl( + env_name=env_name, + root_prefix=root_prefix, + specs=specs, + channels=channels, + ) + if pip_dependencies: + _install_pip_dependencies( + prefix_path=Path(root_prefix) / "envs" / env_name, + dependencies=pip_dependencies + ) - -def create_conda_env_from_specs( +def _create_conda_env_from_specs_impl( env_name, root_prefix, specs, - channels, + channels ): """Create the emscripten environment with the given specs.""" prefix_path = Path(root_prefix) / "envs" / env_name + + # Cleanup tmp dir in case it's not empty + shutil.rmtree(Path(root_prefix) / "envs", ignore_errors=True) + Path(root_prefix).mkdir(parents=True, exist_ok=True) + + if MAMBA_PYTHON_AVAILABLE: mamba_create( env_name=env_name,