pip (#6)

example_envs/env_with_pip.yaml

@@ -0,0 +1,9 @@
+name: xeus-lite
+channels:
+  - https://repo.mamba.pm/emscripten-forge 
+  - conda-forge
+dependencies:
+  - xeus-python 
+  - xeus-sqlite
+  - pip:
+    - python-random-name-generator

jupyterlite_xeus/_pip.py

@@ -0,0 +1,100 @@
+import sys
+import shutil
+import os
+from subprocess import run as subprocess_run
+from tempfile import TemporaryDirectory
+from pathlib import Path
+import csv
+from .constants import PYTHON_VERSION
+
+def _install_pip_dependencies(
+    prefix_path, 
+    dependencies, 
+    log=None
+):
+    # Why is this so damn complicated?
+    # Isn't it easier to download the .whl ourselves? pip is hell
+
+    if log is not None:
+        log.warning(
+            """
+            Installing pip dependencies. This is very much experimental so use
+            this feature at your own risks.
+            Note that you can only install pure-python packages.
+            pip is being run with the --no-deps option to not pull undesired
+            system-specific dependencies, so please install your package dependencies
+            from emscripten-forge or conda-forge.
+            """
+        )
+
+    # Installing with pip in another prefix that has a different Python version IS NOT POSSIBLE
+    # So we need to do this whole mess "manually"
+    pkg_dir = TemporaryDirectory()
+
+    subprocess_run(
+        [
+            sys.executable,
+            "-m",
+            "pip",
+            "install",
+            *dependencies,
+            # Install in a tmp directory while we process it
+            "--target",
+            pkg_dir.name,
+            # Specify the right Python version
+            "--python-version",
+            PYTHON_VERSION,
+            # No dependency installed
+            "--no-deps",
+            "--no-input",
+            "--verbose",
+        ],
+        check=True,
+    )
+
+    # We need to read the RECORD and try to be smart about what goes
+    # under site-packages and what goes where
+    packages_dist_info = Path(pkg_dir.name).glob("*.dist-info")
+
+    for package_dist_info in packages_dist_info:
+        with open(package_dist_info / "RECORD") as record:
+            record_content = record.read()
+            record_csv = csv.reader(record_content.splitlines())
+            all_files = [_file[0] for _file in record_csv]
+
+            # List of tuples: (path: str, inside_site_packages: bool)
+            files = [(_file, not _file.startswith("../../")) for _file in all_files]
+
+            # Why?
+            fixed_record_data = record_content.replace("../../", "../../../")
+
+        # OVERWRITE RECORD file
+        with open(package_dist_info / "RECORD", "w") as record:
+            record.write(fixed_record_data)
+
+        non_supported_files = [".so", ".a", ".dylib", ".lib", ".exe.dll"]
+
+        # COPY files under `prefix_path`
+        for _file, inside_site_packages in files:
+            path = Path(_file)
+
+            # FAIL if .so / .a / .dylib / .lib / .exe / .dll
+            if path.suffix in non_supported_files:
+                raise RuntimeError(
+                    "Cannot install binary PyPI package, only pure Python packages are supported"
+                )
+
+            file_path = _file[6:] if not inside_site_packages else _file
+            install_path = (
+                prefix_path
+                if not inside_site_packages
+                else prefix_path / "lib" / f"python{PYTHON_VERSION}" / "site-packages"
+            )
+
+            src_path = Path(pkg_dir.name) / file_path
+            dest_path = install_path / file_path
+
+            os.makedirs(dest_path.parent, exist_ok=True)
+
+            shutil.copy(src_path, dest_path)
+

jupyterlite_xeus/add_on.py

@@ -17,9 +17,7 @@
 
 from .prefix_bundler import get_prefix_bundler
 from .create_conda_env import create_conda_env_from_yaml,create_conda_env_from_specs
-
-EXTENSION_NAME = "xeus"
-STATIC_DIR = Path("@jupyterlite") / EXTENSION_NAME / "static"
+from .constants import EXTENSION_NAME, STATIC_DIR
 
 
 def get_kernel_binaries(path):
@@ -109,13 +107,6 @@ def create_prefix(self):
                 channels=["conda-forge", "https://repo.mamba.pm/emscripten-forge"],
             )
 
-
-
-
-
-
-
-
     def copy_kernels_from_prefix(self):
 
         if not os.path.exists(self.prefix) or not os.path.isdir(self.prefix):
@@ -147,8 +138,6 @@ def copy_kernels_from_prefix(self):
         )
 
 
-
-
     def copy_kernel(self, kernel_dir, kernel_wasm, kernel_js):
 
         kernel_spec = json.loads((kernel_dir / "kernel.json").read_text(**UTF8))
@@ -228,13 +217,8 @@ def copy_jupyterlab_extensions_from_prefix(self, manager):
         # Find the federated extensions in the emscripten-env and install them
         prefix = Path(self.prefix)
         for pkg_json in self.env_extensions(prefix / SHARE_LABEXTENSIONS):
-            print("pkg_json", pkg_json)
             yield from self.safe_copy_jupyterlab_extension(pkg_json)
 
-        yield from self.register_jupyterlab_extension(manager)
-
-    def register_jupyterlab_extension(self, manager):
-
         jupyterlite_json = manager.output_dir / JUPYTERLITE_JSON
         lab_extensions_root = manager.output_dir / LAB_EXTENSIONS
         lab_extensions = self.env_extensions(lab_extensions_root)

jupyterlite_xeus/constants.py

@@ -0,0 +1,10 @@
+from pathlib import Path
+
+
+EXTENSION_NAME = "xeus"
+STATIC_DIR = Path("@jupyterlite") / EXTENSION_NAME / "static"
+
+
+PYTHON_MAJOR = 3
+PYTHON_MINOR = 11
+PYTHON_VERSION = f"{PYTHON_MAJOR}.{PYTHON_MINOR}"

jupyterlite_xeus/create_conda_env.py

@@ -4,6 +4,8 @@
 from subprocess import run as subprocess_run
 import os
 import yaml
+
+from ._pip import _install_pip_dependencies
 try:
     from mamba.api import create as mamba_create
     MAMBA_PYTHON_AVAILABLE = True
@@ -16,6 +18,26 @@
 PLATFORM = "emscripten-wasm32"
 
 
+
+def _extract_specs(env_location, env_data):
+
+    specs = []
+    pip_dependencies = []
+
+    # iterate dependencies
+    for dependency in env_data.get("dependencies", []):
+        if isinstance(dependency, str):
+           specs.append(dependency)
+        elif isinstance(dependency, dict) and "pip" in dependency:
+            for pip_dependency in dependency["pip"]:
+                # If it's a local Python package, make its path relative to the environment file
+                if (env_location / pip_dependency).is_dir():
+                    pip_dependencies.append(env_location.parent / pip_dependency).resolve()
+                else:
+                    pip_dependencies.append(pip_dependency)
+
+    return specs, pip_dependencies
+
 def create_conda_env_from_yaml(
     env_name,
     root_prefix,
@@ -27,30 +49,55 @@ def create_conda_env_from_yaml(
 
     # get the channels
     channels = yaml_content.get("channels", [])
+
     # get the specs
-    specs = yaml_content.get("dependencies", [])
+    specs, pip_dependencies = _extract_specs(env_file.parent, yaml_content)
 
     create_conda_env_from_specs(
         env_name=env_name,
         root_prefix=root_prefix,
         specs=specs,
         channels=channels,
+        pip_dependencies=pip_dependencies
     )
 
 
+def create_conda_env_from_specs(
+    env_name,
+    root_prefix,
+    specs,
+    channels,
+    pip_dependencies=None,
+):
+    _create_conda_env_from_specs_impl(
+        env_name=env_name,
+        root_prefix=root_prefix,
+        specs=specs,
+        channels=channels,
+    )   
+    if pip_dependencies:
+        _install_pip_dependencies(
+            prefix_path=Path(root_prefix) / "envs" / env_name,
+            dependencies=pip_dependencies
+        )
 
 
 
-
-def create_conda_env_from_specs(
+def _create_conda_env_from_specs_impl(
     env_name,
     root_prefix,
     specs,
-    channels,
+    channels
 ):
     """Create the emscripten environment with the given specs."""
     prefix_path = Path(root_prefix) / "envs" / env_name
 
+
+    # Cleanup tmp dir in case it's not empty
+    shutil.rmtree(Path(root_prefix) / "envs", ignore_errors=True)
+    Path(root_prefix).mkdir(parents=True, exist_ok=True)
+
+
     if MAMBA_PYTHON_AVAILABLE:
         mamba_create(
             env_name=env_name,