fix: support relative path dependencies from within crates too

refactor: refactored examples
docs: updated README.md

README.md

@@ -277,7 +277,7 @@ And if this is a common occurrence, I would love to hear your use case and why t
 ### Can I use something else than `pyo3`?
 Sure! Though I recommend using `pyo3` due to it's simplicity, you're completely free to use any other library, for example [`rust-cpython`](https://github.com/dgrunwald/rust-cpython).
 
-There is an example using `rust-cpython` in [examples/doublecount.rs](./examples/doublecount.rs)
+There is an example using `rust-cpython` in [examples/cpython_doublecount.rs](./examples/cpython_doublecount.rs)
 
 ### How can I make compilation faster? 
 

examples/doublecount.rs → examples/cpython_doublecount.rs

@@ -1,12 +1,18 @@
 // rustimport
 
+// Here, we use `rust-cpython` instead of pyO3:
+//  - Notice how we do not have "rustimport:pyo3" in the first line of this
+//    file (-> we don't activate the pyO3 template)
+//  - Below, you find the full manifest (Cargo.toml) definition, which is usually
+//    automatically defined by pyO3
+
 //: [package]
-//: name = "doublecount"
+//: name = "cpython_doublecount"
 //: version = "0.1.0"
 //: authors = ["Bruno Rocha <[email protected]>"]
 //:
 //: [lib]
-//: name = "myrustlib"
+//: name = "cpython_doublecount"
 //: crate-type = ["cdylib"]
 //:
 //: [dependencies.cpython]
@@ -34,7 +40,7 @@ fn count_doubles(_py: Python, val: &str) -> PyResult<u64> {
     Ok(total)
 }
 
-py_module_initializer!(doublecount, |py, m | {
+py_module_initializer!(cpython_doublecount, |py, m | {
     m.add(py, "__doc__", "This module is implemented in Rust.")?;
     m.add(py, "count_doubles", py_fn!(py, count_doubles(val: &str)))?;
     Ok(())

examples/crate_relative_path_dependency/.rustimport

@@ -0,0 +1 @@
+This is a marker-file to make this crate importable by rustimport.

examples/crate_relative_path_dependency/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "crate_relative_path_dependency"
+version = "0.1.0"
+edition = "2021"
+
+
+# You can safely remove the code below to let rustimport define your pyo3 configuration 
+# automatically. It's still possible to add other configuration or dependencies, or overwrite 
+# specific parts here. rustimport will merge your Cargo.toml file into it's generated default
+# configuration.
+[lib]
+# The name of the native library. This is the name which will be used in Python to import the
+# library (i.e. `import crate_relative_path_dependency`).
+name = "crate_relative_path_dependency"
+
+# "cdylib" is necessary to produce a shared library for Python to import from.
+# Downstream Rust code (including code in `bin/`, `examples/`, and `examples/`) will not be able
+# to `use crate_relative_path_dependency;` unless the "rlib" or "lib" crate type is also included, e.g.:
+# crate-type = ["cdylib", "rlib"]
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.23.4", features = ["extension-module"] }
+test_crate = { path = "../test_crate" }

examples/crate_relative_path_dependency/src/lib.rs

@@ -0,0 +1,18 @@
+// rustimport:pyo3
+
+// Instruct rustimport to track any changes in `test_crate` and rebuild this crate automatically:
+
+//d: ../../test_crate/**/*.rs
+//d: ../../test_crate/Cargo.toml
+
+use pyo3::prelude::*;
+use test_crate;
+
+
+#[pyfunction]
+fn say_hello() -> String {
+    format!(
+        "Hello from crate_relative_path_dependency, and also hello from test_crate: \"{}\"",
+         test_crate::say_hello()
+    )
+}

examples/singlefile_templating.rs → examples/pyo3_basic.rs

@@ -17,6 +17,6 @@ fn random_number_from_rust(min: i32, max: i32) -> PyResult<i32> {
     Ok(rand::thread_rng().gen_range(min..max))
 }
 
-// Since we don't write a #[pymodule] macro here manually (see singlefile.rs for an example), the
-// "pyo3" template included in rustimport will automatically generate it for all functions
-// annotated with #[pyfunction] and all structs annotated with #[pyclass].
+// Since we don't write a #[pymodule] macro here manually (see
+// pyo3_manifest_only_templating.rs for an example), the "pyo3" template will automatically generate it for
+// all functions annotated with #[pyfunction] and all structs annotated with #[pyclass].

examples/declarative_module.rs → examples/pyo3_declarative_module.rs

@@ -12,7 +12,7 @@ fn say_hello() -> String {
 // Manually define the Python module using pyO3's declarative module syntax:
 
 #[pymodule]
-mod declarative_module {
+mod pyo3_declarative_module {
     #[pymodule_export]
     use super::say_hello; // Exports the `say_hello` function as part of the module
 }

examples/singlefile_manifest_only_templating.rs → examples/pyo3_manifest_only_templating.rs

@@ -18,7 +18,7 @@ fn try_divide(a: usize, b: usize) -> PyResult<usize> {
 
 /// A Python module implemented in Rust.
 #[pymodule]
-fn singlefile_manifest_only_templating(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
+fn pyo3_manifest_only_templating(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(try_divide, m)?)?;
     Ok(())
 }

examples/singlefile.rs → examples/pyo3_no_template.rs

@@ -1,22 +1,21 @@
 // rustimport
 
-// Note: This file uses rustimport and pyo3, but it does not use rustimport's pyo3
-// template, as it's not applied in the first line (compare to "// rustimport:pyo3").
-// Thus, all configuration for cargo manifest and pyo3 needs to be supplied manually
-// as below.
+// This file uses rustimport and pyo3, but it does not use rustimport's pyo3 template,
+// as it's not applied in the first line (compare to "// rustimport:pyo3"). Thus, all
+// configuration for cargo manifest and pyo3 needs to be supplied manually as below.
 
 // This is our cargo manifest:
 
 //: [package]
-//: name = "singlefile"
+//: name = "pyo3_no_template"
 //: version = "0.1.0"
 //: edition = "2021"
 //:
 //: [lib]
 //: # The name of the native library. This is the name which will be used in Python to import the
 //: # library (i.e. `import string_sum`). If you change this, you must also change the name of the
 //: # `#[pymodule]` below.
-//: name = "singlefile"
+//: name = "pyo3_no_template"
 //:
 //: # Downstream Rust code (including code in `bin/`, `examples/`, and `examples/`) will not be able
 //: # to `use singlefile;` unless the "rlib" or "lib" crate type is also included, e.g.:
@@ -37,10 +36,10 @@ fn sum_as_string(a: usize, b: usize) -> String {
 
 
 // This is our pyo3 module definition.
-// The name of this function must match the `lib.name` setting in the cargo manifest,
-// else Python will not be able to import the module.
+// The name of this function must match the `lib.name` setting in the cargo manifest, otherwise Python
+// will not be able to import the module.
 #[pymodule]
-fn singlefile(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
+fn pyo3_no_template(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
     Ok(())
 }

examples/relative_path_dependency.rs

@@ -0,0 +1,27 @@
+// rustimport:pyo3
+
+// Below, we specify a dependency on a path relative to the current file. Since rustimport builds
+// your extensions in a temporary location, it will automatically rewrite this path and turn it
+// into an absolute one:
+
+//: [dependencies]
+//: test_crate = { path = "./test_crate" }
+
+// If you also want rustimport to automatically rebuild this extension when anything in `test_crate`
+// is changed, you can specify dependency paths like below:
+
+//d: ./test_crate/Cargo.toml
+//d: ./test_crate/**/*.rs
+
+
+
+use pyo3::prelude::*;
+use test_crate;
+
+#[pyfunction]
+fn say_hello() -> String {
+    format!(
+        "Hello from relative_path_dependency.rs, and also hello from test_crate: \"{}\"",
+         test_crate::say_hello()
+    )
+}

examples/test_crate/src/lib.rs

@@ -11,8 +11,8 @@ pub fn say_hello() -> String {
 // rustimport will generate it for you for all functions annotated with
 // #[pyfunction] and all structs annotated with #[pyclass].
 //
-//#[pymodule]
-//fn test_crate(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
-//    m.add_function(wrap_pyfunction!(say_hello, m)?)?;
-//    Ok(())
-//}
+// #[pymodule]
+// fn test_crate(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
+//     m.add_function(wrap_pyfunction!(say_hello, m)?)?;
+//     Ok(())
+// }

rustimport/__main__.py

@@ -58,7 +58,7 @@ def create_extension(fn: str, cwd: str = '.'):
     mod_name = os.path.basename(fn)
 
     if not re.match(r'^[a-zA-Z]\w*(\.rs)?$', mod_name):
-        raise ValueError(f"Invalid extension name: {mod_name}. The name may only contain letters (preferably lowercase), "
+        raise ValueError(f"Invalid extension name: \"{mod_name}\". The name may only contain letters (preferably lowercase), "
                          f"numbers and underscores and should start with a letter.")
 
     path = os.path.realpath(os.path.join(cwd, fn))

rustimport/importable.py

@@ -194,7 +194,12 @@ def dependencies(self) -> List[str]:
         root_path = self.__workspace_path or self.__crate_path
         src_path = os.path.join(self.__crate_path, 'src')
 
-        p = Preprocessor(os.path.join(src_path, 'lib.rs'), lib_name=self.name).process()
+        p = Preprocessor(
+            os.path.join(src_path, 'lib.rs'),
+            lib_name=self.name,
+            cargo_manifest_path=self.__manifest_path,
+            workspace_path=self.__workspace_path,
+        ).process()
 
         return [
             os.path.join(root_path, '**/*.rs'),
@@ -292,13 +297,14 @@ def copy_function(src, dst):
 
     def _preprocess(self, crate_output_subdirectory: str) -> Preprocessor.PreprocessorResult:
         """
-        Calls [Preprocessor.process()] on the crate, updates the source files
-        with the result and returns the result for further usage.
+        Calls [Preprocessor.process()] on the crate, updates the source files with the result
+        and returns the result for further usage.
         """
         preprocessed = Preprocessor(
             os.path.join(self.__crate_path, 'src/lib.rs'),
             lib_name=self.name,
-            cargo_manifest_path=os.path.join(self.__crate_path, 'Cargo.toml'),
+            cargo_manifest_path=self.__manifest_path,
+            workspace_path=self.__workspace_path,
         ).process()
 
         if preprocessed.updated_source is not None:

rustimport/ipython_magic.py

@@ -1,15 +1,15 @@
 import hashlib
 import importlib.util
-import sys
 import subprocess
+import sys
 import time
 from importlib.machinery import ExtensionFileLoader
+from importlib.metadata import version
 from pathlib import Path
 from shutil import which
 
 from IPython.core import magic_arguments
 from IPython.core.magic import Magics, cell_magic, magics_class
-from importlib.metadata import version
 
 try:
     from IPython.paths import get_ipython_cache_dir

rustimport/pre_processing/__init__.py

@@ -1,4 +1,3 @@
-import copy
 import os.path
 import re
 from dataclasses import dataclass
@@ -18,18 +17,18 @@ class PreprocessorResult:
         updated_source: Optional[bytes]
         additional_cargo_args: List[str]
 
-    def __init__(self, path: str, lib_name: str, cargo_manifest_path: Optional[str] = None):
+    def __init__(self, path: str, lib_name: str, cargo_manifest_path: Optional[str] = None, workspace_path: Optional[str] = None):
         self.path = path
         self.lib_name = lib_name
         self.cargo_manifest_path = cargo_manifest_path
+        self.workspace_path = workspace_path
 
     def process(self) -> PreprocessorResult:
         with open(self.path, 'rb') as f:
             contents = f.read()
 
         raw_manifest, template_name, deps = self.__parse_header(contents)
         manifest = toml.loads(raw_manifest.decode())
-        self.__process_manifest(manifest)
 
         if self.cargo_manifest_path is not None:
             with open(self.cargo_manifest_path, 'r') as f:
@@ -38,13 +37,14 @@ def process(self) -> PreprocessorResult:
         if template_name:
             template = all_templates[template_name.lower()](self.path, self.lib_name, contents, manifest)
             templating_result = template.process()
+            manifest = templating_result.cargo_manifest
         else:
             templating_result = None
 
-        final_manifest = templating_result.cargo_manifest if templating_result else manifest
+        self.__process_manifest(manifest)
 
         return self.PreprocessorResult(
-            cargo_manifest=toml.dumps(final_manifest).encode(),
+            cargo_manifest=toml.dumps(manifest).encode(),
             dependency_file_patterns=deps,
             updated_source=templating_result.contents if templating_result else None,
             additional_cargo_args=templating_result.additional_cargo_args if templating_result else [],
@@ -71,7 +71,7 @@ def __parse_header(contents: bytes) -> Tuple[bytes, Optional[str], List[str]]:
 
     def __process_manifest(self, manifest):
         # Convert relative dependency paths into absolute ones in the manifest, to make them resolvable
-        # from the temporary location of the module:
+        # from the temporary location where the importable is being built:
         root = os.path.dirname(self.cargo_manifest_path or self.path)
         dependency_tables = (
             *_query_dict('dependencies', manifest),
@@ -81,10 +81,14 @@ def __process_manifest(self, manifest):
             *_query_dict('target.*.dev-dependencies', manifest),
             *_query_dict('target.*.build-dependencies', manifest),
         )
-        for deps in dependency_tables:  # walk through all dependency sections in the manifest
+        for deps in dependency_tables:  # walk through all dependency tables in the manifest
             for spec in deps.values():  # walk through all individual dependency specifications
-                if 'path' in spec:
-                    spec['path'] = os.path.join(root, spec['path'])  # make path absolute if it is not already
+                if 'path' in spec and not os.path.isabs(spec['path']):
+                    abspath = os.path.normpath(os.path.join(root, spec['path']))
+                    # If we are in a workspace and the dependency is in the same workspace, there is no need
+                    # to make the path absolute; the entire workspace is copied to the temporary directory.
+                    if not (self.workspace_path and _path_is_parent(self.workspace_path, abspath)):
+                        spec['path'] = abspath
 
 
 
@@ -113,3 +117,15 @@ def search(keys, node):
 
     return search(query.split("."), data)
 
+
+def _path_is_parent(parent_path: str, child_path: str):
+    # Smooth out relative path names, note: if you are concerned about symbolic links, you should use
+    # os.path.realpath too:
+    parent_path = os.path.abspath(parent_path)
+    child_path = os.path.abspath(child_path)
+
+    # Compare the common path of the parent and child path with the common path of just the parent path. Using the
+    # commonpath method on just the parent path will regularise the path name in the same way as the comparison that
+    # deals with both paths, removing any trailing path separator:
+    return os.path.commonpath([parent_path]) == os.path.commonpath([parent_path, child_path])
+