From 198ab7f4876d4f0eb214d3684524b37adac9a1e7 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 30 Aug 2023 06:10:18 +0200 Subject: [PATCH] Support for Python libraries like numpy (#7678) --- CHANGELOG.md | 2 + docs/polyglot/README.md | 2 + docs/polyglot/polyglot-bindings.md | 10 +- docs/polyglot/python.md | 72 +++++ .../org/enso/runner/ContextFactory.scala | 17 +- .../interpreter/epb/node/ForeignEvalNode.java | 31 +- test/Tests/src/System/Process_Spec.enso | 282 +++++++++++------- 7 files changed, 291 insertions(+), 125 deletions(-) create mode 100644 docs/polyglot/python.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3226c742b9bc..4246d72dae89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -923,6 +923,7 @@ - [Only use types as State keys][7585] - [Allow Java Enums in case of branches][7607] - [Notification about the project rename action][7613] +- [Use `numpy` & co. from Enso!][7678] [3227]: https://github.com/enso-org/enso/pull/3227 [3248]: https://github.com/enso-org/enso/pull/3248 @@ -1057,6 +1058,7 @@ [7585]: https://github.com/enso-org/enso/pull/7585 [7607]: https://github.com/enso-org/enso/pull/7607 [7613]: https://github.com/enso-org/enso/pull/7613 +[7678]: https://github.com/enso-org/enso/pull/7678 # Enso 2.0.0-alpha.18 (2021-10-12) diff --git a/docs/polyglot/README.md b/docs/polyglot/README.md index 2c48b55a5948..eb73117571fe 100644 --- a/docs/polyglot/README.md +++ b/docs/polyglot/README.md @@ -24,3 +24,5 @@ It also provides language-specific documentation for the various supported polyglot languages. These are as follows: - [**Java:**](./java.md) Information specific to the Java polyglot bindings. +- [**Python:**](./python.md) Information specific to the Python polyglot + bindings. diff --git a/docs/polyglot/polyglot-bindings.md b/docs/polyglot/polyglot-bindings.md index 1ebadda926eb..f35896a364d7 100644 --- a/docs/polyglot/polyglot-bindings.md +++ b/docs/polyglot/polyglot-bindings.md @@ -161,7 +161,7 @@ the language-specific documentation for details. ## Embedded Syntax The term "Embedded Syntax" is our terminology for the ability to use foreign -language syntaxes from directly inside `.enso` files. This system builds upon +language syntaxes directly from inside `.enso` files. This system builds upon the more generic mechanisms used by the [polyglot FFI](#the-polyglot-ffi) to provide a truly seamless user experience. @@ -169,14 +169,14 @@ provide a truly seamless user experience. A polyglot block is introduced as follows: -- The `polyglot` keyword starts a block. -- This must be followed by a language identifier (e.g. `java`). +- The `foreign` keyword starts a block. +- This must be followed by a language identifier (e.g. `python`). - After the language identifier, the remaining syntax behaves like it is an Enso function definition until the `=`. -- After the `=`, the user may write their foreign code. +- After the `=`, the user may write their foreign code as a string. ```ruby -polyglot python concat a b = +foreign python concat a b = """ def concat(a, b): str(a) + str(b) ``` diff --git a/docs/polyglot/python.md b/docs/polyglot/python.md new file mode 100644 index 000000000000..d8e02bf7278a --- /dev/null +++ b/docs/polyglot/python.md @@ -0,0 +1,72 @@ +--- +layout: developer-doc +title: Polyglot Python +category: polyglot +tags: [polyglot, python] +order: 4 +--- + +# Polyglot Python + +This document provides practical example showing polyglot interoperability with +Python in the runtime. Please familiarise yourself with the general operation of +[polyglot bindings](./polyglot-bindings.md). + + + +- [Polyglot Library System](#polyglot-library-system) +- [Using Python Libraries](#using-python-libraries) + + + +## Polyglot Library System + +There is a support for using any Python library from Enso. Steps to include +`numpy` in a new Enso project follows: + +```bash +$ enso-engine*/bin/enso --new numenso +$ find numenso/ +numenso/ +numenso/src +numenso/src/Main.enso +numenso/package.yaml +$ mkdir numenso/polyglot +$ graalvm/bin/gu install python +$ graalvm/bin/graalpy -m venv numenso/polyglot/python +$ ./numenso/polyglot/python/bin/graalpy -m pip install numpy +Successfully installed numpy-1.23.5 +``` + +The above steps instruct Enso to create a new project in `numenso` directory. +Then they create Python virtual environment in `numenso/polyglot/python/` dir - +e.g. in the +[standard location for polyglot](../distribution/packaging.md#the-polyglot-directory) +components of an Enso project. As a last step we activate the virtual +environment and use `pip` manager to install `numpy` library. + +## Using Python Libraries + +As soon as a library is installed into the +[polyglot directory](#polyglot-library-system) it can be used via the +[embedded syntax](polyglot-bindings.md#embedded-syntax): + +```ruby +foreign python random_array s = """ + import numpy + return numpy.random.normal(size=s) + +main = random_array 10 +``` + +Let's modify the `numenso/src/Main.enso` to use `numpy.random.normal` as shown +above. Then we can simply execute the project and obtain a `numpy` array as a +result: + +```bash +$ enso-engine*/bin/enso --run numenso +array([-0.51884419, -0.23670113, -1.20493508, -0.86008709, 0.59403118, + -0.171484 , -1.19455596, -0.30096434, -0.69762239, -0.11411331]) +``` + +The same steps can be applied to any Graal Python supported library. diff --git a/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala b/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala index 0faff393e031..9b5b2df70f7d 100644 --- a/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala +++ b/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala @@ -8,7 +8,7 @@ import org.enso.polyglot.debugger.{ import org.enso.polyglot.{HostAccessFactory, PolyglotContext, RuntimeOptions} import org.graalvm.polyglot.Context -import java.io.{InputStream, OutputStream} +import java.io.{File, InputStream, OutputStream} /** Utility class for creating Graal polyglot contexts. */ @@ -49,7 +49,7 @@ class ContextFactory { executionEnvironment.foreach { name => options.put("enso.ExecutionEnvironment", name) } - val context = Context + val builder = Context .newBuilder() .allowExperimentalOptions(true) .allowAllAccess(true) @@ -88,7 +88,16 @@ class ContextFactory { .logHandler( JavaLoggingLogHandler.create(JavaLoggingLogHandler.defaultLevelMapping) ) - .build - new PolyglotContext(context) + val graalpy = new File( + new File( + new File(new File(new File(projectRoot), "polyglot"), "python"), + "bin" + ), + "graalpy" + ); + if (graalpy.exists()) { + builder.option("python.Executable", graalpy.getAbsolutePath()); + } + new PolyglotContext(builder.build) } } diff --git a/engine/runtime-language-epb/src/main/java/org/enso/interpreter/epb/node/ForeignEvalNode.java b/engine/runtime-language-epb/src/main/java/org/enso/interpreter/epb/node/ForeignEvalNode.java index 3e8bef5ca738..5709c915b7ff 100644 --- a/engine/runtime-language-epb/src/main/java/org/enso/interpreter/epb/node/ForeignEvalNode.java +++ b/engine/runtime-language-epb/src/main/java/org/enso/interpreter/epb/node/ForeignEvalNode.java @@ -1,5 +1,15 @@ package org.enso.interpreter.epb.node; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.enso.interpreter.epb.EpbContext; +import org.enso.interpreter.epb.EpbLanguage; +import org.enso.interpreter.epb.EpbParser; +import org.enso.interpreter.epb.runtime.ForeignParsingException; +import org.enso.interpreter.epb.runtime.GuardedTruffleContext; + import com.oracle.truffle.api.CallTarget; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; @@ -10,14 +20,6 @@ import com.oracle.truffle.api.interop.InteropLibrary; import com.oracle.truffle.api.nodes.RootNode; import com.oracle.truffle.api.source.Source; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.enso.interpreter.epb.EpbContext; -import org.enso.interpreter.epb.EpbLanguage; -import org.enso.interpreter.epb.EpbParser; -import org.enso.interpreter.epb.runtime.ForeignParsingException; -import org.enso.interpreter.epb.runtime.GuardedTruffleContext; public class ForeignEvalNode extends RootNode { private final EpbParser.Result code; @@ -136,12 +138,13 @@ private void parseJs() { private void parsePy() { try { String args = Arrays.stream(argNames).collect(Collectors.joining(",")); - String head = - "import polyglot\n" - + "@polyglot.export_value\n" - + "def polyglot_enso_python_eval(" - + args - + "):\n"; + String head = """ + import site + import polyglot + @polyglot.export_value + def polyglot_enso_python_eval(""" + + args + + "):\n"; String indentLines = code.getForeignSource().lines().map(l -> " " + l).collect(Collectors.joining("\n")); Source source = diff --git a/test/Tests/src/System/Process_Spec.enso b/test/Tests/src/System/Process_Spec.enso index 600a67f89864..3368a0420726 100644 --- a/test/Tests/src/System/Process_Spec.enso +++ b/test/Tests/src/System/Process_Spec.enso @@ -3,107 +3,185 @@ from Standard.Base import all from Standard.Test import Test, Test_Suite import Standard.Test.Extensions -spec = Test.group "Process" <| - Test.specify "should call simple command" <| - result = case Platform.os of - Platform.OS.Windows -> - Process.run "PowerShell" ["-Command", "exit 0"] - _ -> - Process.run "bash" ["-c", "exit 0"] - result.exit_code.should_equal Exit_Code.Success - Test.specify "should return exit code" <| - case Platform.os of - Platform.OS.Unknown -> - Test.fail "Unsupported platform." - Platform.OS.Windows -> - r = Process.run "PowerShell" ["-Command", "exit 42"] - r.exit_code.should_equal <| Exit_Code.Failure 42 - - s = Process.run "PowerShell" ["-Command", "exit 0"] - s.exit_code.should_equal <| Exit_Code.Success - _ -> - r = Process.run "bash" ["-c", "exit 42"] - r.exit_code.should_equal <| Exit_Code.Failure 42 - - s = Process.run "bash" ["-c", "exit 0"] - s.exit_code.should_equal <| Exit_Code.Success - Test.specify "should return stdout" <| - case Platform.os of - Platform.OS.Unknown -> - Test.fail "Unsupported platform." - Platform.OS.Windows -> - builder = Process.new_builder "PowerShell" ["-Command", "[System.Console]::Out.Write('Hello')"] - result = builder.create - result.exit_code.to_number . should_equal 0 - result.stdout . should_equal "Hello" - result.stderr . should_equal "" - - run_result = Process.run "PowerShell" ["-Command", "[System.Console]::Out.Write('Hello')"] - run_result.exit_code.to_number . should_equal 0 - run_result.stdout . should_equal "Hello" - run_result.stderr . should_equal "" - _ -> - builder = Process.new_builder "bash" ["-c", "echo -n Hello"] - result = builder.create - result.exit_code.to_number . should_equal 0 - result.stdout . should_equal "Hello" - result.stderr . should_equal "" - - run_result = Process.run "bash" ["-c", "echo -n Hello"] - run_result.exit_code.to_number . should_equal 0 - run_result.stdout . should_equal "Hello" - run_result.stderr . should_equal "" - Test.specify "should return stderr" <| - case Platform.os of - Platform.OS.Unknown -> - Test.fail "Unsupported platform." - Platform.OS.Windows -> - builder = Process.new_builder "PowerShell" ["-Command", "[System.Console]::Error.Write('Error')"] - result = builder.create - result.exit_code.to_number . should_equal 0 - result.stdout . should_equal "" - result.stderr . should_equal "Error" - - run_result = Process.run "PowerShell" ["-Command", "[System.Console]::Error.Write('Error')"] - run_result.exit_code.to_number . should_equal 0 - run_result.stdout . should_equal "" - run_result.stderr . should_equal "Error" - _ -> - builder = Process.new_builder "bash" ["-c", "echo -n Error 1>&2"] - result = builder.create - result.exit_code.to_number . should_equal 0 - result.stdout . should_equal "" - result.stderr . should_equal "Error" - - run_result = Process.run "bash" ["-c", "echo -n Error 1>&2"] - run_result.exit_code.to_number . should_equal 0 - run_result.stdout . should_equal "" - run_result.stderr . should_equal "Error" - Test.specify "should feed stdin" <| - case Platform.os of - Platform.OS.Unknown -> - Test.fail "Unsupported platform." - Platform.OS.Windows -> - builder = Process.new_builder "PowerShell" ["-Command", "[System.Console]::ReadLine()"] . set_stdin "sample" - result = builder.create - result.exit_code.to_number . should_equal 0 - result.stdout . should_equal 'sample\r\n' - result.stderr . should_equal "" - - run_result = Process.run "PowerShell" ["-Command", "[System.Console]::ReadLine()"] stdin="sample" - run_result.exit_code.to_number . should_equal 0 - run_result.stdout . should_equal 'sample\r\n' - run_result.stderr . should_equal "" - _ -> - builder = Process.new_builder "bash" ["-c", "read line; echo -n $line"] . set_stdin "sample" - result = builder.create - result.exit_code.to_number . should_equal 0 - result.stdout . should_equal "sample" - result.stderr . should_equal "" - - run_result = Process.run "bash" ["-c", "read line; echo -n $line"] stdin="sample" - run_result.exit_code.to_number . should_equal 0 - run_result.stdout . should_equal 'sample' - run_result.stderr . should_equal "" +polyglot java import java.lang.System as Java_System +polyglot java import java.io.File as Java_File + +pending_python_missing = if Polyglot.is_language_installed "python" then Nothing else """ + Can't run Python tests, Python is not installed. + +spec = + Test.group "Process" <| + Test.specify "should call simple command" <| + result = case Platform.os of + Platform.OS.Windows -> + Process.run "PowerShell" ["-Command", "exit 0"] + _ -> + Process.run "bash" ["-c", "exit 0"] + result.exit_code.should_equal Exit_Code.Success + Test.specify "should return exit code" <| + case Platform.os of + Platform.OS.Unknown -> + Test.fail "Unsupported platform." + Platform.OS.Windows -> + r = Process.run "PowerShell" ["-Command", "exit 42"] + r.exit_code.should_equal <| Exit_Code.Failure 42 + + s = Process.run "PowerShell" ["-Command", "exit 0"] + s.exit_code.should_equal <| Exit_Code.Success + _ -> + r = Process.run "bash" ["-c", "exit 42"] + r.exit_code.should_equal <| Exit_Code.Failure 42 + + s = Process.run "bash" ["-c", "exit 0"] + s.exit_code.should_equal <| Exit_Code.Success + Test.specify "should return stdout" <| + case Platform.os of + Platform.OS.Unknown -> + Test.fail "Unsupported platform." + Platform.OS.Windows -> + builder = Process.new_builder "PowerShell" ["-Command", "[System.Console]::Out.Write('Hello')"] + result = builder.create + result.exit_code.to_number . should_equal 0 + result.stdout . should_equal "Hello" + result.stderr . should_equal "" + + run_result = Process.run "PowerShell" ["-Command", "[System.Console]::Out.Write('Hello')"] + run_result.exit_code.to_number . should_equal 0 + run_result.stdout . should_equal "Hello" + run_result.stderr . should_equal "" + _ -> + builder = Process.new_builder "bash" ["-c", "echo -n Hello"] + result = builder.create + result.exit_code.to_number . should_equal 0 + result.stdout . should_equal "Hello" + result.stderr . should_equal "" + + run_result = Process.run "bash" ["-c", "echo -n Hello"] + run_result.exit_code.to_number . should_equal 0 + run_result.stdout . should_equal "Hello" + run_result.stderr . should_equal "" + Test.specify "should return stderr" <| + case Platform.os of + Platform.OS.Unknown -> + Test.fail "Unsupported platform." + Platform.OS.Windows -> + builder = Process.new_builder "PowerShell" ["-Command", "[System.Console]::Error.Write('Error')"] + result = builder.create + result.exit_code.to_number . should_equal 0 + result.stdout . should_equal "" + result.stderr . should_equal "Error" + + run_result = Process.run "PowerShell" ["-Command", "[System.Console]::Error.Write('Error')"] + run_result.exit_code.to_number . should_equal 0 + run_result.stdout . should_equal "" + run_result.stderr . should_equal "Error" + _ -> + builder = Process.new_builder "bash" ["-c", "echo -n Error 1>&2"] + result = builder.create + result.exit_code.to_number . should_equal 0 + result.stdout . should_equal "" + result.stderr . should_equal "Error" + + run_result = Process.run "bash" ["-c", "echo -n Error 1>&2"] + run_result.exit_code.to_number . should_equal 0 + run_result.stdout . should_equal "" + run_result.stderr . should_equal "Error" + Test.specify "should feed stdin" <| + case Platform.os of + Platform.OS.Unknown -> + Test.fail "Unsupported platform." + Platform.OS.Windows -> + builder = Process.new_builder "PowerShell" ["-Command", "[System.Console]::ReadLine()"] . set_stdin "sample" + result = builder.create + result.exit_code.to_number . should_equal 0 + result.stdout . should_equal 'sample\r\n' + result.stderr . should_equal "" + + run_result = Process.run "PowerShell" ["-Command", "[System.Console]::ReadLine()"] stdin="sample" + run_result.exit_code.to_number . should_equal 0 + run_result.stdout . should_equal 'sample\r\n' + run_result.stderr . should_equal "" + _ -> + builder = Process.new_builder "bash" ["-c", "read line; echo -n $line"] . set_stdin "sample" + result = builder.create + result.exit_code.to_number . should_equal 0 + result.stdout . should_equal "sample" + result.stderr . should_equal "" + + run_result = Process.run "bash" ["-c", "read line; echo -n $line"] stdin="sample" + run_result.exit_code.to_number . should_equal 0 + run_result.stdout . should_equal 'sample' + run_result.stderr . should_equal "" + Test.group "Enso on Enso" <| + enso_bin = + p = Java_System.getProperty "truffle.class.path.append" + s = p.split Java_File.separator + paths = s.take (Index_Sub_Range.While _!="..") + j = paths . join Java_File.separator + File.new j / if Platform.os == Platform.OS.Windows then "enso.bat" else "enso" + + create_new_enso_project = + bin = enso_bin + + tmp_file = File.create_temporary_file "enso_prj" "" + dir = tmp_file/".."/(tmp_file.name+".dir") . normalize + res = Process.run bin.path [ "--new", dir.path ] + IO.println res.stdout + IO.println res.stderr + res.exit_code . should_equal Exit_Code.Success + dir + + Test.specify "Create Enso Project with numpy" pending=pending_python_missing <| + setup_venv dir = + gvm = File.new <| Java_System.getProperty "java.home" + python = gvm/"bin"/"graalpy" + res = Process.run python.path [ "-m", "venv", dir.path ] + IO.println res.stdout + IO.println res.stderr + res.exit_code . should_equal Exit_Code.Success + + install_num_py dir = + python = dir/"bin"/"graalpy" + res = Process.run python.path [ "-m", "pip", "install", "numpy" ] + IO.println res.stdout + IO.println res.stderr + res.exit_code . should_equal Exit_Code.Success + + rewrite_main_file dir = + main = dir/"src"/"Main.enso" + main.exists . should_be_true + code = """ + foreign python random_array s = """ + import numpy + return numpy.random.normal(size=s) + + main = random_array 10 + + code . write main on_existing_file=Existing_File_Behavior.Overwrite + + IO.println "==== Generating Enso Project ====" + prj = create_new_enso_project + IO.println "Project ready at "+prj.path + + IO.println "==== Changing Main.enso ====" + rewrite_main_file prj + + IO.println "==== Preparing Python Virtual Environment ====" + setup_venv prj/"polyglot"/"python" + + IO.println "==== Installing numpy ====" + install_num_py prj/"polyglot"/"python" + + IO.println "==== Executing project ====" + + res = Process.run enso_bin.path [ "--run", prj.path ] + IO.println res.stdout + IO.println res.stderr + res.exit_code . should_equal Exit_Code.Success + + IO.println "==== Done ====" + + res.stdout.should_contain "array([" + res.stdout.should_contain "])" main = Test_Suite.run_main spec