diff --git a/.gitignore b/.gitignore index 9d63591daa09..444645571d9b 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,7 @@ bench-report*.xml /enso.lib *.dll +*.dylib *.exe *.pdb *.so diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c83b1cfb59d..3c054bb201f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,15 @@ - A constructor or type definition with a single inline argument definition was previously allowed to use spaces in the argument definition without parentheses. [This is now a syntax error.][11856] +- [Native libraries of projects can be added to `polyglot/lib` directory][11874] +- [Redo stack is no longer lost when interacting with text literals][11908]. - Symetric, transitive and reflexive [equality for intersection types][11897] [11777]: https://github.com/enso-org/enso/pull/11777 [11600]: https://github.com/enso-org/enso/pull/11600 [11856]: https://github.com/enso-org/enso/pull/11856 +[11874]: https://github.com/enso-org/enso/pull/11874 +[11908]: https://github.com/enso-org/enso/pull/11908 [11897]: https://github.com/enso-org/enso/pull/11897 # Enso 2024.5 diff --git a/build.sbt b/build.sbt index a7491b8881c3..e13b2b32d9f0 100644 --- a/build.sbt +++ b/build.sbt @@ -3712,7 +3712,8 @@ lazy val `engine-runner` = project // "-H:-DeleteLocalSymbols", // you may need to set smallJdk := None to use following flags: // "--trace-class-initialization=org.enso.syntax2.Parser", - "-Dnic=nic" + "-Dnic=nic", + "-Dorg.enso.feature.native.lib.output=" + (engineDistributionRoot.value / "bin") ), mainClass = Some("org.enso.runner.Main"), initializeAtRuntime = Seq( @@ -4487,6 +4488,7 @@ def stdLibComponentRoot(name: String): File = val `base-polyglot-root` = stdLibComponentRoot("Base") / "polyglot" / "java" val `table-polyglot-root` = stdLibComponentRoot("Table") / "polyglot" / "java" val `image-polyglot-root` = stdLibComponentRoot("Image") / "polyglot" / "java" +val `image-native-libs` = stdLibComponentRoot("Image") / "polyglot" / "lib" val `google-api-polyglot-root` = stdLibComponentRoot("Google_Api") / "polyglot" / "java" val `database-polyglot-root` = @@ -4654,6 +4656,10 @@ lazy val `std-table` = project ) .dependsOn(`std-base` % "provided") +lazy val extractNativeLibs = taskKey[Unit]( + "Helper task to extract native libraries from OpenCV JAR" +) + lazy val `std-image` = project .in(file("std-bits") / "image") .settings( @@ -4669,15 +4675,21 @@ lazy val `std-image` = project "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", "org.openpnp" % "opencv" % opencvVersion ), - Compile / packageBin := Def.task { - val result = (Compile / packageBin).value - val _ = StdBits - .copyDependencies( + // Extract native libraries from opencv.jar, and put them under + // Standard/Image/polyglot/lib directory. The minimized opencv.jar will + // be put under Standard/Image/polyglot/java directory. + extractNativeLibs := { + StdBits + .extractNativeLibsFromOpenCV( `image-polyglot-root`, - Seq("std-image.jar"), - ignoreScalaLibrary = true + `image-native-libs`, + opencvVersion ) .value + }, + Compile / packageBin := Def.task { + val result = (Compile / packageBin).value + val _ = extractNativeLibs.value result }.value ) diff --git a/docs/polyglot/java.md b/docs/polyglot/java.md index 9efb70568ddc..b698aff493cd 100644 --- a/docs/polyglot/java.md +++ b/docs/polyglot/java.md @@ -44,27 +44,60 @@ The dynamic polyglot system is a dynamic runtime lookup for Java objects, allowing Enso code to work with them through a runtime reflection-style mechanism. It is comprised of the following components: -- `Java.lookup_class : Class.Path -> Maybe Class`: A function that lets users - look up a class by a given name on the runtime classpath. -- `Polyglot.instantiate : Class -> Object`: A function that lets users - instantiate a class into an object. +- `Java.lookup_class : Text -> Any`: A function that lets users look up a class + by a given name on the runtime classpath. - A whole host of functions on the polyglot type that let you dynamically work with object bindings. An example can be found below: ```ruby +from Standard.Base.Polyglot import Java, polyglot + main = class = Java.lookup_class "org.enso.example.TestClass" - instance = Polyglot.instantiate1 class (x -> x * 2) + instance = class.new (x -> x * 2) method = Polyglot.get_member instance "callFunctionAndIncrement" - Polyglot.execute1 method 10 + Polyglot.execute method 10 ``` > The actionables for this section are: > > - Expand on the detail when there is time. +## Native libraries + +Java can load native libraries using, e.g., the +[System.loadLibrary]() +or +[ClassLoader.findLibrary]() +methods. If a Java method loaded from the `polyglot/java` directory in project +`Proj` tries to load a native library via one of the aforementioned mechanisms, +the runtime system will look for the native library in the `polyglot/lib` +directory within the project `Proj`. The runtime system implements this by +overriding the +[ClassLoader.findLibrary]() +method on the `ClassLoader` used to load the Java class. + +The algorithm used to search for the native libraries within the `polyglot/lib` +directory hierarchy conforms to the +[NetBeans JNI specification](https://bits.netbeans.org/23/javadoc/org-openide-modules/org/openide/modules/doc-files/api.html#jni): +Lookup of library with name `native` works roughly in these steps: + +- Add platform-specific prefix and/or suffix to the library name, e.g., + `libnative.so` on Linux. +- Search for the library in the `polyglot/lib` directory. +- Search for the library in the `polyglot/lib/` directory, where `` + is the name of the architecture. +- Search for the library in the `polyglot/lib//` directory, where + `` is the name of the operating system. + +Supported names: + +- Names for `` are `linux`, `macos`, `windows`. + - Note that for simplicity we omit the versions of the operating systems. +- Names for architectures `` are `amd64`, `x86_64`, `x86_32`, `aarch64`. + ## Download a Java Library from Maven Central A typical use-case when bringing in some popular Java library into Enso diff --git a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java index 7b25fd8a11d5..f9fa7eb67875 100644 --- a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java +++ b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java @@ -2,27 +2,41 @@ import static scala.jdk.javaapi.CollectionConverters.asJava; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.util.LinkedHashSet; import java.util.TreeSet; import org.enso.compiler.core.EnsoParser; import org.enso.compiler.core.ir.module.scope.imports.Polyglot; +import org.enso.filesystem.FileSystem$; +import org.enso.pkg.NativeLibraryFinder; import org.enso.pkg.PackageManager$; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.nativeimage.hosted.RuntimeProxyCreation; import org.graalvm.nativeimage.hosted.RuntimeReflection; -import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; public final class EnsoLibraryFeature implements Feature { + private static final String LIB_OUTPUT = "org.enso.feature.native.lib.output"; + private final File nativeLibDir; + + public EnsoLibraryFeature() { + var nativeLibOut = System.getProperty(LIB_OUTPUT); + if (nativeLibOut == null) { + throw new IllegalStateException("Missing system property: " + LIB_OUTPUT); + } + nativeLibDir = new File(nativeLibOut); + if (!nativeLibDir.exists() || !nativeLibDir.isDirectory()) { + var created = nativeLibDir.mkdirs(); + if (!created) { + throw new IllegalStateException("Cannot create directory: " + nativeLibDir); + } + } + } + @Override public void beforeAnalysis(BeforeAnalysisAccess access) { - try { - registerOpenCV(access.getApplicationClassLoader()); - } catch (ReflectiveOperationException ex) { - ex.printStackTrace(); - throw new IllegalStateException(ex); - } + var libs = new LinkedHashSet(); for (var p : access.getApplicationClassPath()) { var p1 = p.getParent(); @@ -53,6 +67,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { */ var classes = new TreeSet(); + var nativeLibPaths = new TreeSet(); try { for (var p : libs) { var result = PackageManager$.MODULE$.Default().loadPackage(p.toFile()); @@ -90,10 +105,19 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { } } } + if (pkg.nativeLibraryDir().exists()) { + var nativeLibs = + NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs()); + for (var nativeLib : nativeLibs) { + var out = new File(nativeLibDir, nativeLib.getName()); + Files.copy(nativeLib.toPath(), out.toPath()); + nativeLibPaths.add(out.getAbsolutePath()); + } + } } } } catch (Exception ex) { - ex.printStackTrace(); + ex.printStackTrace(System.err); throw new IllegalStateException(ex); } System.err.println("Summary for polyglot import java:"); @@ -101,35 +125,6 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { System.err.println(" " + className); } System.err.println("Registered " + classes.size() + " classes for reflection"); - } - - private static void registerOpenCV(ClassLoader cl) throws ReflectiveOperationException { - var moduleOpenCV = cl.getUnnamedModule(); - var currentOS = System.getProperty("os.name").toUpperCase().replaceAll(" .*$", ""); - - var libOpenCV = - switch (currentOS) { - case "LINUX" -> "nu/pattern/opencv/linux/x86_64/libopencv_java470.so"; - case "WINDOWS" -> "nu/pattern/opencv/windows/x86_64/opencv_java470.dll"; - case "MAC" -> { - var arch = System.getProperty("os.arch").toUpperCase(); - yield switch (arch) { - case "X86_64" -> "nu/pattern/opencv/osx/x86_64/libopencv_java470.dylib"; - case "AARCH64" -> "nu/pattern/opencv/osx/ARMv8/libopencv_java470.dylib"; - default -> null; - }; - } - default -> null; - }; - - if (libOpenCV != null) { - var verify = cl.getResource(libOpenCV); - if (verify == null) { - throw new IllegalStateException("Cannot find " + libOpenCV + " resource in " + cl); - } - RuntimeResourceAccess.addResource(moduleOpenCV, libOpenCV); - } else { - throw new IllegalStateException("No resource suggested for " + currentOS); - } + System.err.println("Copied native libraries: " + nativeLibPaths + " into " + nativeLibDir); } } diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java index bb41f3d6a129..24e0595f3653 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java @@ -34,7 +34,7 @@ public class SerializationManagerTest { @Before public void setup() { - packageManager = new PackageManager<>(new TruffleFileSystem()); + packageManager = new PackageManager<>(TruffleFileSystem.INSTANCE); interpreterContext = new InterpreterContext(x -> x); ensoContext = interpreterContext diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java new file mode 100644 index 000000000000..aeee72203ad9 --- /dev/null +++ b/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java @@ -0,0 +1,108 @@ +package org.enso.interpreter.test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +import com.oracle.truffle.api.TruffleFile; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import org.enso.editions.LibraryName; +import org.enso.interpreter.runtime.util.TruffleFileSystem; +import org.enso.pkg.NativeLibraryFinder; +import org.enso.pkg.Package; +import org.enso.test.utils.ContextUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +public class NativeLibraryFinderTest { + + @Rule public final TestRule printContextRule = new PrintSystemInfoRule(); + private Package stdImgPkg; + + @Test + public void standardImageShouldHaveNativeLib() { + try (var ctx = ContextUtils.createDefaultContext()) { + // Evaluate dummy sources to force loading Standard.Image + ContextUtils.evalModule( + ctx, """ + from Standard.Image import all + main = 42 + """); + var ensoCtx = ContextUtils.leakContext(ctx); + var stdImg = + ensoCtx + .getPackageRepository() + .getPackageForLibraryJava(LibraryName.apply("Standard", "Image")); + assertThat(stdImg.isPresent(), is(true)); + this.stdImgPkg = stdImg.get(); + var nativeLibs = + NativeLibraryFinder.listAllNativeLibraries(stdImg.get(), TruffleFileSystem.INSTANCE); + assertThat( + "There should be just single native lib in Standard.Image", nativeLibs.size(), is(1)); + } + } + + public final class PrintSystemInfoRule implements TestRule { + @Override + public Statement apply(Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() { + try { + base.evaluate(); + } catch (Throwable e) { + var sb = new StringBuilder(); + sb.append(System.lineSeparator()); + sb.append(" os.name: ") + .append(System.getProperty("os.name")) + .append(System.lineSeparator()); + sb.append(" os.arch: ") + .append(System.getProperty("os.arch")) + .append(System.lineSeparator()); + var mappedLibName = System.mapLibraryName("opencv_java470"); + sb.append(" Mapped library name: ") + .append(mappedLibName) + .append(System.lineSeparator()); + if (stdImgPkg != null) { + sb.append(" Contents of Standard.Image native library dir:") + .append(System.lineSeparator()); + var nativeLibDir = stdImgPkg.nativeLibraryDir(); + var nativeLibPath = Path.of(nativeLibDir.getAbsoluteFile().getPath()); + var contents = contentsOfDir(nativeLibPath); + contents.forEach( + path -> sb.append(" ").append(path).append(System.lineSeparator())); + } + throw new AssertionError(sb.toString(), e); + } + } + }; + } + } + + private static List contentsOfDir(Path dir) { + var contents = new ArrayList(); + var fileVisitor = + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + contents.add(file.toAbsolutePath().toString()); + return FileVisitResult.CONTINUE; + } + }; + try { + Files.walkFileTree(dir, fileVisitor); + } catch (IOException e) { + throw new AssertionError(e); + } + return contents; + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index c3f3bebd0f44..a70c94046676 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -170,7 +170,7 @@ public EnsoContext( /** Perform expensive initialization logic for the context. */ public void initialize() { - TruffleFileSystem fs = new TruffleFileSystem(); + TruffleFileSystem fs = TruffleFileSystem.INSTANCE; PackageManager packageManager = new PackageManager<>(fs); Optional projectRoot = OptionsHelper.getProjectRoot(environment); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java index 4c2ffb2a2b6e..bac13e762752 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java @@ -4,6 +4,8 @@ import java.net.URLClassLoader; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.enso.interpreter.runtime.util.TruffleFileSystem; +import org.enso.pkg.NativeLibraryFinder; import org.graalvm.polyglot.Context; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,6 +83,31 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE } } + /** + * Find the library with the specified name inside the {@code polyglot/lib} directory of caller's + * project. The search inside the {@code polyglot/lib} directory hierarchy is specified by NetBeans + * JNI specification. + * + *

Note: The current implementation iterates all the {@code polyglot/lib} directories of all + * the packages. + * + * @param libname The library name. Without platform-specific suffix or prefix. + * @return Absolute path to the library if found, or null. + */ + @Override + protected String findLibrary(String libname) { + var pkgRepo = EnsoContext.get(null).getPackageRepository(); + for (var pkg : pkgRepo.getLoadedPackagesJava()) { + var libPath = NativeLibraryFinder.findNativeLibrary(libname, pkg, TruffleFileSystem.INSTANCE); + if (libPath != null) { + return libPath; + } + } + logger.trace("Native library {} not found in any package", libname); + return null; + } + @Override public void close() { loadedClasses.clear(); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/util/TruffleFileSystem.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/util/TruffleFileSystem.java index 56cfb1e4550c..157af944132b 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/util/TruffleFileSystem.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/util/TruffleFileSystem.java @@ -14,7 +14,11 @@ import org.enso.filesystem.FileSystem; /** A {@link TruffleFile}-based implementation of {@link FileSystem}. */ -public class TruffleFileSystem implements FileSystem { +public final class TruffleFileSystem implements FileSystem { + private TruffleFileSystem() {} + + public static final TruffleFileSystem INSTANCE = new TruffleFileSystem(); + @Override public TruffleFile getChild(TruffleFile parent, String childName) { return parent.resolve(childName); @@ -45,6 +49,11 @@ public List getSegments(TruffleFile file) { return Arrays.asList(file.toRelativeUri().getPath().split("/")); } + @Override + public String getAbsolutePath(TruffleFile file) { + return file.getAbsoluteFile().getPath(); + } + @Override public String getName(TruffleFile file) { return file.getName(); diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala index 3723cab9b1ae..41cd630104fb 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala @@ -58,7 +58,7 @@ private class DefaultPackageRepository( private val logger = Logger[DefaultPackageRepository] - implicit private val fs: TruffleFileSystem = new TruffleFileSystem + implicit private val fs: TruffleFileSystem = TruffleFileSystem.INSTANCE private val packageManager = new PackageManager[TruffleFile] private var projectPackage: Option[Package[TruffleFile]] = None diff --git a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java new file mode 100644 index 000000000000..851dcd158b81 --- /dev/null +++ b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java @@ -0,0 +1,101 @@ +package org.enso.pkg; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import org.enso.filesystem.FileSystem; + +/** + * Helper class to find native libraries in packages. The search algorithm complies to the NetBeans + * JNI specification. + */ +public final class NativeLibraryFinder { + + private NativeLibraryFinder() {} + + /** + * Tries to find native library in the given package. + * + * @param libName the name of the library to find, without platform specific prefix or suffix. + * @param pkg the package to search in. + * @return null if not found, absolute path otherwise. + */ + public static String findNativeLibrary(String libName, Package pkg, FileSystem fs) { + var libNameWithSuffix = System.mapLibraryName(libName); + for (var dir : searchPath(pkg, fs)) { + if (!fs.exists(dir)) { + return null; + } + var nativeLib = fs.getChild(dir, libNameWithSuffix); + if (fs.exists(nativeLib)) { + return fs.getAbsolutePath(nativeLib); + } + } + return null; + } + + /** Returns set of native libraries for the given package for the current OS and architecture. */ + public static Set listAllNativeLibraries(Package pkg, FileSystem fs) { + var nativeLibs = new HashSet(); + for (var dir : searchPath(pkg, fs)) { + if (!fs.exists(dir)) { + return nativeLibs; + } + try { + fs.list(dir) + .forEach( + file -> { + var fname = fs.getName(file); + if (fs.isRegularFile(file) && fname.endsWith(nativeLibSuffix())) { + nativeLibs.add(file); + } + }); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + return nativeLibs; + } + + private static String nativeLibSuffix() { + var libName = System.mapLibraryName(""); + return libName.substring(libName.lastIndexOf('.')); + } + + private static List searchPath(Package pkg, FileSystem fs) { + var searchPath = new ArrayList(); + var arch = arch(); + var osName = simpleOsName(); + var libDir = pkg.nativeLibraryDir(); + searchPath.add(libDir); + searchPath.add(fs.getChild(libDir, arch)); + searchPath.add(fs.getChild(fs.getChild(libDir, arch), osName)); + return searchPath; + } + + private static String simpleOsName() { + var osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); + if (osName.contains(" ")) { + // Strip version + osName = osName.substring(0, osName.indexOf(' ')); + } + if (osName.contains("linux")) { + return "linux"; + } else if (osName.contains("mac")) { + return "macos"; + } else if (osName.contains("windows")) { + return "windows"; + } else { + throw new IllegalStateException("Unsupported OS: " + osName); + } + } + + private static String arch() { + var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); + return arch.replace("x86_64", "amd64"); + } +} diff --git a/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala b/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala index e6f99509253e..b593f7977b25 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala @@ -64,6 +64,12 @@ trait FileSystem[F] { */ def getSegments(file: F): java.lang.Iterable[String] + /** Returns absolute path of the given file. + * @param file + * @return + */ + def getAbsolutePath(file: F): String + /** Gets the name of the given file. * * @param file @@ -144,6 +150,8 @@ trait FileSystem[F] { object FileSystem { + val defaultFs = Default + /** Exposes [[FileSystem]] operations through method call syntax. * All methods have the same semantics as the corresponding [[FileSystem]] * methods. @@ -231,5 +239,9 @@ object FileSystem { Files .readAttributes(file.toPath, classOf[BasicFileAttributes]) .creationTime() + + override def getAbsolutePath(file: File): String = { + file.getAbsolutePath + } } } diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala index 32279c2764b0..a13c44bd81ba 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala @@ -40,6 +40,7 @@ class Package[F]( val configFile: F = root.getChild(Package.configFileName) val thumbFile: F = root.getChild(Package.thumbFileName) val polyglotDir: F = root.getChild(Package.polyglotExtensionsDirName) + val nativeLibraryDir: F = polyglotDir.getChild(Package.nativeLibraryDirName) val internalDirectory: F = root.getChild(Package.internalDirName) val irCacheDirectory: F = internalDirectory .getChild(Package.cacheDirName) @@ -599,6 +600,7 @@ object Package { val configFileName = "package.yaml" val sourceDirName = "src" val polyglotExtensionsDirName = "polyglot" + val nativeLibraryDirName = "lib" val internalDirName = ".enso" val mainFileName = "Main.enso" val thumbFileName = "thumb.png" diff --git a/project/JARUtils.scala b/project/JARUtils.scala new file mode 100644 index 000000000000..8d7a122bf2d7 --- /dev/null +++ b/project/JARUtils.scala @@ -0,0 +1,128 @@ +import sbt.{IO, Tracked} +import sbt.std.Streams +import sbt.util.{CacheStoreFactory, FileInfo} + +import java.io.IOException +import java.nio.file.{Files, Path} +import java.util.jar.{JarEntry, JarFile, JarOutputStream} +import scala.util.{Try, Using} + +object JARUtils { + + /** Extracts all file entries starting with `extractPrefix` from `inputJarPath` to `extractedFilesDir`, + * optionally renaming them with `renameFunc`. + * The rest is copied into `outputJarPath`. + * + * @param inputJarPath Path to the JAR archive. Will not be modified. + * @param extractPrefix Prefix of the files to extract. + * @param outputJarPath Path to the output JAR. Input JAR will be copied here without the files + * starting with `extractPrefix`. + * @param extractedFilesDir Destination directory for the extracted files. The prefix from the + * extracted files is tripped. + * @param renameFunc Function that renames the extracted files. The extracted file name is taken + * from the jar entry, and thus may contain slashes. If None is returned, the + * file is ignored and not extracted. + */ + def extractFilesFromJar( + inputJarPath: Path, + extractPrefix: String, + outputJarPath: Path, + extractedFilesDir: Path, + renameFunc: String => Option[String], + logger: sbt.util.Logger, + cacheStoreFactory: CacheStoreFactory + ): Unit = { + val dependencyStore = cacheStoreFactory.make("extract-jar-files") + // Make sure that the actual file extraction is done only iff some of the cached files change. + val cachedFiles = Set( + inputJarPath.toFile + ) + var shouldExtract = false + Tracked.diffInputs(dependencyStore, FileInfo.hash)(cachedFiles) { report => + shouldExtract = + report.modified.nonEmpty || report.removed.nonEmpty || report.added.nonEmpty + } + + if (!shouldExtract) { + logger.debug("No changes in the input JAR, skipping extraction.") + return + } else { + logger.info( + s"Extracting files with prefix '${extractPrefix}' from $inputJarPath to $extractedFilesDir." + ) + } + + ensureDirExistsAndIsClean(outputJarPath.getParent, logger) + ensureDirExistsAndIsClean(extractedFilesDir, logger) + Using(new JarFile(inputJarPath.toFile)) { inputJar => + Using(new JarOutputStream(Files.newOutputStream(outputJarPath))) { + outputJar => + inputJar.stream().forEach { entry => + if (entry.getName.startsWith(extractPrefix) && !entry.isDirectory) { + renameFunc(entry.getName) match { + case Some(strippedEntryName) => + assert(!strippedEntryName.startsWith("/")) + assert(extractedFilesDir.toFile.exists) + val destFile = extractedFilesDir.resolve(strippedEntryName) + if (!destFile.getParent.toFile.exists) { + Files.createDirectories(destFile.getParent) + } + Using(inputJar.getInputStream(entry)) { is => + Files.copy(is, destFile) + }.recover({ case e: IOException => + logger.err( + s"Failed to extract $entry to $destFile: ${e.getMessage}" + ) + e.printStackTrace(System.err) + }) + case None => () + } + } else { + outputJar.putNextEntry(new JarEntry(entry.getName)) + Using(inputJar.getInputStream(entry)) { is => + is.transferTo(outputJar) + }.recover({ case e: IOException => + logger.err( + s"Failed to copy $entry to output JAR: ${e.getMessage}" + ) + e.printStackTrace(System.err) + }) + outputJar.closeEntry() + } + } + }.recover({ case e: IOException => + logger.err( + s"Failed to create output JAR at $outputJarPath: ${e.getMessage}" + ) + e.printStackTrace(System.err) + }) + }.recover({ case e: IOException => + logger.err( + s"Failed to extract files from $inputJarPath to $extractedFilesDir: ${e.getMessage}" + ) + e.printStackTrace(System.err) + }) + } + + private def ensureDirExistsAndIsClean( + path: Path, + logger: sbt.util.Logger + ): Unit = { + require(path != null) + val dir = path.toFile + if (dir.exists && dir.isDirectory) { + // Clean previous contents + IO.delete(IO.listFiles(dir)) + } else { + try { + IO.createDirectory(dir) + } catch { + case e: IOException => + logger.err( + s"Failed to create directory $path: ${e.getMessage}" + ) + e.printStackTrace(System.err) + } + } + } +} diff --git a/project/StdBits.scala b/project/StdBits.scala index 2e34470e3b84..e52de6c98e88 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -19,11 +19,13 @@ object StdBits { * @param ignoreScalaLibrary whether to ignore Scala dependencies that are * added by default be SBT and are not relevant in * pure-Java projects + * @param ignoreDependency A dependency that should be ignored - not copied to the destination */ def copyDependencies( destination: File, providedJarNames: Seq[String], - ignoreScalaLibrary: Boolean + ignoreScalaLibrary: Boolean, + ignoreDependency: Option[ModuleID] = None ): Def.Initialize[Task[Unit]] = Def.task { val libraryUpdates = (Compile / update).value @@ -48,12 +50,26 @@ object StdBits { !graalVmOrgs.contains(orgName) }) ) + val moduleFilter = ignoreDependency match { + case None => graalModuleFilter + case Some(ignoreDepID) => + DependencyFilter.moduleFilter( + organization = new SimpleFilter(orgName => { + !graalVmOrgs.contains( + orgName + ) && orgName != ignoreDepID.organization + }), + name = new SimpleFilter(nm => { + nm != ignoreDepID.name + }) + ) + } val unmanagedFiles = (Compile / unmanagedJars).value.map(_.data) val relevantFiles = libraryUpdates .select( configuration = configFilter, - module = graalModuleFilter, + module = moduleFilter, artifact = DependencyFilter.artifactFilter() ) ++ unmanagedFiles val dependencyStore = @@ -86,6 +102,75 @@ object StdBits { } } + /** Extract native libraries from `opencv.jar` and put them under + * `Standard/Image/polyglot/lib` directory. The minimized `opencv.jar` will + * be put under `Standard/Image/polyglot/java` directory. + * @param imagePolyglotRoot root dir of Std image polyglot dir + * @param imageNativeLibs root dir of Std image lib dir + * @return + */ + def extractNativeLibsFromOpenCV( + imagePolyglotRoot: File, + imageNativeLibs: File, + opencvVersion: String + ): Def.Initialize[Task[Unit]] = Def.task { + // Ensure dependencies are first copied. + val _ = StdBits + .copyDependencies( + imagePolyglotRoot, + Seq("std-image.jar", "opencv.jar"), + ignoreScalaLibrary = true, + ignoreDependency = Some("org.openpnp" % "opencv" % opencvVersion) + ) + .value + val extractPrefix = "nu/pattern/opencv" + + // Make sure that the native libs in the `lib` directory complies with + // `org.enso.interpreter.runtime.NativeLibraryFinder` + def renameFunc(entryName: String): Option[String] = { + val strippedEntryName = entryName.substring(extractPrefix.length + 1) + if ( + strippedEntryName.contains("linux/ARM") || + strippedEntryName.contains("linux/x86_32") || + strippedEntryName.contains("README.md") + ) { + None + } else { + Some( + strippedEntryName + .replace("linux/x86_64", "amd64") + .replace("windows/x86_64", "amd64") + .replace("windows/x86_32", "x86_32") + .replace("osx/ARMv8", "aarch64") + .replace("osx/x86_64", "amd64") + ) + } + } + + val logger = streams.value.log + val openCvJar = JPMSUtils + .filterModulesFromUpdate( + update.value, + Seq("org.openpnp" % "opencv" % opencvVersion), + logger, + moduleName.value, + scalaBinaryVersion.value, + shouldContainAll = true + ) + .head + val outputJarPath = (imagePolyglotRoot / "opencv.jar").toPath + val extractedFilesDir = imageNativeLibs.toPath + JARUtils.extractFilesFromJar( + openCvJar.toPath, + extractPrefix, + outputJarPath, + extractedFilesDir, + renameFunc, + logger, + streams.value.cacheStoreFactory + ) + } + private def updateDependency( jar: File, destinationDir: File, diff --git a/std-bits/image/src/main/java/org/enso/image/Codecs.java b/std-bits/image/src/main/java/org/enso/image/Codecs.java index 9706d6b38878..83f74eb7d1af 100644 --- a/std-bits/image/src/main/java/org/enso/image/Codecs.java +++ b/std-bits/image/src/main/java/org/enso/image/Codecs.java @@ -11,7 +11,7 @@ public class Codecs { public static final int READ_FLAG_EMPTY = -127; static { - OpenCV.loadLocally(); + OpenCV.loadShared(); } /** An error occurred when reading a file. */ diff --git a/std-bits/image/src/main/java/org/enso/image/data/Image.java b/std-bits/image/src/main/java/org/enso/image/data/Image.java index 74d0f6d25072..2fd690060001 100644 --- a/std-bits/image/src/main/java/org/enso/image/data/Image.java +++ b/std-bits/image/src/main/java/org/enso/image/data/Image.java @@ -11,7 +11,7 @@ public class Image { static { - OpenCV.loadLocally(); + OpenCV.loadShared(); } private static final byte MAX_SIGNED_BYTE = -1; diff --git a/std-bits/image/src/main/java/org/enso/image/data/Matrix.java b/std-bits/image/src/main/java/org/enso/image/data/Matrix.java index 65d46569515d..c87c6d830193 100644 --- a/std-bits/image/src/main/java/org/enso/image/data/Matrix.java +++ b/std-bits/image/src/main/java/org/enso/image/data/Matrix.java @@ -7,7 +7,7 @@ public class Matrix { static { - OpenCV.loadLocally(); + OpenCV.loadShared(); } /**