Run whole test/Base_Tests in native image runner (#10296)

enso-org · Jun 21, 2024 · fe2cf49 · fe2cf49
1 parent b5641aa
commit fe2cf49
Show file tree

Hide file tree

Showing 48 changed files with 1,576 additions and 284 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -33,7 +33,7 @@
       "request": "launch",
       "name": "Launch Native Image",
       "nativeImagePath": "${workspaceFolder}/runner",
-      "args": "--help"
+      "args": "--run ${file}"
     }
   ]
 }
diff --git a/build.sbt b/build.sbt
@@ -2549,8 +2549,10 @@ lazy val `engine-runner` = project
       if (smallJdkDirectory.exists()) {
         IO.delete(smallJdkDirectory)
       }
-      val JS_MODULES =
-        "org.graalvm.nativeimage,org.graalvm.nativeimage.builder,org.graalvm.nativeimage.base,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.objectfile,org.graalvm.nativeimage.pointsto,com.oracle.graal.graal_enterprise,com.oracle.svm.svm_enterprise,jdk.compiler.graal,jdk.httpserver,java.naming,java.net.http"
+      val NI_MODULES =
+        "org.graalvm.nativeimage,org.graalvm.nativeimage.builder,org.graalvm.nativeimage.base,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.objectfile,org.graalvm.nativeimage.pointsto,com.oracle.graal.graal_enterprise,com.oracle.svm.svm_enterprise"
+      val JDK_MODULES =
+        "jdk.localedata,jdk.compiler.graal,jdk.httpserver,java.naming,java.net.http"
       val DEBUG_MODULES  = "jdk.jdwp.agent"
       val PYTHON_MODULES = "jdk.security.auth,java.naming"
 
@@ -2578,7 +2580,7 @@ lazy val `engine-runner` = project
       }
 
       val exec =
-        s"$jlink --module-path ${modules.mkString(":")} --output $smallJdkDirectory --add-modules $JS_MODULES,$DEBUG_MODULES,$PYTHON_MODULES"
+        s"$jlink --module-path ${modules.mkString(":")} --output $smallJdkDirectory --add-modules $NI_MODULES,$JDK_MODULES,$DEBUG_MODULES,$PYTHON_MODULES"
       val exitCode = scala.sys.process.Process(exec).!
 
       if (exitCode != 0) {
@@ -2610,6 +2612,9 @@ lazy val `engine-runner` = project
           additionalOptions = Seq(
             "-Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.NoOpLog",
             "-H:IncludeResources=.*Main.enso$",
+            "-H:+AddAllCharsets",
+            "-H:+IncludeAllLocales",
+            "-ea",
             // useful perf & debug switches:
             // "-g",
             // "-H:+SourceLevelDebug",
@@ -2660,6 +2665,7 @@ lazy val `engine-runner` = project
   .dependsOn(`logging-service`)
   .dependsOn(`logging-service-logback` % Runtime)
   .dependsOn(`polyglot-api`)
+  .dependsOn(`enso-test-java-helpers`)
 
 lazy val buildSmallJdk =
   taskKey[File]("Build a minimal JDK used for native image generation")
@@ -3567,7 +3573,7 @@ ThisBuild / buildEngineDistributionNoIndex := {
 lazy val runEngineDistribution =
   inputKey[Unit]("Run or --debug the engine distribution with arguments")
 runEngineDistribution := {
-  buildEngineDistribution.value
+  buildEngineDistributionNoIndex.value
   val args: Seq[String] = spaceDelimited("<arg>").parsed
   DistributionPackage.runEnginePackage(
     engineDistributionRoot.value,

diff --git a/build/build/src/engine/context.rs b/build/build/src/engine/context.rs
@@ -630,28 +630,12 @@ pub async fn runner_sanity_test(
     repo_root: &crate::paths::generated::RepoRoot,
     enso_java: Option<&str>,
 ) -> Result {
-    let factorial_input = "6";
-    let factorial_expected_output = "720";
     let engine_package = repo_root.built_distribution.enso_engine_triple.engine_package.as_path();
     // The engine package is necessary for running the native runner.
     ide_ci::fs::tokio::require_exist(engine_package).await?;
-    let output = Command::new(&repo_root.runner)
-        .args([
-            "--run",
-            repo_root.engine.runner.src.test.resources.factorial_enso.as_str(),
-            factorial_input,
-        ])
-        .set_env_opt(ENSO_JAVA, enso_java)?
-        .set_env(ENSO_DATA_DIRECTORY, engine_package)?
-        .run_stdout()
-        .await?;
-    ensure!(
-        output.contains(factorial_expected_output),
-        "Native runner output does not contain expected result '{factorial_expected_output}'. Output:\n{output}",
-    );
     if enso_java.is_none() {
         let test_base = Command::new(&repo_root.runner)
-            .args(["--run", repo_root.test.join("Base_Tests").as_str(), "^Text"])
+            .args(["--run", repo_root.test.join("Base_Tests").as_str()])
             .set_env_opt(ENSO_JAVA, enso_java)?
             .set_env(ENSO_DATA_DIRECTORY, engine_package)?
             .run_stdout()

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Base_64.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Base_64.enso
@@ -4,6 +4,8 @@ import project.Errors.Encoding_Error.Encoding_Error
 from project.Data.Text.Extensions import all
 
 polyglot java import java.util.Base64
+polyglot java import java.util.Base64.Decoder
+polyglot java import java.util.Base64.Encoder
 
 ## A helper utility for handling base64 encoding.
 type Base_64

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso
@@ -30,6 +30,7 @@ from project.Data.Text.Extensions import all
 from project.Metadata.Choice import Option
 from project.Metadata.Widget import Single_Choice
 
+polyglot java import com.fasterxml.jackson.core.JsonLocation
 polyglot java import com.fasterxml.jackson.core.JsonProcessingException
 polyglot java import com.fasterxml.jackson.databind.JsonNode
 polyglot java import com.fasterxml.jackson.databind.node.ArrayNode

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Regression.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Regression.enso
@@ -9,6 +9,7 @@ import project.Nothing.Nothing
 import project.Panic.Panic
 
 polyglot java import org.enso.base.statistics.FitError
+polyglot java import org.enso.base.statistics.LinearModel
 polyglot java import org.enso.base.statistics.Regression
 
 type Model

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Extensions.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Extensions.enso
@@ -544,7 +544,7 @@ Text.replace self term:(Text | Regex) replacement:Text (case_sensitivity:Case_Se
    Applies the specified cleansings to the text.
 
    Arguments:
-   - remove: A vector of the named patterns to cleanse from the text. The named patterns are 
+   - remove: A vector of the named patterns to cleanse from the text. The named patterns are
       applied in the order they are provided. The same named pattern can be used multiple
       times. The named patterns are:
       - ..Leading_Whitespace: Removes all whitespace from the start of the string.

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date.enso
@@ -34,6 +34,7 @@ polyglot java import java.time.DateTimeException
 polyglot java import java.time.temporal.ChronoField
 polyglot java import java.time.temporal.IsoFields
 polyglot java import org.enso.base.Time_Utils
+polyglot java import org.enso.base.Time_Utils.AdjustOp
 
 ## PRIVATE
    Constructs a new Date from a year, month, and day.

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date_Period.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date_Period.enso
@@ -11,7 +11,10 @@ polyglot java import java.time.temporal.TemporalAdjuster
 polyglot java import java.time.temporal.TemporalAdjusters
 polyglot java import java.time.temporal.TemporalUnit
 polyglot java import org.enso.base.time.CustomTemporalUnits
+polyglot java import org.enso.base.time.Date_Utils
 polyglot java import org.enso.base.time.Date_Period_Utils
+polyglot java import org.enso.base.time.Date_Time_Utils
+polyglot java import org.enso.base.time.Time_Of_Day_Utils
 polyglot java import org.enso.base.Time_Utils
 
 ## Represents a unit of time longer on the scale of days (longer than a day).

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML.enso
@@ -25,19 +25,18 @@ from project.Data.Range.Extensions import all
 from project.Data.Text.Extensions import all
 from project.Metadata import Display, make_single_choice, Widget
 
-polyglot java import java.io.StringReader
 polyglot java import java.lang.Exception as JException
-polyglot java import javax.xml.parsers.DocumentBuilder
-polyglot java import javax.xml.parsers.DocumentBuilderFactory
+polyglot java import javax.xml.xpath.XPath
 polyglot java import javax.xml.xpath.XPathConstants
 polyglot java import javax.xml.xpath.XPathFactory
 polyglot java import org.enso.base.XML_Utils
+polyglot java import org.w3c.dom.Attr
 polyglot java import org.w3c.dom.Document
 polyglot java import org.w3c.dom.Element
 polyglot java import org.w3c.dom.Node
 polyglot java import org.w3c.dom.NodeList
+polyglot java import org.w3c.dom.NamedNodeMap
 polyglot java import org.w3c.dom.Text as Java_Text
-polyglot java import org.xml.sax.InputSource
 polyglot java import org.xml.sax.SAXException
 polyglot java import org.xml.sax.SAXParseException
 
@@ -75,7 +74,8 @@ type XML_Document
     from_stream : Input_Stream -> XML_Document ! XML_Error
     from_stream input_stream:Input_Stream =
         XML_Error.handle_java_exceptions <|
-           input_stream.with_java_stream java_stream-> XML_Document.from_source java_stream
+           input_stream.with_java_stream java_stream->
+                XML_Document.Value (XML_Utils.parseStream java_stream)
 
     ## GROUP Conversions
        ICON convert
@@ -93,20 +93,13 @@ type XML_Document
     from_text : Text -> XML_Document ! XML_Error
     from_text xml_string:Text =
         XML_Error.handle_java_exceptions <|
-            string_reader = StringReader.new xml_string
-            XML_Document.from_source (InputSource.new string_reader)
+            XML_Document.Value (XML_Utils.parseString xml_string)
 
     ## PRIVATE
-       Read XML from an input source.
-    from_source : Any -> XML_Document ! XML_Error
-    from_source input_source =
-        document_builder_factory = DocumentBuilderFactory.newInstance
-        document_builder = document_builder_factory.newDocumentBuilder
-        XML_Utils.setCustomErrorHandler document_builder
-        XML_Document.Value (document_builder.parse input_source)
+       Wrap Java's Document to XML_Document
+    new doc:Document = XML_Document.Value doc
 
-    ## PRIVATE
-    Value (java_document:Document)
+    private Value (java_document:Document)
 
 
     ## GROUP Metadata

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_Secret.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_Secret.enso
@@ -24,6 +24,10 @@ from project.Metadata import make_single_choice
 
 polyglot java import org.enso.base.enso_cloud.EnsoSecretHelper
 polyglot java import org.enso.base.enso_cloud.HideableValue
+polyglot java import org.enso.base.enso_cloud.HideableValue.PlainValue
+polyglot java import org.enso.base.enso_cloud.HideableValue.SecretValue
+polyglot java import org.enso.base.enso_cloud.HideableValue.ConcatValues
+polyglot java import org.enso.base.enso_cloud.HideableValue.Base64EncodeValue
 
 ## A reference to a secret stored in the Enso Cloud.
 type Enso_Secret

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Array_Like_Helpers.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Array_Like_Helpers.enso
@@ -78,7 +78,7 @@ slice vector start end = @Builtin_Method "Array_Like_Helpers.slice"
        `Map_Error`.
      - No_Wrap: The first error is thrown, and is not wrapped in
        `Map_Error`.
-     - Report_Warning: The result for that element is `Nothing`, 
+     - Report_Warning: The result for that element is `Nothing`,
        the error is attached as a warning. Currently unimplemented.
      - Ignore: The result is `Nothing`, and the error is
        ignored.

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Extra_Imports.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Extra_Imports.enso
@@ -14,3 +14,18 @@ polyglot java import org.enso.base.text.CaseFoldedString.Grapheme
 
 # needed by Comparator_Spec:
 polyglot java import org.enso.base.ObjectComparator
+
+# often used in tests
+polyglot java import java.util.ArrayList
+polyglot java import java.util.Map
+polyglot java import java.nio.CharBuffer
+polyglot java import java.nio.file.Path
+polyglot java import java.io.FileInputStream
+polyglot java import java.io.FileOutputStream
+polyglot java import java.math.BigInteger
+polyglot java import java.time.LocalDate
+polyglot java import java.time.LocalDateTime
+polyglot java import java.util.function.Function
+polyglot java import java.lang.Thread
+polyglot java import java.lang.Thread.State
+polyglot java import java.lang.Float
diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Meta/Enso_Project.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Meta/Enso_Project.enso
@@ -45,7 +45,7 @@ type Project_Description
        Arguments:
        - prim_root_file: The primitive root file of the project.
        - prim_config: The primitive config of the project.
-    private Value prim_root_file prim_config
+    private Value prim_root_file ns:Text n:Text
 
     ## GROUP Metadata
        ICON folder
@@ -78,7 +78,7 @@ type Project_Description
 
              enso_project.name
     name : Text
-    name self = self.prim_config.name
+    name self = self.n
 
     ## GROUP Metadata
        ICON metadata
@@ -89,7 +89,7 @@ type Project_Description
 
              enso_project.namespace
     namespace : Text
-    namespace self = self.prim_config.namespace
+    namespace self = self.ns
 
 ## ICON enso_icon
    Returns the Enso project description for the project that the engine was

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso
@@ -30,12 +30,17 @@ from project.Data.Json.Extensions import all
 
 polyglot java import java.lang.Exception as JException
 polyglot java import java.net.http.HttpClient
+polyglot java import java.net.http.HttpClient.Redirect
+polyglot java import java.net.http.HttpClient.Version
+polyglot java import java.net.http.HttpClient.Builder as ClientBuilder
 polyglot java import java.net.http.HttpRequest
 polyglot java import java.net.http.HttpRequest.BodyPublisher
+polyglot java import java.net.http.HttpRequest.BodyPublishers
+polyglot java import java.net.http.HttpRequest.Builder
 polyglot java import java.net.InetSocketAddress
 polyglot java import java.net.ProxySelector
-polyglot java import java.nio.file.Path
 polyglot java import javax.net.ssl.SSLContext
+polyglot java import org.enso.base.file_system.File_Utils
 polyglot java import org.enso.base.enso_cloud.EnsoSecretHelper
 polyglot java import org.enso.base.net.http.MultipartBodyBuilder
 polyglot java import org.enso.base.net.http.UrlencodedBodyBuilder
@@ -242,7 +247,7 @@ resolve_body_to_publisher_and_boundary body:Request_Body =
             json.if_not_error <|
                 Pair.new (body_publishers.ofString json) Nothing
         Request_Body.Binary file ->
-            path = Path.of file.path
+            path = File_Utils.toPath file.path
             Pair.new (body_publishers.ofFile path) Nothing
         Request_Body.Form_Data form_data url_encoded ->
             build_form_body_publisher form_data url_encoded

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso
@@ -25,6 +25,10 @@ from project.Data.Text.Extensions import all
 from project.Metadata import Display, Widget
 from project.Network.HTTP.Response_Body import decode_format_selector
 
+polyglot java import java.net.http.HttpHeaders
+polyglot java import org.enso.base.enso_cloud.EnsoHttpResponse
+polyglot java import java.util.Optional
+
 type Response
     ## PRIVATE
 

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso
@@ -42,13 +42,12 @@ from project.System.File_Format import Auto_Detect, File_Format
 polyglot java import java.io.File as Java_File
 polyglot java import java.io.InputStream as Java_Input_Stream
 polyglot java import java.io.OutputStream as Java_Output_Stream
-polyglot java import java.nio.file.FileSystems
-polyglot java import java.nio.file.Path
 polyglot java import java.nio.file.StandardCopyOption
 polyglot java import java.nio.file.StandardOpenOption
 polyglot java import java.time.ZonedDateTime
 polyglot java import org.enso.base.DryRunFileManager
 polyglot java import org.enso.base.file_system.FileSystemSPI
+polyglot java import org.enso.base.file_system.File_Utils
 
 ## PRIVATE
 file_types : Vector
@@ -795,11 +794,10 @@ type File
             _ ->
                 used_filter = if recursive.not || name_filter.contains "**" then name_filter else
                     (if name_filter.starts_with "*" then "*" else "**/") + name_filter
-                fs = FileSystems.getDefault
-                matcher = fs.getPathMatcher "glob:"+used_filter
+                matcher = File_Utils.matchPath "glob:"+used_filter
                 all_files.filter file->
                     pathStr = self.relativize file . path
-                    matcher.matches (Path.of pathStr)
+                    File_Utils.matches matcher pathStr
 
     ## GROUP Metadata
        ICON metadata
@@ -894,11 +892,11 @@ Writable_File.from (that : File) = if Data_Link.is_data_link that then Data_Link
 ## PRIVATE
 local_file_copy (source : File) (destination : File) (replace_existing : Boolean) -> Nothing =
     File_Error.handle_java_exceptions source <|
-        copy_options = if replace_existing then [StandardCopyOption.REPLACE_EXISTING] else []
+        copy_options = if replace_existing then [StandardCopyOption.REPLACE_EXISTING.to_text] else []
         source.copy_builtin destination copy_options
 
 ## PRIVATE
 local_file_move (source : File) (destination : File) (replace_existing : Boolean) -> Nothing =
     File_Error.handle_java_exceptions source <|
-        copy_options = if replace_existing then [StandardCopyOption.REPLACE_EXISTING] else []
+        copy_options = if replace_existing then [StandardCopyOption.REPLACE_EXISTING.to_text] else []
         source.move_builtin destination copy_options
diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/File_Access.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/File_Access.enso
@@ -49,18 +49,20 @@ type File_Access
     ## PRIVATE
 
        Convert this object into a representation understandable by the JVM.
-    to_java : StandardOpenOption
-    to_java self = case self of
-        File_Access.Append            -> StandardOpenOption.APPEND
-        File_Access.Create            -> StandardOpenOption.CREATE
-        File_Access.Create_New        -> StandardOpenOption.CREATE_NEW
-        File_Access.Delete_On_Close   -> StandardOpenOption.DELETE_ON_CLOSE
-        File_Access.Dsync             -> StandardOpenOption.DSYNC
-        File_Access.Read              -> StandardOpenOption.READ
-        File_Access.Sparse            -> StandardOpenOption.SPARSE
-        File_Access.Sync              -> StandardOpenOption.SYNC
-        File_Access.Truncate_Existing -> StandardOpenOption.TRUNCATE_EXISTING
-        File_Access.Write             -> StandardOpenOption.WRITE
+    to_java : Text
+    to_java self =
+        java_option = case self of
+            File_Access.Append            -> StandardOpenOption.APPEND
+            File_Access.Create            -> StandardOpenOption.CREATE
+            File_Access.Create_New        -> StandardOpenOption.CREATE_NEW
+            File_Access.Delete_On_Close   -> StandardOpenOption.DELETE_ON_CLOSE
+            File_Access.Dsync             -> StandardOpenOption.DSYNC
+            File_Access.Read              -> StandardOpenOption.READ
+            File_Access.Sparse            -> StandardOpenOption.SPARSE
+            File_Access.Sync              -> StandardOpenOption.SYNC
+            File_Access.Truncate_Existing -> StandardOpenOption.TRUNCATE_EXISTING
+            File_Access.Write             -> StandardOpenOption.WRITE
+        java_option.to_text
 
     ## PRIVATE
     ensure_only_allowed_options (operation_name : Text) (allowed_options : Vector) (got_options : Vector) ~action =

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/File_Permissions.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/File_Permissions.enso
@@ -3,6 +3,8 @@ import project.Data.Text.Text
 import project.Data.Vector.Vector
 
 polyglot java import java.nio.file.attribute.PosixFilePermission
+polyglot java import java.nio.file.attribute.PosixFilePermissions
+polyglot java import java.util.Set
 
 type Permission
     ## Permission for read access for a given entity.
@@ -101,7 +103,8 @@ type File_Permissions
     ## PRIVATE
        ADVANCED
        Converts a Java `Set` of Java `PosixFilePermission` to `File_Permissions`.
-    from_java_set java_set =
+    from_java_set permissions:Text =
+        java_set = PosixFilePermissions.fromString permissions
         vecs = Vector.build_multiple 3 builders->
             owner = builders.at 0
             group = builders.at 1