From be7f441d42645b2c209dfd5f0dea4ca341cccabf Mon Sep 17 00:00:00 2001 From: Vojin Jovanovic Date: Thu, 13 Feb 2025 10:24:59 +0100 Subject: [PATCH 1/2] Options to include all classpath elements --- .../nativeimage/hosted/RuntimeReflection.java | 2 +- .../impl/RuntimeReflectionSupport.java | 2 +- substratevm/CHANGELOG.md | 1 + .../graal/pointsto/ClassInclusionPolicy.java | 18 ++- .../graal/pointsto/PointsToAnalysis.java | 2 +- .../com/oracle/svm/core/SubstrateOptions.java | 14 ++ .../svm/core/hub/ClassForNameSupport.java | 3 +- .../svm/hosted/ClassLoaderSupportImpl.java | 6 +- .../hosted/NativeImageClassLoaderSupport.java | 144 +++++++++++++----- .../svm/hosted/NativeImageGenerator.java | 53 +++++++ .../hosted/reflect/ReflectionDataBuilder.java | 15 +- .../SubstitutionReflectivityFilter.java | 3 + .../com/oracle/svm/util/ReflectionUtil.java | 2 +- vm/mx.vm/mx_vm_benchmark.py | 40 ++--- 14 files changed, 232 insertions(+), 73 deletions(-) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeReflection.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeReflection.java index 07ef05fedfff..fc3b93d8139b 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeReflection.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeReflection.java @@ -231,7 +231,7 @@ public static void registerAllFields(Class declaringClass) { * @since 23.0 */ public static void registerAllDeclaredFields(Class declaringClass) { - ImageSingletons.lookup(RuntimeReflectionSupport.class).registerAllDeclaredFieldsQuery(ConfigurationCondition.alwaysTrue(), declaringClass); + ImageSingletons.lookup(RuntimeReflectionSupport.class).registerAllDeclaredFields(ConfigurationCondition.alwaysTrue(), declaringClass); } /** diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java index 68e78b2c975b..071987452994 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java @@ -48,7 +48,7 @@ public interface RuntimeReflectionSupport extends ReflectionRegistry { void registerAllFieldsQuery(ConfigurationCondition condition, Class clazz); - void registerAllDeclaredFieldsQuery(ConfigurationCondition condition, Class clazz); + void registerAllDeclaredFields(ConfigurationCondition condition, Class clazz); void registerAllConstructorsQuery(ConfigurationCondition condition, boolean queriedOnly, Class clazz); diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index bd900af80a42..349a4e9476e6 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -29,6 +29,7 @@ At runtime, premain runtime options are set along with main class' arguments in * (GR-60081) Native Image now targets `armv8.1-a` by default on AArch64. Use `-march=compatibility` for best compatibility or `-march=native` for best performance if the native executable is deployed on the same machine or on a machine with the same CPU features. To list all available machine types, use `-march=list`. * (GR-60234) Remove `"customTargetConstructorClass"` field from the serialization JSON metadata. All possible constructors are now registered by default when registering a type for serialization. `RuntimeSerialization.registerWithTargetConstructorClass` is now deprecated. * (GR-60237) Include serialization JSON reachability metadata as part of reflection metadata by introducing the `"serializable"` flag for reflection entries. +* (GR-54953) Add options `-H:IncludeAllMetadataForClassPathEntry`, `-H:+IncludeAllForClassPath`, and `-H:IncludeAllMetadataForModule` for bulk inclusion of reachability metadata. ## GraalVM for JDK 23 (Internal Version 24.1.0) * (GR-51520) The old class initialization strategy, which was deprecated in GraalVM for JDK 22, is removed. The option `StrictImageHeap` no longer has any effect. diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ClassInclusionPolicy.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ClassInclusionPolicy.java index d2da3fd59396..46826a3a0d7b 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ClassInclusionPolicy.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ClassInclusionPolicy.java @@ -56,12 +56,28 @@ public void setBigBang(BigBang bb) { this.bb = bb; } + public static boolean isClassIncludedBase(Class cls) { + if (Feature.class.isAssignableFrom(cls)) { + return false; + } + + if (AnnotationAccess.isAnnotationPresent(cls, TargetClass.class)) { + return false; + } + try { + Class enclosingClass = cls.getEnclosingClass(); + return enclosingClass == null || isClassIncludedBase(enclosingClass); + } catch (LinkageError e) { + return true; + } + } + /** * Determine if the given class needs to be included in the image according to the policy. */ public boolean isClassIncluded(Class cls) { Class enclosingClass = cls.getEnclosingClass(); - return !Feature.class.isAssignableFrom(cls) && !AnnotationAccess.isAnnotationPresent(cls, TargetClass.class) && (enclosingClass == null || isClassIncluded(enclosingClass)); + return isClassIncludedBase(cls) && (enclosingClass == null || isClassIncluded(enclosingClass)); } /** diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java index dfc9d5df06e4..ccef46168d6a 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java @@ -666,7 +666,7 @@ public void onTypeInstantiated(AnalysisType type) { /* Register the type as instantiated with all its super types. */ assert type.isInstantiated() : type; - AnalysisError.guarantee(type.isArray() || (type.isInstanceClass() && !type.isAbstract())); + AnalysisError.guarantee(type.isArray() || (type.isInstanceClass() && !type.isAbstract()), "Type %s must be either an array, or a non abstract instance class", type.getName()); TypeState typeState = TypeState.forExactType(this, type, true); TypeState typeStateNonNull = TypeState.forExactType(this, type, false); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 132e39a3744a..29a4ca44ab50 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -1316,6 +1316,9 @@ public enum ReportingMode { @Option(help = "Include all classes, methods, and fields from given modules", type = OptionType.Debug) // public static final HostedOptionKey IncludeAllFromModule = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build()); + @Option(help = "Include all classes, methods, fields, and resources from given modules for dynamic access", type = OptionType.Debug) // + public static final HostedOptionKey EnableDynamicAccessForModule = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build()); + @Option(help = "Include all classes, methods, fields, and resources from given paths", type = OptionType.Debug) // public static final HostedOptionKey IncludeAllFromPath = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build()); @@ -1323,9 +1326,20 @@ public enum ReportingMode { public static final HostedOptionKey IncludeAllFromPackage = new HostedOptionKey<>( AccumulatingLocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); + @Option(help = "Include all classes, methods and fields from the given packages for dynamic access", type = OptionType.Debug) // + public static final HostedOptionKey EnableDynamicAccessForPackage = new HostedOptionKey<>( + AccumulatingLocatableMultiOptionValue.Strings.buildWithCommaDelimiter()); + + @Option(help = "Include all classes, methods, fields, and resources from given paths for dynamic access", type = OptionType.Debug) // + public static final HostedOptionKey EnableDynamicAccessForClassPathEntry = new HostedOptionKey<>( + AccumulatingLocatableMultiOptionValue.Strings.build()); + @Option(help = "Include all classes, methods, fields, and resources from the class path", type = OptionType.Debug) // public static final HostedOptionKey IncludeAllFromClassPath = new HostedOptionKey<>(false); + @Option(help = "Include all classes, methods, fields, and resources for dynamic access for the whole classpath", type = OptionType.Debug) // + public static final HostedOptionKey EnableDynamicAccess = new HostedOptionKey<>(false); + public static boolean includeAll() { return IncludeAllFromModule.hasBeenSet() || IncludeAllFromPath.hasBeenSet() || IncludeAllFromPackage.hasBeenSet() || IncludeAllFromClassPath.hasBeenSet(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/ClassForNameSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/ClassForNameSupport.java index 18a05ad417a1..7c5f55ed0d30 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/ClassForNameSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/ClassForNameSupport.java @@ -26,6 +26,7 @@ import static com.oracle.svm.core.MissingRegistrationUtils.throwMissingRegistrationErrors; +import java.lang.reflect.Modifier; import java.util.EnumSet; import java.util.Objects; @@ -159,7 +160,7 @@ public void registerNegativeQuery(ConfigurationCondition condition, String class @Platforms(Platform.HOSTED_ONLY.class) public void registerUnsafeAllocated(ConfigurationCondition condition, Class clazz) { - if (!clazz.isArray()) { + if (!clazz.isArray() && !clazz.isInterface() && !Modifier.isAbstract(clazz.getModifiers())) { var conditionSet = unsafeInstantiatedClasses.putIfAbsent(clazz, RuntimeConditionSet.createHosted(condition)); if (conditionSet != null) { conditionSet.addCondition(condition); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java index 2c87c1d0be94..ef1841bcf3a2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ClassLoaderSupportImpl.java @@ -116,7 +116,8 @@ public void collectResources(ResourceCollector resourceCollector) { /* Collect remaining resources from classpath */ classLoaderSupport.classpath().stream().parallel().forEach(classpathFile -> { - boolean includeCurrent = classLoaderSupport.getJavaPathsToInclude().contains(classpathFile) || classLoaderSupport.includeAllFromClassPath(); + boolean includeCurrent = classLoaderSupport.getPathsToInclude().contains(classpathFile) || classLoaderSupport.includeAllFromClassPath() || + classLoaderSupport.getClassPathEntriesToEnableDynamicAccess().contains(classpathFile) || classLoaderSupport.isEnableDynamicAccess(); try { if (Files.isDirectory(classpathFile)) { scanDirectory(classpathFile, resourceCollector, includeCurrent); @@ -132,7 +133,8 @@ public void collectResources(ResourceCollector resourceCollector) { private void collectResourceFromModule(ResourceCollector resourceCollector, ResourceLookupInfo info) { ModuleReference moduleReference = info.resolvedModule.reference(); try (ModuleReader moduleReader = moduleReference.open()) { - boolean includeCurrent = classLoaderSupport.getJavaModuleNamesToInclude().contains(info.resolvedModule().name()); + boolean includeCurrent = classLoaderSupport.getModuleNamesToInclude().contains(info.resolvedModule().name()) || + classLoaderSupport.getModuleNamesToEnableDynamicAccess().contains(info.resolvedModule().name()); List resourcesFound = new ArrayList<>(); moduleReader.list().forEach(resourceName -> { var conditionsWithOrigins = shouldIncludeEntry(info.module, resourceCollector, resourceName, moduleReference.location().orElse(null), includeCurrent); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index 474e1b834b20..4de15d63f3cc 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -24,6 +24,10 @@ */ package com.oracle.svm.hosted; +import static com.oracle.svm.core.SubstrateOptions.EnableDynamicAccess; +import static com.oracle.svm.core.SubstrateOptions.EnableDynamicAccessForClassPathEntry; +import static com.oracle.svm.core.SubstrateOptions.EnableDynamicAccessForModule; +import static com.oracle.svm.core.SubstrateOptions.EnableDynamicAccessForPackage; import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromClassPath; import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromModule; import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromPackage; @@ -129,16 +133,25 @@ public final class NativeImageClassLoaderSupport { public final AnnotationExtractor annotationExtractor; - private Set javaModuleNamesToInclude; + private Set moduleNamesToInclude; + private Set moduleNamesToEnableDynamicAccess; + private Set javaPackagesToInclude; - private Set javaPathsToInclude; + private Set packagesToEnableDynamicAccess; + + private Set pathsToInclude; + private Set classPathEntriesToEnableDynamicAccess; + private boolean includeAllFromClassPath; + private boolean enableDynamicAccess; private Optional libGraalLoader; private List classLoaders; private final Set> classesToIncludeUnconditionally = ConcurrentHashMap.newKeySet(); private final Set includedJavaPackages = ConcurrentHashMap.newKeySet(); + private final Set> classesToIncludeMetadata = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set classNamesToIncludeMetadata = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Method implAddReadsAllUnnamed = ReflectionUtil.lookupMethod(Module.class, "implAddReadsAllUnnamed"); private final Method implAddEnableNativeAccess = ReflectionUtil.lookupMethod(Module.class, "implAddEnableNativeAccess"); @@ -247,28 +260,19 @@ private static Path stringToPath(String path) { } public void loadAllClasses(ForkJoinPool executor, ImageClassLoader imageClassLoader) { - guarantee(javaModuleNamesToInclude == null && javaPackagesToInclude == null, "This method should be executed only once."); - javaModuleNamesToInclude = Collections.unmodifiableSet(new HashSet<>(IncludeAllFromModule.getValue(parsedHostedOptions).values())); - /* Verify all modules are present */ - final Set allModules = Stream.concat(modulepathModuleFinder.findAll().stream(), upgradeAndSystemModuleFinder.findAll().stream()) - .map(m -> m.descriptor().name()) - .collect(Collectors.toSet()); - javaModuleNamesToInclude.stream() - .filter(m -> !allModules.contains(m)) - .findAny().ifPresent(m -> missingFromSetOfEntriesError(m, allModules, "module-path", IncludeAllFromModule)); + guarantee(moduleNamesToInclude == null && javaPackagesToInclude == null, "This method should be executed only once."); + + moduleNamesToInclude = collectAndVerifyModulesFromOption(IncludeAllFromModule); + moduleNamesToEnableDynamicAccess = collectAndVerifyModulesFromOption(EnableDynamicAccessForModule); javaPackagesToInclude = Set.copyOf(IncludeAllFromPackage.getValue(parsedHostedOptions).values()); + packagesToEnableDynamicAccess = Set.copyOf(EnableDynamicAccessForPackage.getValue(parsedHostedOptions).values()); - javaPathsToInclude = IncludeAllFromPath.getValue(parsedHostedOptions).values().stream() - .map(NativeImageClassLoaderSupport::stringToPath) - .map(Path::toAbsolutePath) - .collect(Collectors.toUnmodifiableSet()); - /* Verify all paths are present */ - javaPathsToInclude.stream() - .filter(p -> !classpath().contains(p)) - .findAny().ifPresent(p -> missingFromSetOfEntriesError(p, classpath(), "classpath", IncludeAllFromPath)); + pathsToInclude = collectAndVerifyPathsFromOption(IncludeAllFromPath); + classPathEntriesToEnableDynamicAccess = collectAndVerifyPathsFromOption(EnableDynamicAccessForClassPathEntry); includeAllFromClassPath = IncludeAllFromClassPath.getValue(parsedHostedOptions); + enableDynamicAccess = EnableDynamicAccess.getValue(parsedHostedOptions); new LoadClassHandler(executor, imageClassLoader).run(); @@ -286,6 +290,36 @@ public void loadAllClasses(ForkJoinPool executor, ImageClassLoader imageClassLoa } } + private Set collectAndVerifyModulesFromOption(HostedOptionKey option) { + var optionModuleNames = Collections.unmodifiableSet(new HashSet<>(option.getValue(parsedHostedOptions).values())); + + /* Verify all modules are present */ + final Set allModules = Stream.concat(modulepathModuleFinder.findAll().stream(), upgradeAndSystemModuleFinder.findAll().stream()) + .map(m -> m.descriptor().name()) + .collect(Collectors.toSet()); + optionModuleNames.stream() + .filter(m -> !allModules.contains(m)) + .findAny() + .ifPresent(m -> missingFromSetOfEntriesError(m, allModules, "module-path", option)); + + return optionModuleNames; + } + + private Set collectAndVerifyPathsFromOption(HostedOptionKey option) { + var optionPathsInclude = option.getValue(parsedHostedOptions).values().stream() + .map(NativeImageClassLoaderSupport::stringToPath) + .map(Path::toAbsolutePath) + .collect(Collectors.toUnmodifiableSet()); + + /* Verify all paths are present */ + optionPathsInclude.stream() + .filter(p -> !classpath().contains(p)) + .findAny() + .ifPresent(p -> missingFromSetOfEntriesError(p, classpath(), "classpath", option)); + + return optionPathsInclude; + } + private static void missingFromSetOfEntriesError(Object entry, Collection allEntries, String typeOfEntry, HostedOptionKey option) { String sortedEntries = allEntries.stream() @@ -596,6 +630,10 @@ public void setupLibGraalClassLoader() { } } + public Set getClassNamesToIncludeMetadata() { + return new HashSet<>(classNamesToIncludeMetadata); + } + private record AddExportsAndOpensAndReadsFormatValue(Module module, String packageName, List targetModules) { } @@ -735,8 +773,8 @@ private void run() { Set additionalSystemModules = upgradeAndSystemModuleFinder.findAll().stream() .map(v -> v.descriptor().name()) + .filter(n -> getModuleNamesToInclude().contains(n) || getModuleNamesToEnableDynamicAccess().contains(n)) .collect(Collectors.toSet()); - additionalSystemModules.retainAll(getJavaModuleNamesToInclude()); requiresInit.addAll(additionalSystemModules); Set explicitlyAddedModules = ModuleSupport.parseModuleSetModifierProperty(ModuleSupport.PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES); @@ -758,11 +796,8 @@ private void run() { } /* Verify all package inclusion requests were successful */ - for (String packageName : javaPackagesToInclude) { - if (!includedJavaPackages.contains(packageName)) { - missingFromSetOfEntriesError(packageName, includedJavaPackages, "package", IncludeAllFromPackage); - } - } + verifyPackageInclusionSuccessful(NativeImageClassLoaderSupport.this.javaPackagesToInclude, IncludeAllFromPackage); + verifyPackageInclusionSuccessful(NativeImageClassLoaderSupport.this.packagesToEnableDynamicAccess, EnableDynamicAccessForPackage); } private void initModule(ModuleReference moduleReference, boolean moduleRequiresInit) { @@ -774,7 +809,8 @@ private void initModule(ModuleReference moduleReference, boolean moduleRequiresI } try (ModuleReader moduleReader = moduleReference.open()) { Module module = optionalModule.get(); - final boolean includeUnconditionally = javaModuleNamesToInclude.contains(module.getName()); + final boolean includeUnconditionally = moduleNamesToInclude.contains(module.getName()); + final boolean includeForReflection = moduleNamesToEnableDynamicAccess.contains(module.getName()); var container = moduleReference.location().orElseThrow(); if (ModuleLayer.boot().equals(module.getLayer())) { builderURILocations.add(container); @@ -784,7 +820,7 @@ private void initModule(ModuleReference moduleReference, boolean moduleRequiresI String className = extractClassName(moduleResource, fileSystemSeparatorChar); if (className != null) { currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource; - executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit)); + executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit, includeForReflection)); } entriesProcessed.increment(); }); @@ -794,7 +830,8 @@ private void initModule(ModuleReference moduleReference, boolean moduleRequiresI } private void loadClassesFromPath(Path path) { - final boolean includeUnconditionally = javaPathsToInclude.contains(path) || includeAllFromClassPath; + final boolean includeUnconditionally = pathsToInclude.contains(path) || includeAllFromClassPath; + final boolean includeAllMetadata = classPathEntriesToEnableDynamicAccess.contains(path) || enableDynamicAccess; if (ClasspathUtils.isJar(path)) { try { URI container = path.toAbsolutePath().toUri(); @@ -808,7 +845,7 @@ private void loadClassesFromPath(Path path) { } if (probeJarFileSystem != null) { try (FileSystem jarFileSystem = probeJarFileSystem) { - loadClassesFromPath(container, jarFileSystem.getPath("/"), null, Collections.emptySet(), includeUnconditionally); + loadClassesFromPath(container, jarFileSystem.getPath("/"), null, Collections.emptySet(), includeUnconditionally, includeAllMetadata); } } } catch (ClosedByInterruptException ignored) { @@ -818,13 +855,14 @@ private void loadClassesFromPath(Path path) { } } else { URI container = path.toUri(); - loadClassesFromPath(container, path, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES, includeUnconditionally); + loadClassesFromPath(container, path, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES, includeUnconditionally, + includeAllMetadata); } } private static final String CLASS_EXTENSION = ".class"; - private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set excludes, boolean includeUnconditionally) { + private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set excludes, boolean includeUnconditionally, boolean includeAllMetadata) { boolean useFilter = root.equals(excludeRoot); if (useFilter) { String excludesStr = excludes.stream().map(Path::toString).collect(Collectors.joining(", ")); @@ -850,7 +888,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { String className = extractClassName(fileName, fileSystemSeparatorChar); if (className != null) { currentlyProcessedEntry = file.toUri().toString(); - executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally, true)); + executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally, true, includeAllMetadata)); } entriesProcessed.increment(); return FileVisitResult.CONTINUE; @@ -934,7 +972,7 @@ private String extractClassName(String fileName, char fileSystemSeparatorChar) { return strippedClassFileName.equals("module-info") ? null : strippedClassFileName.replace(fileSystemSeparatorChar, '.'); } - private void handleClassFileName(URI container, Module module, String className, boolean includeUnconditionally, boolean classRequiresInit) { + private void handleClassFileName(URI container, Module module, String className, boolean includeUnconditionally, boolean classRequiresInit, boolean includeAllMetadata) { if (classRequiresInit) { synchronized (classes) { EconomicSet classNames = classes.get(container); @@ -962,6 +1000,9 @@ private void handleClassFileName(URI container, Module module, String className, } catch (AssertionError error) { VMError.shouldNotReachHere(error); } catch (Throwable t) { + if (includeAllMetadata) { + classNamesToIncludeMetadata.add(className); + } ImageClassLoader.handleClassLoadingError(t); } if (clazz != null) { @@ -973,11 +1014,22 @@ private void handleClassFileName(URI container, Module module, String className, if (classRequiresInit) { imageClassLoader.handleClass(clazz); } + if (includeAllMetadata || packagesToEnableDynamicAccess.contains(packageName)) { + classesToIncludeMetadata.add(clazz); + } } imageClassLoader.watchdog.recordActivity(); } } + private void verifyPackageInclusionSuccessful(Set packagesToInclude, HostedOptionKey packagesOption) { + for (String packageName : packagesToInclude) { + if (!includedJavaPackages.contains(packageName)) { + missingFromSetOfEntriesError(packageName, includedJavaPackages, "package", packagesOption); + } + } + } + public void reportBuilderClassesInApplication() { EconomicMap> builderClasses = EconomicMap.create(); EconomicMap> applicationClasses = EconomicMap.create(); @@ -1011,21 +1063,39 @@ public void reportBuilderClassesInApplication() { } } - public Set getJavaModuleNamesToInclude() { - return javaModuleNamesToInclude; + public Set getModuleNamesToInclude() { + return moduleNamesToInclude; + } + + public Set getModuleNamesToEnableDynamicAccess() { + return moduleNamesToEnableDynamicAccess; + } + + public Set getPathsToInclude() { + return pathsToInclude; } - public Set getJavaPathsToInclude() { - return javaPathsToInclude; + public Set getClassPathEntriesToEnableDynamicAccess() { + return classPathEntriesToEnableDynamicAccess; } public boolean includeAllFromClassPath() { return includeAllFromClassPath; } + public boolean isEnableDynamicAccess() { + return enableDynamicAccess; + } + public List> getClassesToIncludeUnconditionally() { return classesToIncludeUnconditionally.stream() .sorted(Comparator.comparing(Class::getTypeName)) .collect(Collectors.toList()); } + + public List> getClassesToIncludeMetadata() { + return classesToIncludeMetadata.stream() + .sorted(Comparator.comparing(Class::getTypeName)) + .collect(Collectors.toList()); + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index 0915546147bb..c3f12239372b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -71,9 +72,14 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.nativeimage.hosted.Feature.OnAnalysisExitAccess; +import org.graalvm.nativeimage.hosted.RuntimeJNIAccess; +import org.graalvm.nativeimage.hosted.RuntimeProxyCreation; +import org.graalvm.nativeimage.hosted.RuntimeReflection; import org.graalvm.nativeimage.impl.AnnotationExtractor; import org.graalvm.nativeimage.impl.CConstantValueSupport; +import org.graalvm.nativeimage.impl.ConfigurationCondition; import org.graalvm.nativeimage.impl.RuntimeClassInitializationSupport; +import org.graalvm.nativeimage.impl.RuntimeReflectionSupport; import org.graalvm.nativeimage.impl.SizeOfSupport; import org.graalvm.word.PointerBase; @@ -1068,6 +1074,12 @@ protected void setupNativeImage(String imageName, OptionValues options, Map bb.registerTypeForBaseImage(cls)); + loader.classLoaderSupport.getClassesToIncludeMetadata().parallelStream() + .filter(ClassInclusionPolicy::isClassIncludedBase) + .forEach(NativeImageGenerator::registerClassFullyForReflection); + for (String className : loader.classLoaderSupport.getClassNamesToIncludeMetadata()) { + RuntimeReflection.registerClassLookup(className); + } registerEntryPointStubs(entryPoints); } @@ -1076,6 +1088,47 @@ protected void setupNativeImage(String imageName, OptionValues options, Map clazz) { + RuntimeReflection.register(clazz); + + /* Register every single-interface proxy */ + if (clazz.isInterface()) { + RuntimeProxyCreation.register(clazz); + } + + // GR-62143 Register all fields is very slow. + // RuntimeReflection.registerAllDeclaredFields(clazz); + RuntimeReflection.registerAllDeclaredMethods(clazz); + RuntimeReflection.registerAllDeclaredConstructors(clazz); + + RuntimeJNIAccess.register(clazz); + try { + for (Method declaredMethod : clazz.getDeclaredMethods()) { + RuntimeReflection.register(declaredMethod); + RuntimeJNIAccess.register(declaredMethod); + } + for (Constructor declaredConstructor : clazz.getDeclaredConstructors()) { + RuntimeReflection.register(declaredConstructor); + RuntimeJNIAccess.register(declaredConstructor); + } + for (var declaredField : clazz.getDeclaredFields()) { + RuntimeJNIAccess.register(declaredField); + // GR-62143 Registering all fields is very slow. + // RuntimeSerialization.register(declaredField); + } + } catch (LinkageError e) { + /* If we can't link we can not register for JNI */ + } + + // GR-62143 Registering all fields is very slow. + // RuntimeSerialization.register(clazz); + + // Missing an API for registering types as unsafe allocated + if (!(clazz.isArray() || clazz.isInterface() || clazz.isPrimitive() || Modifier.isAbstract(clazz.getModifiers()))) { + ImageSingletons.lookup(RuntimeReflectionSupport.class).register(ConfigurationCondition.alwaysTrue(), true, clazz); + } + } + protected void registerEntryPointStubs(Map entryPoints) { entryPoints.forEach((method, entryPointData) -> CEntryPointCallStubSupport.singleton().registerStubForMethod(method, () -> entryPointData)); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/ReflectionDataBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/ReflectionDataBuilder.java index 3e9e09ea28e8..e49b1b833ae4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/ReflectionDataBuilder.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/ReflectionDataBuilder.java @@ -549,7 +549,7 @@ public void registerAllFieldsQuery(ConfigurationCondition condition, boolean que } @Override - public void registerAllDeclaredFieldsQuery(ConfigurationCondition condition, Class clazz) { + public void registerAllDeclaredFields(ConfigurationCondition condition, Class clazz) { registerAllDeclaredFieldsQuery(condition, false, clazz); } @@ -582,13 +582,8 @@ private void registerField(ConfigurationCondition cnd, boolean queriedOnly, Fiel boolean exists = classFields.containsKey(analysisField); boolean shouldRegisterReachabilityHandler = classFields.isEmpty(); var cndValue = classFields.computeIfAbsent(analysisField, f -> new ConditionalRuntimeValue<>(RuntimeConditionSet.emptySet(), reflectField)); - if (!queriedOnly) { - /* queryOnly methods are conditioned by the type itself */ - cndValue.getConditions().addCondition(cnd); - } - if (!exists) { - registerTypesForField(analysisField, reflectField, true); + registerTypesForField(analysisField, reflectField, queriedOnly); /* * The image needs to know about subtypes shadowing fields registered for reflection to @@ -620,6 +615,8 @@ private void registerField(ConfigurationCondition cnd, boolean queriedOnly, Fiel * registered as queried. */ if (!queriedOnly) { + /* queryOnly methods are conditioned on the type itself */ + cndValue.getConditions().addCondition(cnd); registerTypesForField(analysisField, reflectField, false); } } @@ -1067,11 +1064,11 @@ private boolean shouldExcludeClass(Class clazz) { private static T queryGenericInfo(Callable callable) { try { return callable.call(); - } catch (MalformedParameterizedTypeException | TypeNotPresentException | LinkageError e) { + } catch (MalformedParameterizedTypeException | TypeNotPresentException | LinkageError | AssertionError e) { /* These are rethrown at run time, so we can simply ignore them when querying. */ return null; } catch (Throwable t) { - throw VMError.shouldNotReachHere(t); + throw VMError.shouldNotReachHere(callable.toString(), t); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/SubstitutionReflectivityFilter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/SubstitutionReflectivityFilter.java index 8ae0b0b2225f..5e8c0e7e1b41 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/SubstitutionReflectivityFilter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/SubstitutionReflectivityFilter.java @@ -36,6 +36,7 @@ import com.oracle.svm.core.annotate.InjectAccessors; import com.oracle.svm.core.annotate.TargetClass; +import jdk.graal.compiler.api.replacements.Fold; import jdk.vm.ci.meta.ResolvedJavaType; /** @@ -74,6 +75,8 @@ public static boolean shouldExclude(Executable method, AnalysisMetaAccess metaAc return true; } else if (aMethod.isAnnotationPresent(Delete.class)) { return true; // accesses would fail at runtime + } else if (aMethod.isAnnotationPresent(Fold.class)) { + return true; // accesses can contain hosted elements } else if (aMethod.isSynthetic() && aMethod.getDeclaringClass().isAnnotationPresent(TargetClass.class)) { /* * Synthetic methods are usually methods injected by javac to provide access to diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ReflectionUtil.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ReflectionUtil.java index 43f661063491..58addcbbdfbe 100644 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ReflectionUtil.java +++ b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ReflectionUtil.java @@ -122,7 +122,7 @@ public static Constructor lookupConstructor(boolean optional, Class de openModule(declaringClass); result.setAccessible(true); return result; - } catch (ReflectiveOperationException ex) { + } catch (ReflectiveOperationException | NoClassDefFoundError ex) { if (optional) { return null; } diff --git a/vm/mx.vm/mx_vm_benchmark.py b/vm/mx.vm/mx_vm_benchmark.py index 419ef750aa34..5ab7cd47a61d 100644 --- a/vm/mx.vm/mx_vm_benchmark.py +++ b/vm/mx.vm/mx_vm_benchmark.py @@ -238,10 +238,18 @@ def __init__(self, vm: NativeImageVM, bm_suite: BenchmarkSuite | NativeImageBenc base_image_build_args += ['--gc=' + vm.gc] + ['-H:+SpawnIsolates'] if vm.native_architecture: base_image_build_args += ['-march=native'] + if vm.dynamic_access: + base_image_build_args += ['-H:+EnableDynamicAccess', + '--initialize-at-build-time=com.sun.jmx.mbeanserver.DefaultMXBeanMappingFactory$CollectionMapping', + '--initialize-at-build-time=com.sun.jmx.mbeanserver.DefaultMXBeanMappingFactory$ArrayMapping', + '--initialize-at-build-time=com.sun.jmx.interceptor.DefaultMBeanServerInterceptor$ListenerWrapper', + '--initialize-at-build-time=oracle.jdbc.logging.annotations.Blinder', + '--initialize-at-build-time=oracle.sql.converter.CharacterConverterFactoryJDBC', + '--initialize-at-build-time=oracle.sql.CharacterSetFactoryDefault', + '--initialize-at-build-time=oracle.jdbc.logging.runtime.Features', + ] if vm.analysis_context_sensitivity: base_image_build_args += ['-H:AnalysisContextSensitivity=' + vm.analysis_context_sensitivity, '-H:-RemoveSaturatedTypeFlows', '-H:+AliasArrayTypeFlows'] - if vm.no_inlining_before_analysis: - base_image_build_args += ['-H:-InlineBeforeAnalysis'] if vm.optimization_level: base_image_build_args += ['-' + vm.optimization_level] if vm.async_sampler: @@ -530,6 +538,7 @@ def __init__(self, name, config_name, extra_java_args=None, extra_launcher_args= self.is_llvm = False self.gc = None self.native_architecture = False + self.dynamic_access = False self.use_upx = False self.use_open_type_world = False self.use_compacting_gc = False @@ -545,7 +554,6 @@ def __init__(self, name, config_name, extra_java_args=None, extra_launcher_args= self.force_profile_inference = False self.profile_inference_debug = False self.analysis_context_sensitivity = None - self.no_inlining_before_analysis = False self.optimization_level = None self._configure_comma_separated_configs(config_name) if ',' in config_name: @@ -567,6 +575,8 @@ def config_name(self): config = [] if self.native_architecture is True: config += ["native-architecture"] + if self.dynamic_access is True: + config += ["dynamic-access"] if self.use_string_inlining is True: config += ["string-inlining"] if self.use_open_type_world is True: @@ -601,8 +611,6 @@ def config_name(self): if sensitivity.startswith("_"): sensitivity = sensitivity[1:] config += [sensitivity] - if self.no_inlining_before_analysis is True: - config += ["no-inline"] if self.jdk_profiles_collect is True: config += ["jdk-profiles-collect"] if self.adopted_jdk_pgo is True: @@ -638,7 +646,8 @@ def _configure_from_name(self, config_name): mx.abort(f"config_name must be set. Use 'default' for the default {self.__class__.__name__} configuration.") # This defines the allowed config names for NativeImageVM. The ones registered will be available via --jvm-config - rule = r'^(?Pnative-architecture-)?(?Pstring-inlining-)?(?Potw-)?(?Pcompacting-gc-)?(?Pgate-)?(?Pupx-)?(?Pquickbuild-)?(?Pg1gc-)?' \ + rule = r'^(?Pnative-architecture-)?(?Pstring-inlining-)?(?Potw-)?(?Pcompacting-gc-)?(?Pdynamic-access-)?' \ + r'(?Pgate-)?(?Pupx-)?(?Pquickbuild-)?(?Pg1gc-)?' \ r'(?Pllvm-)?(?Ppgo-|pgo-sampler-)?(?Pinline-)?' \ r'(?Pinsens-|allocsens-|1obj-|2obj1h-|3obj2h-|4obj3h-)?(?Pno-inline-)?(?Pjdk-profiles-collect-|adopted-jdk-pgo-)?' \ r'(?Pprofile-inference-feature-extraction-|profile-inference-pgo-|profile-inference-debug-)?(?Psafepoint-sampler-|async-sampler-)?(?PO0-|O1-|O2-|O3-|Os-)?(default-)?(?Pce-|ee-)?$' @@ -653,6 +662,10 @@ def _configure_from_name(self, config_name): mx.logv(f"'native-architecture' is enabled for {config_name}") self.native_architecture = True + if matching.group("dynamic_access") is not None: + mx.logv(f"'dynamic-access' is enabled for {config_name}") + self.dynamic_access = True + if matching.group("string_inlining") is not None: mx.logv(f"'string-inlining' is enabled for {config_name}") self.use_string_inlining = True @@ -788,14 +801,6 @@ def generate_profiling_package_prefixes(): else: mx.abort(f"Unknown configuration for optimization level: {olevel}") - if matching.group("no_inlining_before_analysis") is not None: - option = matching.group("no_inlining_before_analysis")[:-1] - if option == "no-inline": - mx.logv(f"not doing inlining before analysis for {config_name}") - self.no_inlining_before_analysis = True - else: - mx.abort(f"Unknown configuration for no inlining before analysis: {option}") - if matching.group("analysis_context_sensitivity") is not None: context_sensitivity = matching.group("analysis_context_sensitivity")[:-1] if context_sensitivity in ["insens", "allocsens"]: @@ -1936,18 +1941,15 @@ def register_graalvm_vms(): mx_polybenchmarks_benchmark.rules = polybenchmark_rules optimization_levels = ['O0', 'O1', 'O2', 'O3', 'Os'] - - # Inlining before analysis is done by default analysis_context_sensitivity = ['insens', 'allocsens', '1obj', '2obj1h', '3obj2h', '4obj3h'] - analysis_context_sensitivity_no_inline = [f"{analysis_component}-no-inline" for analysis_component in analysis_context_sensitivity] for short_name, config_suffix in [('niee', 'ee'), ('ni', 'ce')]: if any(component.short_name == short_name for component in mx_sdk_vm_impl.registered_graalvm_components(stage1=False)): - for main_config in ['default', 'gate', 'llvm', 'native-architecture'] + analysis_context_sensitivity + analysis_context_sensitivity_no_inline: + for main_config in ['default', 'gate', 'llvm', 'native-architecture', 'dynamic-access'] + analysis_context_sensitivity: final_config_name = f'{main_config}-{config_suffix}' mx_benchmark.add_java_vm(NativeImageVM('native-image', final_config_name, ['--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED']), _suite, 10) # ' ' force the empty O<> configs as well - for main_config in ['llvm', 'native-architecture', 'g1gc', 'native-architecture-g1gc', ''] + analysis_context_sensitivity + analysis_context_sensitivity_no_inline: + for main_config in ['llvm', 'native-architecture', 'g1gc', 'native-architecture-g1gc', 'dynamic-access', ''] + analysis_context_sensitivity: for optimization_level in optimization_levels: if len(main_config) > 0: final_config_name = f'{main_config}-{optimization_level}-{config_suffix}' From 05d86079701dd1ce0c842ddf1b1d89d644719673 Mon Sep 17 00:00:00 2001 From: Vojin Jovanovic Date: Thu, 13 Feb 2025 10:21:50 +0100 Subject: [PATCH 2/2] Test in benchmarks --- .../graalvm/nativeimage/hosted/RuntimeReflection.java | 2 +- .../nativeimage/impl/RuntimeReflectionSupport.java | 2 +- vm/mx.vm/mx_vm_benchmark.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeReflection.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeReflection.java index fc3b93d8139b..3e8113d319e0 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeReflection.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/RuntimeReflection.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java index 071987452994..962055a87d24 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/vm/mx.vm/mx_vm_benchmark.py b/vm/mx.vm/mx_vm_benchmark.py index 5ab7cd47a61d..81ca4f5d58c2 100644 --- a/vm/mx.vm/mx_vm_benchmark.py +++ b/vm/mx.vm/mx_vm_benchmark.py @@ -173,6 +173,17 @@ def __init__(self, vm: NativeImageVM, bm_suite: BenchmarkSuite | NativeImageBenc self.log_dir: Path = self.output_dir self.ml_log_dump_path: Path = self.output_dir / f"{self.executable_name}.ml.log.csv" base_image_build_args = ['--no-fallback', '-g'] + base_image_build_args += [ + '-H:+EnableDynamicAccess', + # Our own bug + '--initialize-at-build-time=com.sun.jmx.mbeanserver.DefaultMXBeanMappingFactory$CollectionMapping', + '--initialize-at-build-time=com.sun.jmx.mbeanserver.DefaultMXBeanMappingFactory$ArrayMapping', + '--initialize-at-build-time=com.sun.jmx.interceptor.DefaultMBeanServerInterceptor$ListenerWrapper', + '--initialize-at-build-time=oracle.jdbc.logging.annotations.Blinder', + '--initialize-at-build-time=oracle.sql.converter.CharacterConverterFactoryJDBC', + '--initialize-at-build-time=oracle.sql.CharacterSetFactoryDefault', + '--initialize-at-build-time=oracle.jdbc.logging.runtime.Features', + ] base_image_build_args += ['-H:+VerifyGraalGraphs', '-H:+VerifyPhases', '--diagnostics-mode'] if vm.is_gate else [] base_image_build_args += ['-H:+ReportExceptionStackTraces'] base_image_build_args += bm_suite.build_assertions(self.benchmark_name, vm.is_gate)