diff --git a/.github/actions/do-build/action.yml b/.github/actions/do-build/action.yml
index 3deb7f4b8f803..79eddf8c70f0e 100644
--- a/.github/actions/do-build/action.yml
+++ b/.github/actions/do-build/action.yml
@@ -66,7 +66,7 @@ runs:
shell: bash
- name: 'Upload build logs'
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: failure-logs-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: failure-logs
@@ -74,7 +74,7 @@ runs:
# This is the best way I found to abort the job with an error message
- name: 'Notify about build failures'
- uses: actions/github-script@v6
+ uses: actions/github-script@v7
with:
script: core.setFailed('Build failed. See summary for details.')
if: steps.check.outputs.failure == 'true'
diff --git a/.github/actions/get-bootjdk/action.yml b/.github/actions/get-bootjdk/action.yml
index 1e569dd47c570..25ee1d8dfa0aa 100644
--- a/.github/actions/get-bootjdk/action.yml
+++ b/.github/actions/get-bootjdk/action.yml
@@ -65,7 +65,7 @@ runs:
- name: 'Check cache for BootJDK'
id: get-cached-bootjdk
- uses: actions/cache@v3
+ uses: actions/cache@v4
with:
path: bootjdk/jdk
key: boot-jdk-${{ inputs.platform }}-${{ steps.sha256.outputs.value }}
diff --git a/.github/actions/get-bundles/action.yml b/.github/actions/get-bundles/action.yml
index 956e1520cfbaa..0e52320a35060 100644
--- a/.github/actions/get-bundles/action.yml
+++ b/.github/actions/get-bundles/action.yml
@@ -48,14 +48,14 @@ runs:
steps:
- name: 'Download bundles artifact'
id: download-bundles
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
continue-on-error: true
- name: 'Download bundles artifact (retry)'
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
diff --git a/.github/actions/get-jtreg/action.yml b/.github/actions/get-jtreg/action.yml
index 3f5786b22d3dc..faedcc18807c0 100644
--- a/.github/actions/get-jtreg/action.yml
+++ b/.github/actions/get-jtreg/action.yml
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -41,7 +41,7 @@ runs:
- name: 'Check cache for JTReg'
id: get-cached-jtreg
- uses: actions/cache@v3
+ uses: actions/cache@v4
with:
path: jtreg/installed
key: jtreg-${{ steps.version.outputs.value }}
@@ -56,8 +56,14 @@ runs:
- name: 'Build JTReg'
run: |
+ # If runner architecture is x64 set JAVA_HOME_17_X64 otherwise set to JAVA_HOME_17_arm64
+ if [[ '${{ runner.arch }}' == 'X64' ]]; then
+ JDK="$JAVA_HOME_17_X64"
+ else
+ JDK="$JAVA_HOME_17_arm64"
+ fi
# Build JTReg and move files to the proper locations
- bash make/build.sh --jdk "$JAVA_HOME_17_X64"
+ bash make/build.sh --jdk "$JDK"
mkdir ../installed
mv build/images/jtreg/* ../installed
working-directory: jtreg/src
diff --git a/.github/actions/get-msys2/action.yml b/.github/actions/get-msys2/action.yml
index d36957e3b37e6..82022a6e233ea 100644
--- a/.github/actions/get-msys2/action.yml
+++ b/.github/actions/get-msys2/action.yml
@@ -30,8 +30,7 @@ runs:
using: composite
steps:
- name: 'Install MSYS2'
- # use a specific release of msys2/setup-msys2 to prevent jtreg build failures on newer release
- uses: msys2/setup-msys2@7efe20baefed56359985e327d329042cde2434ff
+ uses: msys2/setup-msys2@v2.22.0
with:
install: 'autoconf tar unzip zip make'
path-type: minimal
diff --git a/.github/actions/upload-bundles/action.yml b/.github/actions/upload-bundles/action.yml
index 88f7f6e8107a5..b35ee3a42e98c 100644
--- a/.github/actions/upload-bundles/action.yml
+++ b/.github/actions/upload-bundles/action.yml
@@ -69,7 +69,7 @@ runs:
shell: bash
- name: 'Upload bundles artifact'
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
diff --git a/.github/scripts/gen-test-results.sh b/.github/scripts/gen-test-results.sh
index 73edb8b3d11fe..9e85eef4dc08d 100644
--- a/.github/scripts/gen-test-results.sh
+++ b/.github/scripts/gen-test-results.sh
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -44,8 +44,8 @@ for test in $failures $errors; do
base_path="$(echo "$test" | tr '#' '_')"
report_file="$report_dir/$base_path.jtr"
hs_err_files=$(ls $report_dir/$base_path/hs_err*.log 2> /dev/null || true)
+ replay_files=$(ls $report_dir/$base_path/replay*.log 2> /dev/null || true)
echo "#### $test"
-
echo 'View test results '
echo ''
echo '```'
@@ -73,6 +73,20 @@ for test in $failures $errors; do
echo ''
fi
+ if [[ "$replay_files" != "" ]]; then
+ echo 'View HotSpot replay file '
+ echo ''
+ for replay in $replay_files; do
+ echo '```'
+ echo "$replay:"
+ echo ''
+ cat "$replay"
+ echo '```'
+ done
+
+ echo 'Building the JDK
Autoconf GNU Make GNU Bash Graphviz
+and Pandoc Running
Configure
@@ -524,7 +526,7 @@ Linux
The basic tooling is provided as part of the core operating system,
but you will most likely need to install developer packages.
For apt-based distributions (Debian, Ubuntu, etc), try this:
-sudo apt-get install build-essentialsudo apt-get install build-essential autoconfFor rpm-based distributions (Fedora, Red Hat, etc), try this:
sudo yum groupinstall "Development Tools"For Alpine Linux, aside from basic tooling, install the GNU versions
@@ -599,7 +601,7 @@
All compilers are expected to be able to handle the C11 language
standard for C, and C++14 for C++.
gcc
-The minimum accepted version of gcc is 6.0. Older versions will not
+
The minimum accepted version of gcc is 10.0. Older versions will not
be accepted by configure.
The JDK is currently known to compile successfully with gcc version
13.2 or newer.
@@ -753,6 +755,8 @@ Fontconfig
sudo apt-get install libfontconfig-dev.To install on an rpm-based Linux, try running
sudo yum install fontconfig-devel.
+To install on Alpine Linux, try running
+sudo apk add fontconfig-dev.
Use --with-fontconfig-include=<path> and
--with-fontconfig=<path> if configure
@@ -860,6 +864,13 @@
GNU Bash
href="https://www.gnu.org/software/bash">GNU Bash . No other shells
are supported.
At least version 3.2 of GNU Bash must be used.
+Graphviz and Pandoc
+In order to build the full docs (see the
+--enable-full-docs configure option) Graphviz and Pandoc are required. Any recent versions
+should work. For reference, and subject to change, Oracle builds use
+Graphviz 9.0.0 and Pandoc 2.19.2.
To build the JDK, you need a "configuration", which consists of a
directory where to store the build output, coupled with information
@@ -2155,15 +2166,26 @@
Using Multiple
configure from there, e.g.
mkdir build/<name> && cd build/<name> && bash ../../configure.
Then you can build that configuration using
-make CONF_NAME=<name> or
-make CONF=<pattern>, where
-<pattern> is a substring matching one or several
-configurations, e.g. CONF=debug. The special empty pattern
-(CONF=) will match all available configuration, so
-make CONF= hotspot will build the hotspot
-target for all configurations. Alternatively, you can execute
-make in the configuration directory, e.g.
-cd build/<name> && make.
+make CONF=<selector>, where
+<selector> is interpreted as follows:
+
+If <selector> exacly matches the name of a
+configuration, this and only this configuration will be selected.
+If <selector> matches (i.e. is a substring of)
+the names of several configurations, then all these configurations will
+be selected.
+If <selector> is empty (i.e. CONF=),
+then all configurations will be selected.
+If <selector> begins with !, then
+all configurations not matching the string following
+! will be selected.
+
+A more specialized version, CONF_NAME=<name> also
+exists, which will only match if the given <name>
+exactly matches a single configuration.
+Alternatively, you can execute make in the configuration
+directory, e.g. cd build/<name> && make.
+make CONF_NAME=<name> or
Handling Reconfigurations
If you update the repository and part of the configure script has
changed, the build system will force you to re-run
diff --git a/doc/building.md b/doc/building.md
index abbd935f652ab..14409b4066839 100644
--- a/doc/building.md
+++ b/doc/building.md
@@ -349,7 +349,7 @@ will most likely need to install developer packages.
For apt-based distributions (Debian, Ubuntu, etc), try this:
```
-sudo apt-get install build-essential
+sudo apt-get install build-essential autoconf
```
For rpm-based distributions (Fedora, Red Hat, etc), try this:
@@ -403,7 +403,7 @@ C, and C++14 for C++.
### gcc
-The minimum accepted version of gcc is 6.0. Older versions will not be accepted
+The minimum accepted version of gcc is 10.0. Older versions will not be accepted
by `configure`.
The JDK is currently known to compile successfully with gcc version 13.2 or
@@ -572,6 +572,7 @@ required on all platforms except Windows and macOS.
libfontconfig-dev`.
* To install on an rpm-based Linux, try running `sudo yum install
fontconfig-devel`.
+* To install on Alpine Linux, try running `sudo apk add fontconfig-dev`.
Use `--with-fontconfig-include=` and `--with-fontconfig=` if
`configure` does not automatically locate the platform Fontconfig files.
@@ -684,6 +685,14 @@ shells are supported.
At least version 3.2 of GNU Bash must be used.
+### Graphviz and Pandoc
+
+In order to build the full docs (see the `--enable-full-docs`
+configure option) [Graphviz](https://www.graphviz.org) and
+[Pandoc](https://pandoc.org) are required. Any recent versions should
+work. For reference, and subject to change, Oracle builds use Graphviz
+9.0.0 and Pandoc 2.19.2.
+
## Running Configure
To build the JDK, you need a "configuration", which consists of a directory
@@ -1943,12 +1952,25 @@ configuration with the name ``. Alternatively, you can create a directory
under `build` and run `configure` from there, e.g. `mkdir build/ && cd
build/ && bash ../../configure`.
-Then you can build that configuration using `make CONF_NAME=` or `make
-CONF=`, where `` is a substring matching one or several
-configurations, e.g. `CONF=debug`. The special empty pattern (`CONF=`) will
-match *all* available configuration, so `make CONF= hotspot` will build the
-`hotspot` target for all configurations. Alternatively, you can execute `make`
-in the configuration directory, e.g. `cd build/ && make`.
+Then you can build that configuration using `make CONF=`, where
+`` is interpreted as follows:
+
+* If `` exacly matches the name of a configuration, this and only
+ this configuration will be selected.
+* If `` matches (i.e. is a substring of) the names of several
+ configurations, then all these configurations will be selected.
+* If `` is empty (i.e. `CONF=`), then all configurations will be
+ selected.
+* If `` begins with `!`, then all configurations **not** matching the
+ string following `!` will be selected.
+
+A more specialized version, `CONF_NAME=` also exists, which will only
+match if the given `` exactly matches a single configuration.
+
+Alternatively, you can execute `make` in the configuration directory, e.g. `cd
+build/ && make`.
+
+`make CONF_NAME=` or
### Handling Reconfigurations
diff --git a/make/Docs.gmk b/make/Docs.gmk
index 5f9ec99fcae0f..d0c01f0283d43 100644
--- a/make/Docs.gmk
+++ b/make/Docs.gmk
@@ -139,11 +139,6 @@ ifeq ($(IS_DRAFT), true)
endif
DRAFT_TEXT := This specification is not final and is subject to change. \
Use is subject to license terms .
-
- # Workaround stylesheet bug
- HEADER_STYLE := style="margin-top: 9px;"
-else
- HEADER_STYLE := style="margin-top: 14px;"
endif
# $1 - Relative prefix to COPYRIGHT_URL
@@ -339,7 +334,7 @@ define SetupApiDocsGenerationBody
$1_DOC_TITLE := $$($1_LONG_NAME)$$($1_SHORT_NAME) \
+ $1_HEADER_TITLE :=
$$($1_SHORT_NAME) \
$$(DRAFT_MARKER_STR)
ifneq ($$($1_OTHER_VERSIONS), )
$1_JAVADOC_BOTTOM := $$(call JAVADOC_BOTTOM,
Other versions. )
@@ -647,7 +642,7 @@ ifeq ($(ENABLE_PANDOC), true)
GLOBAL_SPECS_DEFAULT_CSS_FILE := $(DOCS_OUTPUTDIR)/resources/jdk-default.css
# Unset the following to suppress the link to the tool guides
NAV_LINK_GUIDES := --nav-link-guides
- HEADER_RIGHT_SIDE_INFO :=
$(subst &,&,$(JDK_SHORT_NAME))$(DRAFT_MARKER_STR)
+ HEADER_RIGHT_SIDE_INFO :=
$(subst &,&,$(JDK_SHORT_NAME)) $(DRAFT_MARKER_STR)
$(foreach m, $(ALL_MODULES), \
$(eval SPECS_$m := $(call FindModuleSpecsDirs, $m)) \
diff --git a/make/Global.gmk b/make/Global.gmk
index e5e76b475b941..1df6c5fb6bc4b 100644
--- a/make/Global.gmk
+++ b/make/Global.gmk
@@ -87,10 +87,9 @@ help:
$(info $(_) # (gensrc, java, copy, libs, launchers, gendata))
$(info )
$(info Make control variables)
- $(info $(_) CONF= # Build all configurations (note, assignment is empty))
- $(info $(_) CONF=
# Build the configuration(s) with a name matching)
- $(info $(_) # )
- $(info $(_) CONF_NAME= # Build the configuration with exactly the )
+ $(info $(_) CONF= # Select which configuration(s) to build)
+ $(info $(_) CONF= # Select all configurations (note, assignment is empty))
+ $(info $(_) CONF_NAME= # Select the configuration with the name )
$(info $(_) SPEC= # Build the configuration given by the spec file)
$(info $(_) LOG= # Change the log level from warn to )
$(info $(_) # Available log levels are:)
diff --git a/make/Hsdis.gmk b/make/Hsdis.gmk
index 7496a3a2cf1b4..6de0e628a5288 100644
--- a/make/Hsdis.gmk
+++ b/make/Hsdis.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -38,7 +38,7 @@ HSDIS_OUTPUT_DIR := $(SUPPORT_OUTPUTDIR)/hsdis
REAL_HSDIS_NAME := hsdis-$(OPENJDK_TARGET_CPU_LEGACY_LIB)$(SHARED_LIBRARY_SUFFIX)
BUILT_HSDIS_LIB := $(HSDIS_OUTPUT_DIR)/$(REAL_HSDIS_NAME)
-HSDIS_TOOLCHAIN := TOOLCHAIN_DEFAULT
+HSDIS_LINK_TYPE := C
HSDIS_TOOLCHAIN_CFLAGS := $(CFLAGS_JDKLIB)
HSDIS_TOOLCHAIN_LDFLAGS := $(LDFLAGS_JDKLIB)
@@ -59,8 +59,8 @@ endif
ifeq ($(HSDIS_BACKEND), llvm)
# Use C++ instead of C
+ HSDIS_LINK_TYPE := C++
HSDIS_TOOLCHAIN_CFLAGS := $(CXXFLAGS_JDKLIB)
- HSDIS_TOOLCHAIN := TOOLCHAIN_LINK_CXX
ifeq ($(call isTargetOs, linux), true)
LLVM_OS := pc-linux-gnu
@@ -91,14 +91,11 @@ ifeq ($(HSDIS_BACKEND), binutils)
endif
endif
- $(eval $(call DefineNativeToolchain, TOOLCHAIN_MINGW, \
- CC := $(MINGW_BASE)-gcc, \
- LD := $(MINGW_BASE)-ld, \
- OBJCOPY := $(MINGW_BASE)-objcopy, \
- RC := $(RC), \
- SYSROOT_CFLAGS := --sysroot=$(MINGW_SYSROOT), \
- SYSROOT_LDFLAGS := --sysroot=$(MINGW_SYSROOT), \
- ))
+ BUILD_HSDIS_CC := $(MINGW_BASE)-gcc
+ BUILD_HSDIS_LD := $(MINGW_BASE)-ld
+ BUILD_HSDIS_OBJCOPY := $(MINGW_BASE)-objcopy
+ BUILD_HSDIS_SYSROOT_CFLAGS := --sysroot=$(MINGW_SYSROOT)
+ BUILD_HSDIS_SYSROOT_LDFLAGS := --sysroot=$(MINGW_SYSROOT)
MINGW_SYSROOT_LIB_PATH := $(MINGW_SYSROOT)/mingw/lib
ifeq ($(wildcard $(MINGW_SYSROOT_LIB_PATH)), )
@@ -122,8 +119,8 @@ ifeq ($(HSDIS_BACKEND), binutils)
TOOLCHAIN_TYPE := gcc
OPENJDK_TARGET_OS := linux
+ OPENJDK_TARGET_OS_TYPE := unix
CC_OUT_OPTION := -o$(SPACE)
- LD_OUT_OPTION := -o$(SPACE)
GENDEPS_FLAGS := -MMD -MF
CFLAGS_DEBUG_SYMBOLS := -g
DISABLED_WARNINGS :=
@@ -131,7 +128,6 @@ ifeq ($(HSDIS_BACKEND), binutils)
CFLAGS_WARNINGS_ARE_ERRORS := -Werror
SHARED_LIBRARY_FLAGS := -shared
- HSDIS_TOOLCHAIN := TOOLCHAIN_MINGW
HSDIS_TOOLCHAIN_CFLAGS :=
HSDIS_TOOLCHAIN_LDFLAGS := -L$(MINGW_GCC_LIB_PATH) -L$(MINGW_SYSROOT_LIB_PATH)
MINGW_DLLCRT := $(MINGW_SYSROOT_LIB_PATH)/dllcrt2.o
@@ -144,9 +140,9 @@ endif
$(eval $(call SetupJdkLibrary, BUILD_HSDIS, \
NAME := hsdis, \
+ LINK_TYPE := $(HSDIS_LINK_TYPE), \
SRC := $(TOPDIR)/src/utils/hsdis/$(HSDIS_BACKEND), \
EXTRA_HEADER_DIRS := $(TOPDIR)/src/utils/hsdis, \
- TOOLCHAIN := $(HSDIS_TOOLCHAIN), \
OUTPUT_DIR := $(HSDIS_OUTPUT_DIR), \
OBJECT_DIR := $(HSDIS_OUTPUT_DIR), \
DISABLED_WARNINGS_gcc := undef format-nonliteral sign-compare, \
diff --git a/make/InitSupport.gmk b/make/InitSupport.gmk
index 31c80e2f7267f..4b14c4f9ad951 100644
--- a/make/InitSupport.gmk
+++ b/make/InitSupport.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -202,8 +202,14 @@ ifeq ($(HAS_SPEC),)
matching_confs := $$(strip $$(all_confs))
else
# Otherwise select those that contain the given CONF string
- matching_confs := $$(strip $$(foreach var, $$(all_confs), \
- $$(if $$(findstring $$(CONF), $$(var)), $$(var))))
+ ifeq ($$(patsubst !%,,$$(CONF)),)
+ # A CONF starting with ! means we should negate the search term
+ matching_confs := $$(strip $$(foreach var, $$(all_confs), \
+ $$(if $$(findstring $$(subst !,,$$(CONF)), $$(var)), ,$$(var))))
+ else
+ matching_confs := $$(strip $$(foreach var, $$(all_confs), \
+ $$(if $$(findstring $$(CONF), $$(var)), $$(var))))
+ endif
ifneq ($$(filter $$(CONF), $$(matching_confs)), )
# If we found an exact match, use that
matching_confs := $$(CONF)
@@ -421,8 +427,9 @@ else # $(HAS_SPEC)=true
# Cleanup after a compare build
define CleanupCompareBuild
- # If running with a COMPARE_BUILD patch, reverse-apply it
- $(if $(COMPARE_BUILD_PATCH), cd $(topdir) && $(PATCH) -R -p1 < $(COMPARE_BUILD_PATCH))
+ # If running with a COMPARE_BUILD patch, reverse-apply it, but continue
+ # even if that fails (can happen with removed files).
+ $(if $(COMPARE_BUILD_PATCH), cd $(topdir) && $(PATCH) -R -p1 < $(COMPARE_BUILD_PATCH) || true)
# Move this build away and restore the original build
$(MKDIR) -p $(topdir)/build/compare-build
$(MV) $(OUTPUTDIR) $(COMPARE_BUILD_OUTPUTDIR)
diff --git a/make/Main.gmk b/make/Main.gmk
index 5534a68f13bab..b0b7565c138e2 100644
--- a/make/Main.gmk
+++ b/make/Main.gmk
@@ -969,20 +969,28 @@ else
jdk.jdeps-gendata: java
- # The ct.sym generation uses all the moduleinfos as input
- jdk.compiler-gendata: $(GENSRC_MODULEINFO_TARGETS) $(JAVA_TARGETS)
- # jdk.compiler-gendata needs the BUILD_JDK. If the BUILD_JDK was supplied
- # externally, no extra prerequisites are needed.
+ # jdk.compiler gendata generates ct.sym, which requires all generated
+ # java source and compiled classes present.
+ jdk.compiler-gendata: $(JAVA_TARGETS)
+
+ # jdk.javadoc gendata generates element-list, which requires all java sources
+ # but not compiled classes.
+ jdk.javadoc-gendata: $(GENSRC_TARGETS)
+
+ # ct.sym and element-list generation also needs the BUILD_JDK. If the
+ # BUILD_JDK was supplied externally, no extra prerequisites are needed.
ifeq ($(CREATE_BUILDJDK), true)
ifneq ($(CREATING_BUILDJDK), true)
# When cross compiling and an external BUILD_JDK wasn't supplied, it's
# produced by the create-buildjdk target.
jdk.compiler-gendata: create-buildjdk
+ jdk.javadoc-gendata: create-buildjdk
endif
else ifeq ($(EXTERNAL_BUILDJDK), false)
# When not cross compiling, the BUILD_JDK is the interim jdk image, and
# the javac launcher is needed.
jdk.compiler-gendata: jdk.compiler-launchers
+ jdk.javadoc-gendata: jdk.compiler-launchers
endif
# Declare dependencies between jmod targets.
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index b82d2b2b6015d..b0291e4eff4e7 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -620,11 +620,16 @@ define SetupRunMicroTestBody
$1_MICRO_WARMUP_TIME := -w $$(MICRO_WARMUP_TIME)
endif
+# Microbenchmarks are executed from the root of the test image directory.
+# This enables JMH tests to add dependencies using relative paths such as
+# -Djava.library.path=micro/native
+
run-test-$1: pre-run-test
$$(call LogWarn)
$$(call LogWarn, Running test '$$($1_TEST)')
$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/micro, ( \
+ $$(CD) $$(TEST_IMAGE_DIR) && \
$$(FIXPATH) $$($1_MICRO_TEST_JDK)/bin/java $$($1_MICRO_JAVA_OPTIONS) \
-jar $$($1_MICRO_BENCHMARKS_JAR) \
$$($1_MICRO_ITER) $$($1_MICRO_FORK) $$($1_MICRO_TIME) \
diff --git a/make/autoconf/basic.m4 b/make/autoconf/basic.m4
index 5a1fbc0795274..91d7a55010887 100644
--- a/make/autoconf/basic.m4
+++ b/make/autoconf/basic.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -123,6 +123,22 @@ AC_DEFUN_ONCE([BASIC_SETUP_BUILD_ENV],
]
)
AC_SUBST(BUILD_ENV)
+
+ if test "x$LOCALE" != x; then
+ # Check if we actually have C.UTF-8; if so, use it
+ if $LOCALE -a | $GREP -q -E "^C\.(utf8|UTF-8)$"; then
+ LOCALE_USED=C.UTF-8
+ else
+ AC_MSG_WARN([C.UTF-8 locale not found, using C locale])
+ LOCALE_USED=C
+ fi
+ else
+ AC_MSG_WARN([locale command not not found, using C locale])
+ LOCALE_USED=C
+ fi
+
+ export LC_ALL=$LOCALE_USED
+ AC_SUBST(LOCALE_USED)
])
###############################################################################
diff --git a/make/autoconf/basic_tools.m4 b/make/autoconf/basic_tools.m4
index f9ecbab7ac099..0c084561f2ef0 100644
--- a/make/autoconf/basic_tools.m4
+++ b/make/autoconf/basic_tools.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -54,6 +54,7 @@ AC_DEFUN_ONCE([BASIC_SETUP_FUNDAMENTAL_TOOLS],
UTIL_REQUIRE_SPECIAL(SED, [AC_PROG_SED])
# Tools only needed on some platforms
+ UTIL_LOOKUP_PROGS(LOCALE, locale)
UTIL_LOOKUP_PROGS(PATHTOOL, cygpath wslpath)
UTIL_LOOKUP_PROGS(CMD, cmd.exe, $PATH:/cygdrive/c/windows/system32:/mnt/c/windows/system32:/c/windows/system32)
])
diff --git a/make/autoconf/build-aux/pkg.m4 b/make/autoconf/build-aux/pkg.m4
index 5f4b22bb27f05..ddb685e9bc35f 100644
--- a/make/autoconf/build-aux/pkg.m4
+++ b/make/autoconf/build-aux/pkg.m4
@@ -25,7 +25,7 @@
# questions.
#
-#
+#
# Copyright © 2004 Scott James Remnant .
#
# This program is free software; you can redistribute it and/or modify
@@ -54,18 +54,18 @@ AC_DEFUN([PKG_PROG_PKG_CONFIG],
m4_pattern_allow([^PKG_CONFIG(_PATH)?$])
AC_ARG_VAR([PKG_CONFIG], [path to pkg-config utility])dnl
if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then
- AC_PATH_TOOL([PKG_CONFIG], [pkg-config])
+ AC_PATH_TOOL([PKG_CONFIG], [pkg-config])
fi
if test -n "$PKG_CONFIG"; then
- _pkg_min_version=m4_default([$1], [0.9.0])
- AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version])
- if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then
- AC_MSG_RESULT([yes])
- else
- AC_MSG_RESULT([no])
- PKG_CONFIG=""
- fi
-
+ _pkg_min_version=m4_default([$1], [0.9.0])
+ AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version])
+ if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then
+ AC_MSG_RESULT([yes])
+ else
+ AC_MSG_RESULT([no])
+ PKG_CONFIG=""
+ fi
+
fi[]dnl
])# PKG_PROG_PKG_CONFIG
@@ -97,7 +97,7 @@ m4_define([_PKG_CONFIG],
elif test -n "$PKG_CONFIG"; then
PKG_CHECK_EXISTS([$3],
[pkg_cv_[]$1=`$PKG_CONFIG --[]$2 "$3" 2>/dev/null`],
- [pkg_failed=yes])
+ [pkg_failed=yes])
else
pkg_failed=untried
fi[]dnl
@@ -143,14 +143,14 @@ See the pkg-config man page for more details.])
if test $pkg_failed = yes; then
_PKG_SHORT_ERRORS_SUPPORTED
if test $_pkg_short_errors_supported = yes; then
- $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors "$2" 2>&1`
- else
- $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors "$2" 2>&1`
+ $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors "$2" 2>&1`
+ else
+ $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors "$2" 2>&1`
fi
- # Put the nasty error message in config.log where it belongs
- echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD
+ # Put the nasty error message in config.log where it belongs
+ echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD
- ifelse([$4], , [AC_MSG_ERROR(dnl
+ ifelse([$4], , [AC_MSG_ERROR(dnl
[Package requirements ($2) were not met:
$$1_PKG_ERRORS
@@ -160,10 +160,10 @@ installed software in a non-standard prefix.
_PKG_TEXT
])],
- [AC_MSG_RESULT([no])
+ [AC_MSG_RESULT([no])
$4])
elif test $pkg_failed = untried; then
- ifelse([$4], , [AC_MSG_FAILURE(dnl
+ ifelse([$4], , [AC_MSG_FAILURE(dnl
[The pkg-config script could not be found or is too old. Make sure it
is in your PATH or set the PKG_CONFIG environment variable to the full
path to pkg-config.
@@ -171,11 +171,11 @@ path to pkg-config.
_PKG_TEXT
To get pkg-config, see " | \
+ $CC $CFLAGS -v -E - 2>&1 | \
+ $GREP stddef | $TAIL -1 | $TR -s " " | $CUT -d'"' -f2`
+ if test "x$GCC_SYSTEM_INCLUDE" != "x"; then
+ GCC_SYSTEM_INCLUDE=`$DIRNAME $GCC_SYSTEM_INCLUDE`
+ GCC_INCLUDE_DEBUG_MAP_FLAGS="$GCC_INCLUDE_DEBUG_MAP_FLAGS \
+ -fdebug-prefix-map=${GCC_SYSTEM_INCLUDE}/=/usr/local/gcc_include/"
+ fi
+
+ # Add g++ system include mapping => /usr/local/gxx_include
+ # Find location of cstddef using build C++ compiler
+ GXX_SYSTEM_INCLUDE=`$ECHO "#include " | \
+ $CXX $CXXFLAGS -v -E -x c++ - 2>&1 | \
+ $GREP cstddef | $TAIL -1 | $TR -s " " | $CUT -d'"' -f2`
+ if test "x$GXX_SYSTEM_INCLUDE" != "x"; then
+ GXX_SYSTEM_INCLUDE=`$DIRNAME $GXX_SYSTEM_INCLUDE`
+ GCC_INCLUDE_DEBUG_MAP_FLAGS="$GCC_INCLUDE_DEBUG_MAP_FLAGS \
+ -fdebug-prefix-map=${GXX_SYSTEM_INCLUDE}/=/usr/local/gxx_include/"
+ fi
+
+ # Add to debug prefix cflags
+ DEBUG_PREFIX_CFLAGS="$DEBUG_PREFIX_CFLAGS $GCC_INCLUDE_DEBUG_MAP_FLAGS"
+])
+
AC_DEFUN([FLAGS_SETUP_WARNINGS],
[
# Set default value.
@@ -425,13 +477,14 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
[
#### OS DEFINES, these should be independent on toolchain
if test "x$OPENJDK_TARGET_OS" = xlinux; then
- CFLAGS_OS_DEF_JVM="-DLINUX"
- CFLAGS_OS_DEF_JDK="-D_GNU_SOURCE -D_REENTRANT -D_LARGEFILE64_SOURCE"
+ CFLAGS_OS_DEF_JVM="-DLINUX -D_FILE_OFFSET_BITS=64"
+ CFLAGS_OS_DEF_JDK="-D_GNU_SOURCE -D_REENTRANT -D_FILE_OFFSET_BITS=64"
elif test "x$OPENJDK_TARGET_OS" = xmacosx; then
CFLAGS_OS_DEF_JVM="-D_ALLBSD_SOURCE -D_DARWIN_C_SOURCE -D_XOPEN_SOURCE"
CFLAGS_OS_DEF_JDK="-D_ALLBSD_SOURCE -D_DARWIN_UNLIMITED_SELECT"
elif test "x$OPENJDK_TARGET_OS" = xaix; then
- CFLAGS_OS_DEF_JVM="-DAIX"
+ CFLAGS_OS_DEF_JVM="-DAIX -D_LARGE_FILES"
+ CFLAGS_OS_DEF_JDK="-D_LARGE_FILES"
elif test "x$OPENJDK_TARGET_OS" = xbsd; then
CFLAGS_OS_DEF_JDK="-D_ALLBSD_SOURCE"
elif test "x$OPENJDK_TARGET_OS" = xwindows; then
@@ -489,7 +542,7 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
ALWAYS_DEFINES_JVM="-D_GNU_SOURCE"
elif test "x$TOOLCHAIN_TYPE" = xxlc; then
ALWAYS_DEFINES_JVM="-D_REENTRANT"
- ALWAYS_DEFINES_JDK="-D_GNU_SOURCE -D_REENTRANT -D_LARGEFILE64_SOURCE -DSTDC"
+ ALWAYS_DEFINES_JDK="-D_GNU_SOURCE -D_REENTRANT -DSTDC"
elif test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
# Access APIs for Windows 8 and above
# see https://docs.microsoft.com/en-us/cpp/porting/modifying-winver-and-win32-winnt?view=msvc-170
diff --git a/make/autoconf/flags-ldflags.m4 b/make/autoconf/flags-ldflags.m4
index 195c1d341595f..58bc4a44bfbdf 100644
--- a/make/autoconf/flags-ldflags.m4
+++ b/make/autoconf/flags-ldflags.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -93,7 +93,7 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
BASIC_LDFLAGS_JVM_ONLY="-Wl,-lC_r -bbigtoc"
elif test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
- BASIC_LDFLAGS="-nologo -opt:ref"
+ BASIC_LDFLAGS="-opt:ref"
BASIC_LDFLAGS_JDK_ONLY="-incremental:no"
BASIC_LDFLAGS_JVM_ONLY="-opt:icf,8 -subsystem:windows"
fi
diff --git a/make/autoconf/flags-other.m4 b/make/autoconf/flags-other.m4
index 7e2521ffef3b0..8d4d405b07639 100644
--- a/make/autoconf/flags-other.m4
+++ b/make/autoconf/flags-other.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -33,9 +33,6 @@ AC_DEFUN([FLAGS_SETUP_ARFLAGS],
# FIXME: figure out if we should select AR flags depending on OS or toolchain.
if test "x$OPENJDK_TARGET_OS" = xaix; then
ARFLAGS="-X64"
- elif test "x$OPENJDK_TARGET_OS" = xwindows; then
- # lib.exe is used as AR to create static libraries.
- ARFLAGS="-nologo -NODEFAULTLIB:MSVCRT"
else
ARFLAGS=""
fi
@@ -43,6 +40,18 @@ AC_DEFUN([FLAGS_SETUP_ARFLAGS],
AC_SUBST(ARFLAGS)
])
+AC_DEFUN([FLAGS_SETUP_LIBFLAGS],
+[
+ # LIB is used to create static libraries on Windows
+ if test "x$OPENJDK_TARGET_OS" = xwindows; then
+ LIBFLAGS="-nodefaultlib:msvcrt"
+ else
+ LIBFLAGS=""
+ fi
+
+ AC_SUBST(LIBFLAGS)
+])
+
AC_DEFUN([FLAGS_SETUP_STRIPFLAGS],
[
## Setup strip.
diff --git a/make/autoconf/flags.m4 b/make/autoconf/flags.m4
index 8c029f7d2f58f..147382f398eed 100644
--- a/make/autoconf/flags.m4
+++ b/make/autoconf/flags.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -364,24 +364,12 @@ AC_DEFUN([FLAGS_SETUP_TOOLCHAIN_CONTROL],
if test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
CC_OUT_OPTION=-Fo
- LD_OUT_OPTION=-out:
- AR_OUT_OPTION=-out:
else
# The option used to specify the target .o,.a or .so file.
# When compiling, how to specify the to be created object file.
CC_OUT_OPTION='-o$(SPACE)'
- # When linking, how to specify the output
- LD_OUT_OPTION='-o$(SPACE)'
- # When archiving, how to specify the destination static archive.
- if test "x$OPENJDK_TARGET_OS" = xmacosx; then
- AR_OUT_OPTION='-r -cs$(SPACE)'
- else
- AR_OUT_OPTION='-rcs$(SPACE)'
- fi
fi
AC_SUBST(CC_OUT_OPTION)
- AC_SUBST(LD_OUT_OPTION)
- AC_SUBST(AR_OUT_OPTION)
# Generate make dependency files
if test "x$TOOLCHAIN_TYPE" = xgcc; then
@@ -423,6 +411,7 @@ AC_DEFUN([FLAGS_SETUP_FLAGS],
FLAGS_SETUP_LDFLAGS
FLAGS_SETUP_ARFLAGS
+ FLAGS_SETUP_LIBFLAGS
FLAGS_SETUP_STRIPFLAGS
FLAGS_SETUP_RCFLAGS
FLAGS_SETUP_NMFLAGS
diff --git a/make/autoconf/spec.gmk.template b/make/autoconf/spec.gmk.template
index b78d3da6b32ed..863a51eeb4afa 100644
--- a/make/autoconf/spec.gmk.template
+++ b/make/autoconf/spec.gmk.template
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -56,8 +56,8 @@ COMMA := ,
# What make to use for main processing, after bootstrapping top-level Makefile.
MAKE := @MAKE@
-# Make sure all shell commands are executed with the C locale
-export LC_ALL := C
+# Make sure all shell commands are executed with a proper locale
+export LC_ALL := @LOCALE_USED@
# Make sure we override any local CLASSPATH variable
export CLASSPATH := @CLASSPATH@
@@ -498,8 +498,6 @@ COMPILER_COMMAND_FILE_FLAG := @COMPILER_COMMAND_FILE_FLAG@
COMPILER_BINDCMD_FILE_FLAG := @COMPILER_BINDCMD_FILE_FLAG@
CC_OUT_OPTION := @CC_OUT_OPTION@
-LD_OUT_OPTION := @LD_OUT_OPTION@
-AR_OUT_OPTION := @AR_OUT_OPTION@
# Flags used for overriding the default opt setting for a C/C++ source file.
C_O_FLAG_HIGHEST_JVM := @C_O_FLAG_HIGHEST_JVM@
@@ -604,10 +602,10 @@ BUILD_SYSROOT_LDFLAGS := @BUILD_SYSROOT_LDFLAGS@
AS := @AS@
-# AR is used to create a static library (is ar in unix, lib.exe in windows)
AR := @AR@
ARFLAGS := @ARFLAGS@
-
+LIB := @LIB@
+LIBFLAGS := @LIBFLAGS@
NM := @NM@
NMFLAGS := @NMFLAGS@
STRIP := @STRIP@
@@ -619,10 +617,6 @@ INSTALL_NAME_TOOL := @INSTALL_NAME_TOOL@
METAL := @METAL@
METALLIB := @METALLIB@
-# Options to linker to specify a mapfile.
-# (Note absence of := assignment, because we do not want to evaluate the macro body here)
-SET_SHARED_LIBRARY_MAPFILE = @SET_SHARED_LIBRARY_MAPFILE@
-
#
# Options for generating debug symbols
COMPILE_WITH_DEBUG_SYMBOLS := @COMPILE_WITH_DEBUG_SYMBOLS@
diff --git a/make/autoconf/toolchain.m4 b/make/autoconf/toolchain.m4
index 289eec3356b91..de0b6f0adfb17 100644
--- a/make/autoconf/toolchain.m4
+++ b/make/autoconf/toolchain.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -51,7 +51,7 @@ TOOLCHAIN_DESCRIPTION_xlc="IBM XL C/C++"
# Minimum supported versions, empty means unspecified
TOOLCHAIN_MINIMUM_VERSION_clang="3.5"
-TOOLCHAIN_MINIMUM_VERSION_gcc="6.0"
+TOOLCHAIN_MINIMUM_VERSION_gcc="10.0"
TOOLCHAIN_MINIMUM_VERSION_microsoft="19.28.0.0" # VS2019 16.8, aka MSVC 14.28
TOOLCHAIN_MINIMUM_VERSION_xlc="16.1.0.0011"
@@ -389,6 +389,10 @@ AC_DEFUN_ONCE([TOOLCHAIN_POST_DETECTION],
# This is necessary since AC_PROG_CC defaults CFLAGS to "-g -O2"
CFLAGS="$ORG_CFLAGS"
CXXFLAGS="$ORG_CXXFLAGS"
+
+ # filter out some unwanted additions autoconf may add to CXX; we saw this on macOS with autoconf 2.72
+ UTIL_GET_NON_MATCHING_VALUES(cxx_filtered, $CXX, -std=c++11 -std=gnu++11)
+ CXX="$cxx_filtered"
])
# Check if a compiler is of the toolchain type we expect, and save the version
@@ -728,11 +732,10 @@ AC_DEFUN_ONCE([TOOLCHAIN_DETECT_TOOLCHAIN_CORE],
AC_SUBST(AS)
#
- # Setup the archiver (AR)
+ # Setup tools for creating static libraries (AR/LIB)
#
if test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
- # The corresponding ar tool is lib.exe (used to create static libraries)
- UTIL_LOOKUP_TOOLCHAIN_PROGS(AR, lib)
+ UTIL_LOOKUP_TOOLCHAIN_PROGS(LIB, lib)
elif test "x$TOOLCHAIN_TYPE" = xgcc; then
UTIL_LOOKUP_TOOLCHAIN_PROGS(AR, ar gcc-ar)
else
diff --git a/make/autoconf/util.m4 b/make/autoconf/util.m4
index 349a04a089ee4..3fae951224e3b 100644
--- a/make/autoconf/util.m4
+++ b/make/autoconf/util.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -199,7 +199,7 @@ AC_DEFUN([UTIL_GET_NON_MATCHING_VALUES],
if test -z "$legal_values"; then
$1="$2"
else
- result=`$GREP -Fvx "$legal_values" <<< "$values_to_check" | $GREP -v '^$'`
+ result=`$GREP -Fvx -- "$legal_values" <<< "$values_to_check" | $GREP -v '^$'`
$1=${result//$'\n'/ }
fi
])
@@ -226,7 +226,7 @@ AC_DEFUN([UTIL_GET_MATCHING_VALUES],
if test -z "$illegal_values"; then
$1=""
else
- result=`$GREP -Fx "$illegal_values" <<< "$values_to_check" | $GREP -v '^$'`
+ result=`$GREP -Fx -- "$illegal_values" <<< "$values_to_check" | $GREP -v '^$'`
$1=${result//$'\n'/ }
fi
])
diff --git a/make/common/MakeBase.gmk b/make/common/MakeBase.gmk
index 252d9dd50da68..3858b652ee65c 100644
--- a/make/common/MakeBase.gmk
+++ b/make/common/MakeBase.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -152,6 +152,10 @@ define SetupLogging
endif
endif
+ ifneq ($$(findstring $$(LOG_LEVEL), debug trace),)
+ SHELL := $$(SHELL) -x
+ endif
+
ifeq ($$(LOG_LEVEL), trace)
SHELL_NO_RECURSE := $$(SHELL)
# Shell redefinition trick inspired by http://www.cmcrossroads.com/ask-mr-make/6535-tracing-rule-execution-in-gnu-make
diff --git a/make/common/NativeCompilation.gmk b/make/common/NativeCompilation.gmk
index 68d1dba27ffcc..13b0318b4c776 100644
--- a/make/common/NativeCompilation.gmk
+++ b/make/common/NativeCompilation.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -23,9 +23,11 @@
# questions.
#
-# When you read this source. Remember that $(sort ...) has the side effect
-# of removing duplicates. It is actually this side effect that is
-# desired whenever sort is used below!
+################################################################################
+# This is the top-level entry point for our native compilation and linking.
+# It contains the SetupNativeCompilation macro, but is supported by helper
+# macros in the make/common/native directory.
+################################################################################
ifndef _NATIVE_COMPILATION_GMK
_NATIVE_COMPILATION_GMK := 1
@@ -34,463 +36,14 @@ ifeq ($(_MAKEBASE_GMK), )
$(error You must include MakeBase.gmk prior to including NativeCompilation.gmk)
endif
-################################################################################
-# Create exported symbols file for static libraries
-################################################################################
-
-# get the exported symbols from mapfiles and if there
-# is no mapfile, get them from the archive
-define GetSymbols
- $(RM) $$(@D)/$$(basename $$(@F)).symbols; \
- if [ ! -z $$($1_MAPFILE) -a -e $$($1_MAPFILE) ]; then \
- $(ECHO) "Getting symbols from mapfile $$($1_MAPFILE)"; \
- $(AWK) '/global:/','/local:/' $$($1_MAPFILE) | \
- $(SED) -e 's/#.*//;s/global://;s/local://;s/\;//;s/^[ ]*/_/;/^_$$$$/d' | \
- $(EGREP) -v "JNI_OnLoad|JNI_OnUnload|Agent_OnLoad|Agent_OnUnload|Agent_OnAttach" > \
- $$(@D)/$$(basename $$(@F)).symbols || true; \
- $(NM) $(NMFLAGS) $$($1_TARGET) | $(GREP) " T " | \
- $(EGREP) "JNI_OnLoad|JNI_OnUnload|Agent_OnLoad|Agent_OnUnload|Agent_OnAttach" | \
- $(CUT) -d ' ' -f 3 >> $$(@D)/$$(basename $$(@F)).symbols || true;\
- else \
- $(ECHO) "Getting symbols from nm"; \
- $(NM) $(NMFLAGS) -m $$($1_TARGET) | $(GREP) "__TEXT" | \
- $(EGREP) -v "non-external|private extern|__TEXT,__eh_frame" | \
- $(SED) -e 's/.* //' > $$(@D)/$$(basename $$(@F)).symbols; \
- fi
-endef
-
-################################################################################
-# Creates a recipe that creates a compile_commands.json fragment. Remove any
-# occurrences of FIXPATH programs from the command to show the actual invocation.
-#
-# Param 1: Name of file to create
-# Param 2: Working directory
-# Param 3: Source file
-# Param 4: Compile command
-################################################################################
-define WriteCompileCommandsFragment
- $(call LogInfo, Creating compile commands fragment for $(notdir $3))
- $(call MakeDir, $(dir $1))
- $(call WriteFile,{ \
- "directory": "$(strip $(call FixPath, $2))"$(COMMA) \
- "file": "$(strip $(call FixPath, $3))"$(COMMA) \
- "command": "$(strip $(subst $(DQUOTE),\$(DQUOTE),$(subst \,\\,\
- $(subst $(FIXPATH),,$(call FixPath, $4)))))" \
- }$(COMMA), \
- $1)
-endef
-
-################################################################################
-# Define a native toolchain configuration that can be used by
-# SetupNativeCompilation calls
-#
-# Parameter 1 is the name of the toolchain definition
-#
-# Remaining parameters are named arguments:
-# EXTENDS - Optional parent definition to get defaults from
-# CC - The C compiler
-# CXX - The C++ compiler
-# LD - The Linker
-# AR - Static linker
-# AS - Assembler
-# MT - Windows MT tool
-# RC - Windows RC tool
-# OBJCOPY - The objcopy tool for debug symbol handling
-# STRIP - The tool to use for stripping debug symbols
-# SYSROOT_CFLAGS - Compiler flags for using the specific sysroot
-# SYSROOT_LDFLAGS - Linker flags for using the specific sysroot
-DefineNativeToolchain = $(NamedParamsMacroTemplate)
-define DefineNativeToolchainBody
- # If extending another definition, get default values from that,
- # otherwise, nothing more needs to be done as variable assignments
- # already happened in NamedParamsMacroTemplate.
- ifneq ($$($1_EXTENDS), )
- $$(call SetIfEmpty, $1_CC, $$($$($1_EXTENDS)_CC))
- $$(call SetIfEmpty, $1_CXX, $$($$($1_EXTENDS)_CXX))
- $$(call SetIfEmpty, $1_LD, $$($$($1_EXTENDS)_LD))
- $$(call SetIfEmpty, $1_AR, $$($$($1_EXTENDS)_AR))
- $$(call SetIfEmpty, $1_AS, $$($$($1_EXTENDS)_AS))
- $$(call SetIfEmpty, $1_MT, $$($$($1_EXTENDS)_MT))
- $$(call SetIfEmpty, $1_RC, $$($$($1_EXTENDS)_RC))
- $$(call SetIfEmpty, $1_OBJCOPY, $$($$($1_EXTENDS)_OBJCOPY))
- $$(call SetIfEmpty, $1_STRIP, $$($$($1_EXTENDS)_STRIP))
- $$(call SetIfEmpty, $1_SYSROOT_CFLAGS, $$($$($1_EXTENDS)_SYSROOT_CFLAGS))
- $$(call SetIfEmpty, $1_SYSROOT_LDFLAGS, $$($$($1_EXTENDS)_SYSROOT_LDFLAGS))
- endif
-endef
-
-# Create a default toolchain with the main compiler and linker
-$(eval $(call DefineNativeToolchain, TOOLCHAIN_DEFAULT, \
- CC := $(CC), \
- CXX := $(CXX), \
- LD := $(LD), \
- AR := $(AR), \
- AS := $(AS), \
- MT := $(MT), \
- RC := $(RC), \
- OBJCOPY := $(OBJCOPY), \
- STRIP := $(STRIP), \
- SYSROOT_CFLAGS := $(SYSROOT_CFLAGS), \
- SYSROOT_LDFLAGS := $(SYSROOT_LDFLAGS), \
-))
-
-# Create a toolchain where linking is done with the C++ linker
-$(eval $(call DefineNativeToolchain, TOOLCHAIN_LINK_CXX, \
- EXTENDS := TOOLCHAIN_DEFAULT, \
- LD := $(LDCXX), \
-))
-
-# Create a toolchain with the BUILD compiler, used for build tools that
-# are to be run during the build.
-$(eval $(call DefineNativeToolchain, TOOLCHAIN_BUILD, \
- CC := $(BUILD_CC), \
- CXX := $(BUILD_CXX), \
- LD := $(BUILD_LD), \
- AR := $(BUILD_AR), \
- AS := $(BUILD_AS), \
- OBJCOPY := $(BUILD_OBJCOPY), \
- STRIP := $(BUILD_STRIP), \
- SYSROOT_CFLAGS := $(BUILD_SYSROOT_CFLAGS), \
- SYSROOT_LDFLAGS := $(BUILD_SYSROOT_LDFLAGS), \
-))
-
-# BUILD toolchain with the C++ linker
-$(eval $(call DefineNativeToolchain, TOOLCHAIN_BUILD_LINK_CXX, \
- EXTENDS := TOOLCHAIN_BUILD, \
- LD := $(BUILD_LDCXX), \
-))
-
-################################################################################
-
-# Extensions of files handled by this macro.
-NATIVE_SOURCE_EXTENSIONS := %.S %.c %.cpp %.cc %.m %.mm
-
-# Replaces native source extensions with the object file extension in a string.
-# Param 1: the string containing source file names with extensions
-# The surrounding strip is needed to keep additional whitespace out
-define replace_with_obj_extension
-$(strip \
- $(foreach extension, $(NATIVE_SOURCE_EXTENSIONS), \
- $(patsubst $(extension),%$(OBJ_SUFFIX), $(filter $(extension), $1))) \
-)
-endef
-
-# This pattern is used to transform the output of the microsoft CL compiler
-# into a make syntax dependency file (.d)
-WINDOWS_SHOWINCLUDE_SED_PATTERN := \
- -e '/^Note: including file:/!d' \
- -e 's|Note: including file: *||' \
- -e 's|\r||g' \
- -e 's|\\|/|g' \
- -e 's|^\([a-zA-Z]\):|$(WINENV_PREFIX)/\1|g' \
- -e '\|$(TOPDIR)|I !d' \
- -e 's|$$$$| \\|g' \
- #
-
-# This pattern is used to transform a dependency file (.d) to a list
-# of make targets for dependent files (.d.targets)
-DEPENDENCY_TARGET_SED_PATTERN := \
- -e 's/\#.*//' \
- -e 's/^[^:]*: *//' \
- -e 's/ *\\$$$$//' \
- -e 's/^[ ]*//' \
- -e '/^$$$$/ d' \
- -e 's/$$$$/ :/' \
- #
-
-################################################################################
-# When absolute paths are not allowed in the output, and the compiler does not
-# support any options to avoid it, we need to rewrite compile commands to use
-# relative paths. By doing this, the __FILE__ macro will resolve to relative
-# paths. The relevant input paths on the command line are the -I flags and the
-# path to the source file itself.
-#
-# The macro MakeCommandRelative is used to rewrite the command line like this:
-# 'CD $(WORKSPACE_ROOT) && '
-# and changes all paths in cmd to be relative to the workspace root. This only
-# works properly if the build dir is inside the workspace root. If it's not,
-# relative paths are still calculated, but depending on the distance between the
-# dirs, paths in the build dir may end up as essentially absolute anyway.
-#
-# The fix-deps-file macro is used to adjust the contents of the generated make
-# dependency files to contain paths compatible with make.
-#
-REWRITE_PATHS_RELATIVE = false
-ifeq ($(ALLOW_ABSOLUTE_PATHS_IN_OUTPUT)-$(FILE_MACRO_CFLAGS), false-)
- REWRITE_PATHS_RELATIVE = true
-endif
-
-# CCACHE_BASEDIR needs fix-deps-file as makefiles use absolute filenames for
-# object files while CCACHE_BASEDIR will make ccache relativize all paths for
-# its compiler. The compiler then produces relative dependency files.
-# make does not know a relative and absolute filename is the same so it will
-# ignore such dependencies. This only applies when the OUTPUTDIR is inside
-# the WORKSPACE_ROOT.
-ifneq ($(CCACHE), )
- ifneq ($(filter $(WORKSPACE_ROOT)/%, $(OUTPUTDIR)), )
- REWRITE_PATHS_RELATIVE = true
- endif
-endif
-
-ifeq ($(REWRITE_PATHS_RELATIVE), true)
- # Need to handle -I flags as both '-Ifoo' and '-I foo'.
- MakeCommandRelative = \
- $(CD) $(WORKSPACE_ROOT) && \
- $(foreach o, $1, \
- $(if $(filter $(WORKSPACE_ROOT)/% $(OUTPUTDIR)/%, $o), \
- $(call RelativePath, $o, $(WORKSPACE_ROOT)) \
- , \
- $(if $(filter -I$(WORKSPACE_ROOT)/%, $o), \
- -I$(call RelativePath, $(patsubst -I%, %, $o), $(WORKSPACE_ROOT)) \
- , \
- $o \
- ) \
- ) \
- )
-
- # When compiling with relative paths, the deps file may come out with relative
- # paths, and that path may start with './'. First remove any leading ./, then
- # add WORKSPACE_ROOT to any line not starting with /, while allowing for
- # leading spaces. There may also be multiple entries on the same line, so start
- # with splitting such lines.
- # Non GNU sed (BSD on macosx) cannot substitute in literal \n using regex.
- # Instead use a bash escaped literal newline. To avoid having unmatched quotes
- # ruin the ability for an editor to properly syntax highlight this file, define
- # that newline sequence as a separate variable and add the closing quote behind
- # a comment.
- sed_newline := \'$$'\n''#'
- define fix-deps-file
- $(SED) \
- -e 's|\([^ ]\) \{1,\}\([^\\:]\)|\1 \\$(sed_newline) \2|g' \
- $1.tmp \
- | $(SED) \
- -e 's|^\([ ]*\)\./|\1|' \
- -e '/^[ ]*[^/ ]/s|^\([ ]*\)|\1$(WORKSPACE_ROOT)/|' \
- > $1
- endef
-else
- # By default the MakeCommandRelative macro does nothing.
- MakeCommandRelative = $1
-
- # No adjustment is needed.
- define fix-deps-file
- $(MV) $1.tmp $1
- endef
-endif
-
-################################################################################
-# GetEntitlementsFile
-# Find entitlements file for executable when signing on macosx. If no
-# specialized file is found, returns the default file.
-# $1 Executable to find entitlements file for.
-ENTITLEMENTS_DIR := $(TOPDIR)/make/data/macosxsigning
-ifeq ($(MACOSX_CODESIGN_MODE), debug)
- CODESIGN_PLIST_SUFFIX := -debug
-else
- CODESIGN_PLIST_SUFFIX :=
-endif
-DEFAULT_ENTITLEMENTS_FILE := $(ENTITLEMENTS_DIR)/default$(CODESIGN_PLIST_SUFFIX).plist
-
-GetEntitlementsFile = \
- $(foreach f, $(ENTITLEMENTS_DIR)/$(strip $(notdir $1))$(CODESIGN_PLIST_SUFFIX).plist, \
- $(if $(wildcard $f), $f, $(DEFAULT_ENTITLEMENTS_FILE)) \
- )
+include native/CompileFile.gmk
+include native/DebugSymbols.gmk
+include native/Flags.gmk
+include native/Link.gmk
+include native/LinkMicrosoft.gmk
+include native/Paths.gmk
################################################################################
-# Create the recipe needed to compile a single native source file.
-#
-# Parameter 1 is the name of the rule, based on the name of the library/
-# program being build and the name of the source code file, e.g.
-# BUILD_LIBFOO_fooMain.cpp.
-#
-# Remaining parameters are named arguments:
-# FILE - The full path of the source file to compiler
-# BASE - The name of the rule for the entire binary to build ($1)
-#
-SetupCompileNativeFile = $(NamedParamsMacroTemplate)
-define SetupCompileNativeFileBody
- $1_FILENAME := $$(notdir $$($1_FILE))
-
- # The target file to be generated.
- $1_OBJ := $$($$($1_BASE)_OBJECT_DIR)/$$(call replace_with_obj_extension, \
- $$($1_FILENAME))
-
- # Generate the corresponding compile_commands.json fragment.
- $1_OBJ_JSON = $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
- $$(OUTPUTDIR)/,,$$($1_OBJ))).json
- $$($1_BASE)_ALL_OBJS_JSON += $$($1_OBJ_JSON)
-
- # Only continue if this object file hasn't been processed already. This lets
- # the first found source file override any other with the same name.
- ifeq ($$($1_OBJ_PROCESSED), )
- $1_OBJ_PROCESSED := true
- # This is the definite source file to use for $1_FILENAME.
- $1_SRC_FILE := $$($1_FILE)
-
- ifeq ($$($1_OPTIMIZATION), )
- $1_OPT_CFLAGS := $$($$($1_BASE)_OPT_CFLAGS)
- $1_OPT_CXXFLAGS := $$($$($1_BASE)_OPT_CXXFLAGS)
- else
- ifeq ($$($1_OPTIMIZATION), NONE)
- $1_OPT_CFLAGS := $(C_O_FLAG_NONE)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NONE)
- else ifeq ($$($1_OPTIMIZATION), LOW)
- $1_OPT_CFLAGS := $(C_O_FLAG_NORM)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NORM)
- else ifeq ($$($1_OPTIMIZATION), HIGH)
- $1_OPT_CFLAGS := $(C_O_FLAG_HI)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HI)
- else ifeq ($$($1_OPTIMIZATION), HIGHEST)
- $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST)
- else ifeq ($$($1_OPTIMIZATION), HIGHEST_JVM)
- $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST_JVM)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST_JVM)
- else ifeq ($$($1_OPTIMIZATION), SIZE)
- $1_OPT_CFLAGS := $(C_O_FLAG_SIZE)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_SIZE)
- else
- $$(error Unknown value for file OPTIMIZATION: $$($1_OPTIMIZATION))
- endif
- endif
-
- ifneq ($$($$($1_BASE)_PRECOMPILED_HEADER), )
- ifeq ($$(filter $$($1_FILENAME), $$($$($1_BASE)_PRECOMPILED_HEADER_EXCLUDE)), )
- $1_USE_PCH_FLAGS := $$($$($1_BASE)_USE_PCH_FLAGS)
- endif
- endif
-
- ifneq ($(DISABLE_WARNING_PREFIX), )
- $1_WARNINGS_FLAGS := $$(addprefix $(DISABLE_WARNING_PREFIX), \
- $$($$($1_BASE)_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$$($1_FILENAME)) \
- $$($$($1_BASE)_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)_$$($1_FILENAME)))
- endif
-
- $1_BASE_CFLAGS := $$($$($1_BASE)_CFLAGS) $$($$($1_BASE)_EXTRA_CFLAGS) \
- $$($$($1_BASE)_SYSROOT_CFLAGS)
- $1_BASE_CXXFLAGS := $$($$($1_BASE)_CXXFLAGS) $$($$($1_BASE)_EXTRA_CXXFLAGS) \
- $$($$($1_BASE)_SYSROOT_CFLAGS) $$($1_EXTRA_CXXFLAGS)
- $1_BASE_ASFLAGS := $$($$($1_BASE)_ASFLAGS) $$($$($1_BASE)_EXTRA_ASFLAGS)
-
- ifneq ($$(filter %.c, $$($1_FILENAME)), )
- # Compile as a C file
- $1_CFLAGS += $$($1_WARNINGS_FLAGS)
- $1_FLAGS := $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) $$($1_BASE_CFLAGS) \
- $$($1_OPT_CFLAGS) $$($1_CFLAGS) -c
- $1_COMPILER := $$($$($1_BASE)_CC)
- else ifneq ($$(filter %.m, $$($1_FILENAME)), )
- # Compile as an Objective-C file
- $1_CFLAGS += $$($1_WARNINGS_FLAGS)
- $1_FLAGS := -x objective-c $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) \
- $$($1_BASE_CFLAGS) $$($1_OPT_CFLAGS) $$($1_CFLAGS) -c
- $1_COMPILER := $$($$($1_BASE)_CC)
- else ifneq ($$(filter %.S, $$($1_FILENAME)), )
- # Compile as preprocessed assembler file
- $1_FLAGS := $(BASIC_ASFLAGS) $$($1_BASE_ASFLAGS)
- $1_COMPILER := $(AS)
-
- # gcc or clang assembly files must contain an appropriate relative .file
- # path for reproducible builds.
- ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
- # If no absolute paths allowed, work out relative source file path
- # for assembly .file substitution, otherwise use full file path
- ifeq ($(ALLOW_ABSOLUTE_PATHS_IN_OUTPUT), false)
- $1_REL_ASM_SRC := $$(call RelativePath, $$($1_FILE), $(WORKSPACE_ROOT))
- else
- $1_REL_ASM_SRC := $$($1_FILE)
- endif
- $1_FLAGS := $$($1_FLAGS) -DASSEMBLY_SRC_FILE='"$$($1_REL_ASM_SRC)"' \
- -include $(TOPDIR)/make/data/autoheaders/assemblyprefix.h
- endif
- else ifneq ($$(filter %.cpp %.cc %.mm, $$($1_FILENAME)), )
- # Compile as a C++ or Objective-C++ file
- $1_CXXFLAGS += $$($1_WARNINGS_FLAGS)
- $1_FLAGS := $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) $$($1_BASE_CXXFLAGS) \
- $$($1_OPT_CXXFLAGS) $$($1_CXXFLAGS) -c
- $1_COMPILER := $$($$($1_BASE)_CXX)
- else
- $$(error Internal error in NativeCompilation.gmk: no compiler for file $$($1_FILENAME))
- endif
-
- # And this is the dependency file for this obj file.
- $1_DEPS_FILE := $$(patsubst %$(OBJ_SUFFIX),%.d,$$($1_OBJ))
- # The dependency target file lists all dependencies as empty targets to
- # avoid make error "No rule to make target" for removed files
- $1_DEPS_TARGETS_FILE := $$(patsubst %$(OBJ_SUFFIX),%.d.targets,$$($1_OBJ))
-
- # Only try to load individual dependency information files if the global
- # file hasn't been loaded (could happen if make was interrupted).
- ifneq ($$($$($1_BASE)_DEPS_FILE_LOADED), true)
- # Include previously generated dependency information. (if it exists)
- -include $$($1_DEPS_FILE)
- -include $$($1_DEPS_TARGETS_FILE)
- endif
-
- ifneq ($$(strip $$($1_CFLAGS) $$($1_CXXFLAGS) $$($1_OPTIMIZATION)), )
- $1_VARDEPS := $$($1_CFLAGS) $$($1_CXXFLAGS) $$($1_OPTIMIZATION)
- $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, $$($1_OBJ).vardeps)
- endif
-
- $1_OBJ_DEPS := $$($1_SRC_FILE) $$($$($1_BASE)_COMPILE_VARDEPS_FILE) \
- $$($$($1_BASE)_EXTRA_DEPS) $$($1_VARDEPS_FILE)
- $1_COMPILE_OPTIONS := $$($1_FLAGS) $(CC_OUT_OPTION)$$($1_OBJ) $$($1_SRC_FILE)
- # For reproducible builds with gcc and clang ensure random symbol generation is
- # seeded deterministically
- ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
- $1_COMPILE_OPTIONS += -frandom-seed="$$($1_FILENAME)"
- endif
-
- $$($1_OBJ_JSON): $$($1_OBJ_DEPS)
- $$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$($1_SRC_FILE), \
- $$($1_COMPILER) $$($1_COMPILE_OPTIONS))
-
- $$($1_OBJ): $$($1_OBJ_DEPS) | $$($$($1_BASE)_BUILD_INFO)
- $$(call LogInfo, Compiling $$($1_FILENAME) (for $$($$($1_BASE)_BASENAME)))
- $$(call MakeDir, $$(@D))
- ifneq ($(TOOLCHAIN_TYPE), microsoft)
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_COMPILER) $$(GENDEPS_FLAGS) \
- $$(addsuffix .tmp, $$($1_DEPS_FILE)) \
- $$($1_COMPILE_OPTIONS)))
- ifneq ($$($1_DEPS_FILE), )
- $$(call fix-deps-file, $$($1_DEPS_FILE))
- # Create a dependency target file from the dependency file.
- # Solution suggested by:
- # http://make.mad-scientist.net/papers/advanced-auto-dependency-generation/
- $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_DEPS_FILE) \
- > $$($1_DEPS_TARGETS_FILE)
- endif
- else
- # The Visual Studio compiler lacks a feature for generating make
- # dependencies, but by setting -showIncludes, all included files are
- # printed. These are filtered out and parsed into make dependences.
- #
- # Keep as much as possible on one execution line for best performance
- # on Windows. No need to save exit code from compilation since
- # pipefail is always active on Windows.
- ifeq ($$(filter %.S, $$($1_FILENAME)), )
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_COMPILER) -showIncludes $$($1_COMPILE_OPTIONS))) \
- | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
- -e "^$$($1_FILENAME)$$$$" || test "$$$$?" = "1" ; \
- $(ECHO) $$@: \\ > $$($1_DEPS_FILE) ; \
- $(SED) $(WINDOWS_SHOWINCLUDE_SED_PATTERN) $$($1_OBJ).log \
- | $(SORT) -u >> $$($1_DEPS_FILE) ; \
- $(ECHO) >> $$($1_DEPS_FILE) ; \
- $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_DEPS_FILE) > $$($1_DEPS_TARGETS_FILE)
- else
- # For assembler calls just create empty dependency lists
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_COMPILER) $$($1_FLAGS) \
- $(CC_OUT_OPTION)$$($1_OBJ) -Ta $$($1_SRC_FILE))) \
- | $(TR) -d '\r' | $(GREP) -v -e "Assembling:" || test "$$$$?" = "1" ; \
- $(ECHO) > $$($1_DEPS_FILE) ; \
- $(ECHO) > $$($1_DEPS_TARGETS_FILE)
- endif
- endif
- endif
-endef
-
# Setup make rules for creating a native binary (a shared library or an
# executable).
#
@@ -501,7 +54,8 @@ endef
# NAME The base name for the resulting binary, excluding decorations (like *.exe)
# TYPE Type of binary (EXECUTABLE, LIBRARY or STATIC_LIBRARY). Default is LIBRARY.
# SUFFIX Override the default suffix for the output file
-# TOOLCHAIN Name of toolchain setup to use. Defaults to TOOLCHAIN_DEFAULT.
+# TARGET_TYPE The type to target, BUILD or TARGET. Defaults to TARGET.
+# LINK_TYPE The language to use for the linker, C or C++. Defaults to C.
# SRC one or more directory roots to scan for C/C++ files.
# CFLAGS the compiler flags to be used, used both for C and C++.
# CXXFLAGS the compiler flags to be used for c++, if set overrides CFLAGS.
@@ -519,7 +73,8 @@ endef
# used both for C and C++.
# LIBS__ the libraries to link to for the specified target
# OS and toolchain, used both for C and C++.
-# ARFLAGS the archiver flags to be used
+# ARFLAGS the archiver flags to be used on unix platforms
+# LIBFLAGS the flags for the lib tool used on windows
# OBJECT_DIR the directory where we store the object files
# OUTPUT_DIR the directory where the resulting binary is put
# SYMBOLS_DIR the directory where the debug symbols are put, defaults to OUTPUT_DIR
@@ -534,11 +89,18 @@ endef
# VERSIONINFO_RESOURCE Input file for RC. Setting this implies that RC will be run
# RCFLAGS flags for RC.
# EMBED_MANIFEST if true, embed manifest on Windows.
-# MAPFILE mapfile
-# USE_MAPFILE_FOR_SYMBOLS if true and this is a STATIC_BUILD, just copy the
-# mapfile for the output symbols file
-# CC the compiler to use, default is $(CC)
-# LD the linker to use, default is $(LD)
+# CC the C compiler to use
+# CXX the C++ compiler to use
+# LD the Linker to use
+# AR the static linker to use
+# LIB the Windows lib tool to use for creating static libraries
+# AS the assembler to use
+# MT the Windows MT tool to use
+# RC the Windows RC tool to use
+# OBJCOPY the objcopy tool for debug symbol handling
+# STRIP the tool to use for stripping debug symbols
+# SYSROOT_CFLAGS the compiler flags for using the specific sysroot
+# SYSROOT_LDFLAGS the linker flags for using the specific sysroot
# OPTIMIZATION sets optimization level to NONE, LOW, HIGH, HIGHEST, HIGHEST_JVM, SIZE
# DISABLED_WARNINGS_ Disable the given warnings for the specified toolchain
# DISABLED_WARNINGS__ Disable the given warnings for the specified
@@ -573,9 +135,122 @@ endef
# TARGET_DEPS All prerequisites for the target calculated by the macro
# ALL_OBJS All object files
# IMPORT_LIBRARY The import library created for a shared library on Windows
+#
SetupNativeCompilation = $(NamedParamsMacroTemplate)
define SetupNativeCompilationBody
+ # When reading this code, note that macros named Setup are just setting
+ # variables, and macros called Create are setting up rules to create
+ # files. Macros starting with any other verb are more complicated, and can do
+ # all of the above, and also call directly to the shell.
+
+ ###
+ ### Prepare for compilation and linking
+ ###
+
+ $$(eval $$(call VerifyArguments,$1))
+
+ # Setup variables for the rest of this macro to work with
+ $$(eval $$(call SetupBasicVariables,$1))
+
+ # Setup the toolchain to be used
+ $$(eval $$(call SetupToolchain,$1))
+
+ # Find all source files to compile and determine the output object file names
+ $$(eval $$(call SetupSourceFiles,$1))
+ $$(eval $$(call SetupOutputFiles,$1))
+
+ # Setup CFLAGS/CXXFLAGS based on warnings, optimizations, extra flags etc.
+ $$(eval $$(call SetupCompilerFlags,$1))
+
+ # Machinery needed for the build to function properly
+ $$(eval $$(call SetupBuildSystemSupport,$1))
+
+ $$(eval $$(call RemoveSuperfluousOutputFiles,$1))
+
+ # Need to make sure TARGET is first on list before starting to create files
+ $1 := $$($1_TARGET)
+
+ # Have make print information about the library when we start compiling
+ $$(eval $$(call PrintStartInfo,$1))
+
+ ###
+ ### Compile all native source code files
+ ###
+
+ # Create a PCH, if requested
+ $$(eval $$(call CreatePrecompiledHeader,$1))
+
+ # Now call CreateCompiledNativeFile for each source file we are going to compile.
+ $$(foreach file, $$($1_SRCS), \
+ $$(eval $$(call CreateCompiledNativeFile,$1_$$(notdir $$(file)),\
+ FILE := $$(file), \
+ BASE := $1, \
+ )) \
+ )
+
+ ifeq ($(call isTargetOs, windows), true)
+ # On windows we need to create a resource file
+ $$(eval $$(call CreateWindowsResourceFile,$1))
+ endif
+
+ # Setup a library-wide dependency file from individual object file dependency
+ # files, and import it in the makefile.
+ $$(eval $$(call CreateDependencyFile,$1))
+ $$(eval $$(call ImportDependencyFile,$1))
+
+ ###
+ ### Link the object files into a native output library/executable
+ ###
+
+ # Handle native debug symbols
+ $$(eval $$(call CreateDebugSymbols,$1))
+
+ # Prepare for linking
+ $$(eval $$(call SetupLinkerFlags,$1))
+ ifneq ($(TOOLCHAIN_TYPE), microsoft)
+ $$(eval $$(call SetupLinking,$1))
+ endif
+
+ $$(eval $$(call SetupObjectFileList,$1))
+
+ # Link the individually compiled files into a single unit
+ ifneq ($(TOOLCHAIN_TYPE), microsoft)
+ $$(eval $$(call CreateLinkedResult,$1))
+ else
+ $$(eval $$(call CreateLinkedResultMicrosoft,$1))
+ endif
+
+ ifeq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
+ # Override all targets (this is a hack)
+ $1 := $$($1_ALL_OBJS_JSON)
+ endif
+endef
+
+################################################################################
+# Verify that user passed arguments are valid
+define VerifyArguments
+ ifneq ($$($1_NAME), $(basename $$($1_NAME)))
+ $$(error NAME must not contain any directory path in $1)
+ endif
+ ifneq ($(findstring $$($1_SUFFIX), $$($1_NAME)), )
+ $$(error NAME should be specified without suffix: $$($1_SUFFIX) in $1)
+ endif
+ ifneq ($(findstring $$($1_PREFIX), $$($1_NAME)), )
+ $$(error NAME should be specified without prefix: $$($1_PREFIX) in $1)
+ endif
+ ifeq ($$($1_OUTPUT_DIR), )
+ $$(error OUTPUT_DIR is missing in $1)
+ endif
+ ifneq ($$($1_MANIFEST), )
+ ifeq ($$($1_MANIFEST_VERSION), )
+ $$(error If MANIFEST is provided, then MANIFEST_VERSION is required in $1)
+ endif
+ endif
+endef
+################################################################################
+# Setup basic variables
+define SetupBasicVariables
# If type is unspecified, default to LIBRARY
ifeq ($$($1_TYPE), )
$1_TYPE := LIBRARY
@@ -589,8 +264,6 @@ define SetupNativeCompilationBody
endif
endif
- $$(call SetIfEmpty, $1_COMPILE_WITH_DEBUG_SYMBOLS, $$(COMPILE_WITH_DEBUG_SYMBOLS))
-
# STATIC_LIBS is set from Main.gmk when building static versions of certain
# native libraries.
ifeq ($(STATIC_LIBS), true)
@@ -600,15 +273,6 @@ define SetupNativeCompilationBody
# jmods.
$1_OBJECT_DIR := $$($1_OBJECT_DIR)/static
$1_OUTPUT_DIR := $$($1_OBJECT_DIR)
- # For release builds where debug symbols are configured to be moved to
- # separate debuginfo files, disable debug symbols for static libs instead.
- # We don't currently support this configuration and we don't want symbol
- # information in release builds unless explicitly asked to provide it.
- ifeq ($(DEBUG_LEVEL), release)
- ifeq ($(COPY_DEBUG_SYMBOLS), true)
- $1_COMPILE_WITH_DEBUG_SYMBOLS := false
- endif
- endif
endif
ifeq ($$($1_TYPE), EXECUTABLE)
@@ -629,248 +293,53 @@ define SetupNativeCompilationBody
endif
endif
- ifneq ($$($1_NAME), $(basename $$($1_NAME)))
- $$(error NAME must not contain any directory path in $1)
- endif
- ifneq ($(findstring $$($1_SUFFIX), $$($1_NAME)), )
- $$(error NAME should be specified without suffix: $$($1_SUFFIX) in $1)
- endif
- ifneq ($(findstring $$($1_PREFIX), $$($1_NAME)), )
- $$(error NAME should be specified without prefix: $$($1_PREFIX) in $1)
- endif
- ifeq ($$($1_OUTPUT_DIR), )
- $$(error OUTPUT_DIR is missing in $1)
- endif
- ifneq ($$($1_MANIFEST), )
- ifeq ($$($1_MANIFEST_VERSION), )
- $$(error If MANIFEST is provided, then MANIFEST_VERSION is required in $1)
- endif
- endif
-
$1_BASENAME := $$($1_PREFIX)$$($1_NAME)$$($1_SUFFIX)
$1_TARGET := $$($1_OUTPUT_DIR)/$$($1_BASENAME)
$1_NOSUFFIX := $$($1_PREFIX)$$($1_NAME)
$1_SAFE_NAME := $$(strip $$(subst /,_, $1))
+endef
-# Need to make sure TARGET is first on list
- $1 := $$($1_TARGET)
-
- # Setup the toolchain to be used
- $$(call SetIfEmpty, $1_TOOLCHAIN, TOOLCHAIN_DEFAULT)
- $$(call SetIfEmpty, $1_CC, $$($$($1_TOOLCHAIN)_CC))
- $$(call SetIfEmpty, $1_CXX, $$($$($1_TOOLCHAIN)_CXX))
- $$(call SetIfEmpty, $1_LD, $$($$($1_TOOLCHAIN)_LD))
- $$(call SetIfEmpty, $1_AR, $$($$($1_TOOLCHAIN)_AR))
- $$(call SetIfEmpty, $1_AS, $$($$($1_TOOLCHAIN)_AS))
- $$(call SetIfEmpty, $1_MT, $$($$($1_TOOLCHAIN)_MT))
- $$(call SetIfEmpty, $1_RC, $$($$($1_TOOLCHAIN)_RC))
- $$(call SetIfEmpty, $1_OBJCOPY, $$($$($1_TOOLCHAIN)_OBJCOPY))
- $$(call SetIfEmpty, $1_STRIP, $$($$($1_TOOLCHAIN)_STRIP))
- $$(call SetIfEmpty, $1_SYSROOT_CFLAGS, $$($$($1_TOOLCHAIN)_SYSROOT_CFLAGS))
- $$(call SetIfEmpty, $1_SYSROOT_LDFLAGS, $$($$($1_TOOLCHAIN)_SYSROOT_LDFLAGS))
-
- $$(foreach d, $$($1_SRC), $$(if $$(wildcard $$d), , \
- $$(error SRC specified to SetupNativeCompilation $1 contains missing directory $$d)))
-
- $1_SRCS_RAW := $$(call FindFiles, $$($1_SRC))
- # Order src files according to the order of the src dirs
- $1_SRCS := $$(foreach d, $$($1_SRC), $$(filter $$d%, $$($1_SRCS_RAW)))
- $1_SRCS := $$(filter $$(NATIVE_SOURCE_EXTENSIONS), $$($1_SRCS))
- # Extract the C/C++ files.
- ifneq ($$($1_EXCLUDE_PATTERNS), )
- # We must not match the exclude pattern against the src root(s).
- $1_SRCS_WITHOUT_ROOTS := $$($1_SRCS)
- $$(foreach i, $$($1_SRC), $$(eval $1_SRCS_WITHOUT_ROOTS := $$(patsubst \
- $$i/%,%, $$($1_SRCS_WITHOUT_ROOTS))))
- $1_ALL_EXCLUDE_FILES := $$(call containing, $$($1_EXCLUDE_PATTERNS), \
- $$($1_SRCS_WITHOUT_ROOTS))
- endif
- ifneq ($$($1_EXCLUDE_FILES), )
- $1_ALL_EXCLUDE_FILES += $$($1_EXCLUDE_FILES)
- endif
- ifneq ($$($1_ALL_EXCLUDE_FILES), )
- $1_EXCLUDE_FILES_PAT := $$($1_ALL_EXCLUDE_FILES) \
- $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$($1_ALL_EXCLUDE_FILES)))
- $1_EXCLUDE_FILES_PAT := $$(addprefix %, $$($1_EXCLUDE_FILES_PAT))
- $1_SRCS := $$(filter-out $$($1_EXCLUDE_FILES_PAT), $$($1_SRCS))
- endif
- ifneq ($$($1_INCLUDE_FILES), )
- $1_INCLUDE_FILES_PAT := $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$($1_INCLUDE_FILES)))
- $1_SRCS := $$(filter $$($1_INCLUDE_FILES_PAT), $$($1_SRCS))
- endif
- # There can be only a single bin dir root, no need to foreach over the roots.
- $1_BINS := $$(wildcard $$($1_OBJECT_DIR)/*$(OBJ_SUFFIX))
- # Now we have a list of all c/c++ files to compile: $$($1_SRCS)
- # and we have a list of all existing object files: $$($1_BINS)
-
- # Prepend the source/bin path to the filter expressions. Then do the filtering.
- ifneq ($$($1_INCLUDES), )
- $1_SRC_INCLUDES := $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$(addsuffix /%, $$($1_INCLUDES))))
- $1_SRCS := $$(filter $$($1_SRC_INCLUDES), $$($1_SRCS))
- endif
- ifneq ($$($1_EXCLUDES), )
- $1_SRC_EXCLUDES := $$(addsuffix /%, $$($1_EXCLUDES))
- $1_SRC_EXCLUDES += $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$(addsuffix /%, $$($1_EXCLUDES))))
- $1_SRCS := $$(filter-out $$($1_SRC_EXCLUDES), $$($1_SRCS))
- endif
-
- $1_SRCS += $$($1_EXTRA_FILES)
-
- ifeq ($$($1_SRCS), )
- $$(error No sources found for $1 when looking inside the dirs $$($1_SRC))
- endif
-
- ifeq ($$($1_TYPE), EXECUTABLE)
- ifeq ($(UBSAN_ENABLED), true)
- # We need to set the default options for UBSan. This needs to be included in every executable.
- # Rather than copy and paste code to everything with a main function, we add an additional
- # source file to every executable that exports __ubsan_default_options.
- ifneq ($$(filter %.cpp %.cc, $$($1_SRCS)), )
- $1_SRCS += $(TOPDIR)/make/data/ubsan/ubsan_default_options.cpp
- else
- $1_SRCS += $(TOPDIR)/make/data/ubsan/ubsan_default_options.c
- endif
- endif
- endif
-
- # Calculate the expected output from compiling the sources
- $1_EXPECTED_OBJS_FILENAMES := $$(call replace_with_obj_extension, $$(notdir $$($1_SRCS)))
- $1_EXPECTED_OBJS := $$(addprefix $$($1_OBJECT_DIR)/, $$($1_EXPECTED_OBJS_FILENAMES))
- # Are there too many object files on disk? Perhaps because some source file was removed?
- $1_SUPERFLOUS_OBJS := $$(sort $$(filter-out $$($1_EXPECTED_OBJS), $$($1_BINS)))
- # Clean out the superfluous object files.
- ifneq ($$($1_SUPERFLUOUS_OBJS), )
- $$(shell $(RM) -f $$($1_SUPERFLUOUS_OBJS))
- endif
- # Sort to remove duplicates and provide a reproducible order on the input files to the linker.
- $1_ALL_OBJS := $$(sort $$($1_EXPECTED_OBJS) $$($1_EXTRA_OBJECT_FILES))
- ifeq ($(STATIC_LIBS), true)
- # Exclude the object files that match with $1_STATIC_LIB_EXCLUDE_OBJS.
- ifneq ($$($1_STATIC_LIB_EXCLUDE_OBJS), )
- $1_ALL_OBJS := $$(call not-containing, $$($1_STATIC_LIB_EXCLUDE_OBJS), $$($1_ALL_OBJS))
+################################################################################
+# Setup the toolchain variables
+define SetupToolchain
+ ifeq ($$($1_TARGET_TYPE), BUILD)
+ $$(call SetIfEmpty, $1_CC, $(BUILD_CC))
+ $$(call SetIfEmpty, $1_CXX, $(BUILD_CXX))
+ $$(call SetIfEmpty, $1_AR, $(BUILD_AR))
+ $$(call SetIfEmpty, $1_LIB, $(BUILD_LIB))
+ $$(call SetIfEmpty, $1_AS, $(BUILD_AS))
+ $$(call SetIfEmpty, $1_OBJCOPY, $(BUILD_OBJCOPY))
+ $$(call SetIfEmpty, $1_STRIP, $(BUILD_STRIP))
+ $$(call SetIfEmpty, $1_SYSROOT_CFLAGS, $(BUILD_SYSROOT_CFLAGS))
+ $$(call SetIfEmpty, $1_SYSROOT_LDFLAGS, $(BUILD_SYSROOT_LDFLAGS))
+ ifeq ($$($1_LINK_TYPE), C++)
+ $$(call SetIfEmpty, $1_LD, $(BUILD_LDCXX))
+ else
+ $$(call SetIfEmpty, $1_LD, $(BUILD_LD))
endif
- endif
-
- # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS, TOOLCHAIN_TYPE and
- # OPENJDK_TARGET_OS plus OPENJDK_TARGET_CPU pair dependent variables for CFLAGS.
- $1_EXTRA_CFLAGS := $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_CFLAGS_$(OPENJDK_TARGET_OS)) \
- $$($1_CFLAGS_$(TOOLCHAIN_TYPE)) \
- $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU))
-
- ifneq ($(DEBUG_LEVEL), release)
- # Pickup extra debug dependent variables for CFLAGS
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_debug)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)_debug)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_debug)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU)_debug)
- else
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_release)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)_release)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_release)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU)_release)
- endif
- ifeq ($(STATIC_LIBS), true)
- $1_EXTRA_CFLAGS += $$(STATIC_LIBS_CFLAGS)
- endif
-
- # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS and/or TOOLCHAIN_TYPE
- # dependent variables for CXXFLAGS.
- $1_EXTRA_CXXFLAGS := $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)) \
- $$($1_CXXFLAGS_$(TOOLCHAIN_TYPE))
-
- ifneq ($(DEBUG_LEVEL), release)
- # Pickup extra debug dependent variables for CXXFLAGS
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_debug)
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)_debug)
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)_debug)
else
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_release)
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)_release)
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)_release)
- endif
- ifeq ($(STATIC_LIBS), true)
- $1_EXTRA_CXXFLAGS += $$(STATIC_LIB_CFLAGS)
- endif
-
- # If no C++ flags are explicitly set, default to using the C flags.
- # After that, we can set additional C++ flags that should not interfere
- # with the mechanism for copying the C flags by default.
- ifeq ($$($1_CXXFLAGS), )
- $1_CXXFLAGS := $$($1_CFLAGS)
- endif
- ifeq ($$(strip $$($1_EXTRA_CXXFLAGS)), )
- $1_EXTRA_CXXFLAGS := $$($1_EXTRA_CFLAGS)
- endif
-
- ifeq ($$($1_COMPILE_WITH_DEBUG_SYMBOLS), true)
- $1_EXTRA_CFLAGS += $$(CFLAGS_DEBUG_SYMBOLS)
- $1_EXTRA_CXXFLAGS += $$(CFLAGS_DEBUG_SYMBOLS)
- $1_EXTRA_ASFLAGS += $$(ASFLAGS_DEBUG_SYMBOLS)
- endif
-
- # Pass the library name for static JNI library naming
- ifeq ($$($1_TYPE), STATIC_LIBRARY)
- $1_EXTRA_CFLAGS += -DLIBRARY_NAME=$$($1_NAME)
- $1_EXTRA_CXXFLAGS += -DLIBRARY_NAME=$$($1_NAME)
- endif
-
- # Pick up disabled warnings, if possible on this platform.
- ifneq ($(DISABLE_WARNING_PREFIX), )
- $1_EXTRA_CFLAGS += $$(addprefix $(DISABLE_WARNING_PREFIX), \
- $$(DISABLED_WARNINGS) \
- $$(DISABLED_WARNINGS_C) \
- $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)) \
- $$($1_DISABLED_WARNINGS_C_$(TOOLCHAIN_TYPE)) \
- $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)) \
- $$($1_DISABLED_WARNINGS_C_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)))
- $1_EXTRA_CXXFLAGS += $$(addprefix $(DISABLE_WARNING_PREFIX), \
- $$(DISABLED_WARNINGS) \
- $$(DISABLED_WARNINGS_CXX) \
- $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)) \
- $$($1_DISABLED_WARNINGS_CXX_$(TOOLCHAIN_TYPE)) \
- $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)) \
- $$($1_DISABLED_WARNINGS_CXX_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)))
- endif
-
- # Check if warnings should be considered errors.
- # Pick first binary and toolchain specific, then binary specific, then general setting.
- ifeq ($$($1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE)), )
- ifeq ($$($1_WARNINGS_AS_ERRORS), )
- $1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE) := $$(WARNINGS_AS_ERRORS)
+ $$(call SetIfEmpty, $1_CC, $(CC))
+ $$(call SetIfEmpty, $1_CXX, $(CXX))
+ $$(call SetIfEmpty, $1_AR, $(AR))
+ $$(call SetIfEmpty, $1_LIB, $(LIB))
+ $$(call SetIfEmpty, $1_AS, $(AS))
+ $$(call SetIfEmpty, $1_MT, $(MT))
+ $$(call SetIfEmpty, $1_RC, $(RC))
+ $$(call SetIfEmpty, $1_OBJCOPY, $(OBJCOPY))
+ $$(call SetIfEmpty, $1_STRIP, $(STRIP))
+ $$(call SetIfEmpty, $1_SYSROOT_CFLAGS, $(SYSROOT_CFLAGS))
+ $$(call SetIfEmpty, $1_SYSROOT_LDFLAGS, $(SYSROOT_LDFLAGS))
+ ifeq ($$($1_LINK_TYPE), C++)
+ $$(call SetIfEmpty, $1_LD, $(LDCXX))
else
- $1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE) := $$($1_WARNINGS_AS_ERRORS)
+ $$(call SetIfEmpty, $1_LD, $(LD))
endif
endif
+endef
- ifeq ($$($1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE)), true)
- $1_EXTRA_CFLAGS += $(CFLAGS_WARNINGS_ARE_ERRORS)
- $1_EXTRA_CXXFLAGS += $(CFLAGS_WARNINGS_ARE_ERRORS)
- endif
-
- ifeq (NONE, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_NONE)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NONE)
- else ifeq (LOW, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_NORM)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NORM)
- else ifeq (HIGH, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_HI)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HI)
- else ifeq (HIGHEST, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST)
- else ifeq (HIGHEST_JVM, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST_JVM)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST_JVM)
- else ifeq (SIZE, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_SIZE)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_SIZE)
- else ifneq (, $$($1_OPTIMIZATION))
- $$(error Unknown value for OPTIMIZATION: $$($1_OPTIMIZATION))
- endif
-
- $1_BUILD_INFO := $$($1_OBJECT_DIR)/_build-info.marker
-
+################################################################################
+# Setup machinery needed by the build system
+define SetupBuildSystemSupport
# Track variable changes for all variables that affect the compilation command
# lines for all object files in this setup. This includes at least all the
# variables used in the call to add_native_source below.
@@ -879,87 +348,19 @@ define SetupNativeCompilationBody
$$($1_CC) $$($1_CXX) $$($1_AS) $$($1_ASFLAGS)
$1_COMPILE_VARDEPS_FILE := $$(call DependOnVariable, $1_COMPILE_VARDEPS, \
$$($1_OBJECT_DIR)/$$($1_NOSUFFIX).comp.vardeps)
+endef
- ifneq ($$($1_PRECOMPILED_HEADER), )
- ifeq ($(USE_PRECOMPILED_HEADER), true)
- ifeq ($(TOOLCHAIN_TYPE), microsoft)
- $1_PCH_FILE := $$($1_OBJECT_DIR)/$1.pch
- $1_GENERATED_PCH_SRC := $$($1_OBJECT_DIR)/$1_pch.cpp
- $1_GENERATED_PCH_OBJ := $$($1_OBJECT_DIR)/$1_pch$(OBJ_SUFFIX)
-
- $$(eval $$(call SetupCompileNativeFile, $1_$$(notdir $$($1_GENERATED_PCH_SRC)), \
- FILE := $$($1_GENERATED_PCH_SRC), \
- BASE := $1, \
- EXTRA_CXXFLAGS := -Fp$$($1_PCH_FILE) -Yc$$(notdir $$($1_PRECOMPILED_HEADER)), \
- ))
-
- $1_USE_PCH_FLAGS := \
- -Fp$$($1_PCH_FILE) -Yu$$(notdir $$($1_PRECOMPILED_HEADER))
-
- $$($1_ALL_OBJS): $$($1_GENERATED_PCH_OBJ)
-
- # Explicitly add the pch obj file first to ease comparing to old
- # hotspot build.
- $1_ALL_OBJS := $$($1_GENERATED_PCH_OBJ) $$($1_ALL_OBJS)
-
- $$($1_GENERATED_PCH_SRC):
- $(ECHO) "#include \"$$(notdir $$($1_PRECOMPILED_HEADER))\"" > $$@
-
- else ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
- ifeq ($(TOOLCHAIN_TYPE), gcc)
- $1_PCH_FILE := $$($1_OBJECT_DIR)/precompiled/$$(notdir $$($1_PRECOMPILED_HEADER)).gch
- $1_USE_PCH_FLAGS := -I$$($1_OBJECT_DIR)/precompiled
- else ifeq ($(TOOLCHAIN_TYPE), clang)
- $1_PCH_FILE := $$($1_OBJECT_DIR)/precompiled/$$(notdir $$($1_PRECOMPILED_HEADER)).pch
- $1_USE_PCH_FLAGS := -include-pch $$($1_PCH_FILE)
- endif
- $1_PCH_DEPS_FILE := $$($1_PCH_FILE).d
- $1_PCH_DEPS_TARGETS_FILE := $$($1_PCH_FILE).d.targets
-
- -include $$($1_PCH_DEPS_FILE)
- -include $$($1_PCH_DEPS_TARGETS_FILE)
-
- $1_PCH_COMMAND := $$($1_CC) $$($1_CFLAGS) $$($1_EXTRA_CFLAGS) $$($1_SYSROOT_CFLAGS) \
- $$($1_OPT_CFLAGS) -x c++-header -c $(GENDEPS_FLAGS) \
- $$(addsuffix .tmp, $$($1_PCH_DEPS_FILE))
-
- $$($1_PCH_FILE): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
- $$(call LogInfo, Generating precompiled header)
- $$(call MakeDir, $$(@D))
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_PCH_COMMAND) $$< -o $$@))
- $$(call fix-deps-file, $$($1_PCH_DEPS_FILE))
- $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_PCH_DEPS_FILE) \
- > $$($1_PCH_DEPS_TARGETS_FILE)
-
- $$($1_ALL_OBJS): $$($1_PCH_FILE)
-
- # Generate the corresponding compile_commands.json fragment.
- $1_PCH_FILE_JSON := $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
- $$(OUTPUTDIR)/,,$$($1_PCH_FILE))).json
- $1_ALL_OBJS_JSON += $$($1_PCH_FILE_JSON)
-
- $$($1_PCH_FILE_JSON): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
- $$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$<, \
- $$($1_PCH_COMMAND) $$< -o $$($1_PCH_FILE))
- endif
- endif
- endif
-
- # Now call SetupCompileNativeFile for each source file we are going to compile.
- $$(foreach file, $$($1_SRCS), \
- $$(eval $$(call SetupCompileNativeFile, $1_$$(notdir $$(file)),\
- FILE := $$(file), \
- BASE := $1, \
- )) \
- )
-
+################################################################################
+# Have make print information about the library when we start compiling
+define PrintStartInfo
# Setup rule for printing progress info when compiling source files.
# This is a rough heuristic and may not always print accurate information.
# The $1_BUILD_INFO and $1_BUILD_INFO_DEPS variables are used in
# TestFilesCompilation.gmk.
$$(call SetIfEmpty, $1_BUILD_INFO_LOG_MACRO, LogWarn)
$1_BUILD_INFO_DEPS := $$($1_SRCS) $$($1_COMPILE_VARDEPS_FILE)
+ $1_BUILD_INFO := $$($1_OBJECT_DIR)/_build-info.marker
+
$$($1_BUILD_INFO): $$($1_BUILD_INFO_DEPS)
ifeq ($$(wildcard $$($1_TARGET)), )
$$(call $$($1_BUILD_INFO_LOG_MACRO), \
@@ -973,47 +374,12 @@ define SetupNativeCompilationBody
$$(if $$(filter %.vardeps, $$?), due to makefile changes))))
endif
$(TOUCH) $$@
+endef
- # On windows we need to create a resource file
- ifeq ($(call isTargetOs, windows), true)
- ifneq ($$($1_VERSIONINFO_RESOURCE), )
- $1_RES := $$($1_OBJECT_DIR)/$$($1_BASENAME).res
- $1_RES_DEPS_FILE := $$($1_RES).d
- $1_RES_DEPS_TARGETS_FILE := $$($1_RES).d.targets
- -include $$($1_RES_DEPS_FILE)
- -include $$($1_RES_DEPS_TARGETS_FILE)
-
- $1_RES_VARDEPS := $$($1_RC) $$($1_RCFLAGS)
- $1_RES_VARDEPS_FILE := $$(call DependOnVariable, $1_RES_VARDEPS, \
- $$($1_RES).vardeps)
-
- $$($1_RES): $$($1_VERSIONINFO_RESOURCE) $$($1_RES_VARDEPS_FILE)
- $$(call LogInfo, Compiling resource $$(notdir $$($1_VERSIONINFO_RESOURCE)) (for $$($1_BASENAME)))
- $$(call MakeDir, $$(@D) $$($1_OBJECT_DIR))
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_RC) $$($1_RCFLAGS) $$($1_SYSROOT_CFLAGS) $(CC_OUT_OPTION)$$@ \
- $$($1_VERSIONINFO_RESOURCE) 2>&1 ))
- # Windows RC compiler does not support -showIncludes, so we mis-use CL
- # for this. Filter out RC specific arguments that are unknown to CL.
- # For some unknown reason, in this case CL actually outputs the show
- # includes to stderr so need to redirect it to hide the output from the
- # main log.
- $$(call ExecuteWithLog, $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX), \
- $$($1_CC) $$(filter-out -l%, $$($1_RCFLAGS)) \
- $$($1_SYSROOT_CFLAGS) -showIncludes -nologo -TC \
- $(CC_OUT_OPTION)$$($1_RES_DEPS_FILE)$(OBJ_SUFFIX) -P -Fi$$($1_RES_DEPS_FILE).pp \
- $$($1_VERSIONINFO_RESOURCE)) 2>&1 \
- | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
- -e "^$$(notdir $$($1_VERSIONINFO_RESOURCE))$$$$" || test "$$$$?" = "1" ; \
- $(ECHO) $$($1_RES): \\ > $$($1_RES_DEPS_FILE) ; \
- $(SED) $(WINDOWS_SHOWINCLUDE_SED_PATTERN) $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX).log \
- >> $$($1_RES_DEPS_FILE) ; \
- $(ECHO) >> $$($1_RES_DEPS_FILE) ;\
- $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_RES_DEPS_FILE) \
- > $$($1_RES_DEPS_TARGETS_FILE)
- endif
- endif
-
+################################################################################
+# Setup a library-wide dependency file from individual object file dependency
+# files
+define CreateDependencyFile
# Create a rule to collect all the individual make dependency files into a
# single makefile.
$1_DEPS_FILE := $$($1_OBJECT_DIR)/$1.d
@@ -1031,7 +397,11 @@ define SetupNativeCompilationBody
$(MV) $$@.tmp $$@
$1 += $$($1_DEPS_FILE)
+endef
+################################################################################
+# Import the dependency file into the makefile
+define ImportDependencyFile
# The include must be on the .old file, which represents the state from the
# previous invocation of make. The file being included must not have a rule
# defined for it as otherwise make will think it has to run the rule before
@@ -1041,328 +411,6 @@ define SetupNativeCompilationBody
$1_DEPS_FILE_LOADED := true
-include $$($1_DEPS_FILE).old
endif
-
- ifneq ($(DISABLE_MAPFILES), true)
- $1_REAL_MAPFILE := $$($1_MAPFILE)
- endif
-
- # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS and TOOLCHAIN_TYPE
- # dependent variables for LDFLAGS and LIBS, and additionally the pair dependent
- # TOOLCHAIN_TYPE plus OPENJDK_TARGET_OS
- $1_EXTRA_LDFLAGS += $$($1_LDFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_LDFLAGS_$(OPENJDK_TARGET_OS)) \
- $$($1_LDFLAGS_$(TOOLCHAIN_TYPE)) $$($1_LDFLAGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS))
- $1_EXTRA_LIBS += $$($1_LIBS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_LIBS_$(OPENJDK_TARGET_OS)) \
- $$($1_LIBS_$(TOOLCHAIN_TYPE)) $$($1_LIBS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS))
-
- ifneq ($$($1_REAL_MAPFILE), )
- $1_EXTRA_LDFLAGS += $(call SET_SHARED_LIBRARY_MAPFILE,$$($1_REAL_MAPFILE))
- endif
-
- ifneq ($$($1_COPY_DEBUG_SYMBOLS), false)
- $1_COPY_DEBUG_SYMBOLS := $(COPY_DEBUG_SYMBOLS)
- endif
-
- ifneq ($$($1_ZIP_EXTERNAL_DEBUG_SYMBOLS), false)
- $1_ZIP_EXTERNAL_DEBUG_SYMBOLS := $(ZIP_EXTERNAL_DEBUG_SYMBOLS)
- endif
-
- ifeq ($$($1_COPY_DEBUG_SYMBOLS), true)
- ifneq ($$($1_DEBUG_SYMBOLS), false)
- $$(call SetIfEmpty, $1_SYMBOLS_DIR, $$($1_OUTPUT_DIR))
- # Only copy debug symbols for dynamic libraries and programs.
- ifneq ($$($1_TYPE), STATIC_LIBRARY)
- # Generate debuginfo files.
- ifeq ($(call isTargetOs, windows), true)
- $1_EXTRA_LDFLAGS += -debug "-pdb:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).pdb" \
- "-map:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).map"
- ifeq ($(SHIP_DEBUG_SYMBOLS), public)
- $1_EXTRA_LDFLAGS += "-pdbstripped:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).stripped.pdb"
- endif
- $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_BASENAME).pdb \
- $$($1_SYMBOLS_DIR)/$$($1_BASENAME).map
-
- else ifeq ($(call isTargetOs, linux), true)
- $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).debuginfo
- # Setup the command line creating debuginfo files, to be run after linking.
- # It cannot be run separately since it updates the original target file
- # Creating the debuglink is done in another command rather than all at once
- # so we can run it after strip is called, since strip can sometimes mangle the
- # embedded debuglink, which we want to avoid.
- $1_CREATE_DEBUGINFO_CMDS := \
- $$($1_OBJCOPY) --only-keep-debug $$($1_TARGET) $$($1_DEBUGINFO_FILES) $$(NEWLINE)
- $1_CREATE_DEBUGLINK_CMDS := $(CD) $$($1_SYMBOLS_DIR) && \
- $$($1_OBJCOPY) --add-gnu-debuglink=$$($1_DEBUGINFO_FILES) $$($1_TARGET)
-
- else ifeq ($(call isTargetOs, aix), true)
- # AIX does not provide the equivalent of OBJCOPY to extract debug symbols,
- # so we copy the compiled object with symbols to the .debuginfo file, which
- # happens prior to the STRIP_CMD on the original target object file.
- $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).debuginfo
- $1_CREATE_DEBUGINFO_CMDS := $(CP) $$($1_TARGET) $$($1_DEBUGINFO_FILES)
-
- else ifeq ($(call isTargetOs, macosx), true)
- $1_DEBUGINFO_FILES := \
- $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM/Contents/Info.plist \
- $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM/Contents/Resources/DWARF/$$($1_BASENAME)
- $1_CREATE_DEBUGINFO_CMDS := \
- $(DSYMUTIL) --out $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM $$($1_TARGET)
- endif
-
- # Since the link rule creates more than one file that we want to track,
- # we have to use some tricks to get make to cooperate. To properly
- # trigger downstream dependants of $$($1_DEBUGINFO_FILES), we must have
- # a recipe in the rule below. To avoid rerunning the recipe every time
- # have it touch the target. If a debuginfo file is deleted by something
- # external, explicitly delete the TARGET to trigger a rebuild of both.
- ifneq ($$(wildcard $$($1_DEBUGINFO_FILES)), $$($1_DEBUGINFO_FILES))
- $$(call LogDebug, Deleting $$($1_BASENAME) because debuginfo files are missing)
- $$(shell $(RM) $$($1_TARGET))
- endif
- $$($1_DEBUGINFO_FILES): $$($1_TARGET)
- $$(if $$(CORRECT_FUNCTION_IN_RECIPE_EVALUATION), \
- $$(if $$(wildcard $$@), , $$(error $$@ was not created for $$<)) \
- )
- $(TOUCH) $$@
-
- $1 += $$($1_DEBUGINFO_FILES)
-
- ifeq ($$($1_ZIP_EXTERNAL_DEBUG_SYMBOLS), true)
- ifeq ($(call isTargetOs, windows), true)
- $1_DEBUGINFO_ZIP := $$($1_SYMBOLS_DIR)/$$($1_BASENAME).diz
- else
- $1_DEBUGINFO_ZIP := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).diz
- endif
- $1 += $$($1_DEBUGINFO_ZIP)
-
- # The dependency on TARGET is needed for debuginfo files
- # to be rebuilt properly.
- $$($1_DEBUGINFO_ZIP): $$($1_DEBUGINFO_FILES) $$($1_TARGET)
- $(CD) $$($1_SYMBOLS_DIR) && \
- $(ZIPEXE) -q -r $$@ $$(subst $$($1_SYMBOLS_DIR)/,, $$($1_DEBUGINFO_FILES))
-
- endif
- endif # !STATIC_LIBRARY
- endif # $1_DEBUG_SYMBOLS != false
- endif # COPY_DEBUG_SYMBOLS
-
- # Unless specifically set, stripping should only happen if symbols are also
- # being copied.
- $$(call SetIfEmpty, $1_STRIP_SYMBOLS, $$($1_COPY_DEBUG_SYMBOLS))
-
- ifneq ($$($1_STRIP_SYMBOLS), false)
- ifneq ($$($1_STRIP), )
- # Default to using the global STRIPFLAGS. Allow for overriding with an empty value
- $1_STRIPFLAGS ?= $(STRIPFLAGS)
- $1_STRIP_CMD := $$($1_STRIP) $$($1_STRIPFLAGS) $$($1_TARGET)
- endif
- endif
-
- $1_LD_OBJ_ARG := $$($1_ALL_OBJS)
-
- # If there are many object files, use an @-file...
- ifneq ($$(word 17, $$($1_ALL_OBJS)), )
- $1_OBJ_FILE_LIST := $$($1_OBJECT_DIR)/_$1_objectfilenames.txt
- ifneq ($(COMPILER_COMMAND_FILE_FLAG), )
- $1_LD_OBJ_ARG := $(COMPILER_COMMAND_FILE_FLAG)$$($1_OBJ_FILE_LIST)
- else
- # ...except for toolchains which don't support them.
- $1_LD_OBJ_ARG := `cat $$($1_OBJ_FILE_LIST)`
- endif
-
- # If we are building static library, 'AR' on macosx/aix may not support @-file.
- ifeq ($$($1_TYPE), STATIC_LIBRARY)
- ifeq ($(call isTargetOs, macosx aix), true)
- $1_LD_OBJ_ARG := `cat $$($1_OBJ_FILE_LIST)`
- endif
- endif
- endif
-
- # Unfortunately the @-file trick does not work reliably when using clang.
- # Clang does not propagate the @-file parameter to the ld sub process, but
- # instead puts the full content on the command line. At least the llvm ld
- # does not even support an @-file.
- #
- # When linking a large amount of object files, we risk hitting the limit
- # of the command line length even on posix systems if the path length of
- # the output dir is very long due to our use of absolute paths. To
- # mitigate this, use paths relative to the output dir when linking over
- # 500 files with clang and the output dir path is deep.
- ifneq ($$(word 500, $$($1_ALL_OBJS)), )
- ifeq ($$(TOOLCHAIN_TYPE), clang)
- # There is no strlen function in make, but checking path depth is a
- # reasonable approximation.
- ifneq ($$(word 10, $$(subst /, ,$$(OUTPUTDIR))), )
- $1_LINK_OBJS_RELATIVE := true
- $1_ALL_OBJS_RELATIVE := $$(patsubst $$(OUTPUTDIR)/%, %, $$($1_ALL_OBJS))
- endif
- endif
- endif
-
- ifeq ($$($1_TYPE), STATIC_LIBRARY)
- # Include partial linking when building the static library with clang on linux.
- ifeq ($(call isTargetOs, linux), true)
- ifneq ($(findstring $(TOOLCHAIN_TYPE), clang), )
- $1_ENABLE_PARTIAL_LINKING := true
- endif
- endif
-
- $1_VARDEPS := $$($1_AR) $$(ARFLAGS) $$($1_ARFLAGS) $$($1_LIBS) \
- $$($1_EXTRA_LIBS)
- ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
- $1_VARDEPS += $$($1_LD) $$($1_SYSROOT_LDFLAGS)
- endif
- $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
- $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
-
- # Generating a static library, ie object file archive.
- ifeq ($(STATIC_BUILD), true)
- ifeq ($$($1_USE_MAPFILE_FOR_SYMBOLS), true)
- STATIC_MAPFILE_DEP := $$($1_MAPFILE)
- endif
- endif
-
- $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_RES) $$($1_VARDEPS_FILE) $$(STATIC_MAPFILE_DEP)
-
- $1_AR_OBJ_ARG := $$($1_LD_OBJ_ARG)
- # With clang on linux, partial linking is enabled and 'AR' takes the output
- # object from the partial linking step.
- ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
- $1_TARGET_RELOCATABLE := $$($1_OBJECT_DIR)/$$($1_PREFIX)$$($1_NAME)_relocatable$(OBJ_SUFFIX)
- $1_AR_OBJ_ARG := $$($1_TARGET_RELOCATABLE)
- endif
-
- $$($1_TARGET): $$($1_TARGET_DEPS)
- ifneq ($$($1_OBJ_FILE_LIST), )
- ifeq ($$($1_LINK_OBJS_RELATIVE), true)
- $$(eval $$(call ListPathsSafely, $1_ALL_OBJS_RELATIVE, $$($1_OBJ_FILE_LIST)))
- else
- $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
- endif
- endif
- $$(call LogInfo, Building static library $$($1_BASENAME))
- $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
- # Do partial linking.
- ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
- $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_partial_link, \
- $(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
- $$($1_LD) $(LDFLAGS_CXX_PARTIAL_LINKING) $$($1_SYSROOT_LDFLAGS) \
- $(LD_OUT_OPTION)$$($1_TARGET_RELOCATABLE) \
- $$($1_LD_OBJ_ARG))
- endif
- $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
- $(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
- $$($1_AR) $$(ARFLAGS) $$($1_ARFLAGS) $(AR_OUT_OPTION)$$($1_TARGET) $$($1_AR_OBJ_ARG) \
- $$($1_RES))
- ifeq ($(STATIC_BUILD), true)
- ifeq ($$($1_USE_MAPFILE_FOR_SYMBOLS), true)
- $(CP) $$($1_MAPFILE) $$(@D)/$$(basename $$(@F)).symbols
- else
- $(GetSymbols)
- endif
- endif
- else
- # A shared dynamic library or an executable binary has been specified
- ifeq ($$($1_TYPE), LIBRARY)
- # Generating a dynamic library.
- $1_EXTRA_LDFLAGS += $$(call SET_SHARED_LIBRARY_NAME,$$($1_BASENAME))
-
- # Create loadmap on AIX. Helps in diagnosing some problems.
- ifneq ($(COMPILER_BINDCMD_FILE_FLAG), )
- $1_EXTRA_LDFLAGS += $(COMPILER_BINDCMD_FILE_FLAG)$$($1_OBJECT_DIR)/$$($1_NOSUFFIX).loadmap
- endif
- endif
-
- ifeq ($(call isTargetOs, windows), true)
- ifeq ($$($1_EMBED_MANIFEST), true)
- $1_EXTRA_LDFLAGS += -manifest:embed
- endif
-
- $1_IMPORT_LIBRARY := $$($1_OBJECT_DIR)/$$($1_NAME).lib
- $1_EXTRA_LDFLAGS += "-implib:$$($1_IMPORT_LIBRARY)"
- ifeq ($$($1_TYPE), LIBRARY)
- # To properly trigger downstream dependants of the import library, just as
- # for debug files, we must have a recipe in the rule. To avoid rerunning
- # the recipe every time have it touch the target. If an import library
- # file is deleted by something external, explicitly delete the target to
- # trigger a rebuild of both.
- ifneq ($$(wildcard $$($1_IMPORT_LIBRARY)), $$($1_IMPORT_LIBRARY))
- $$(call LogDebug, Deleting $$($1_BASENAME) because import library is missing)
- $$(shell $(RM) $$($1_TARGET))
- endif
- $$($1_IMPORT_LIBRARY): $$($1_TARGET)
- $(TOUCH) $$@
-
- $1 += $$($1_IMPORT_LIBRARY)
- endif
- endif
-
- $1_VARDEPS := $$($1_LD) $$($1_SYSROOT_LDFLAGS) $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) \
- $$($1_LIBS) $$($1_EXTRA_LIBS) $$($1_MT) \
- $$($1_CREATE_DEBUGINFO_CMDS) $$($1_MANIFEST_VERSION) \
- $$($1_STRIP_CMD) $$($1_CREATE_DEBUGLINK_CMDS)
- $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
- $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
-
- $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_RES) $$($1_MANIFEST) \
- $$($1_REAL_MAPFILE) $$($1_VARDEPS_FILE)
-
- $$($1_TARGET): $$($1_TARGET_DEPS)
- ifneq ($$($1_OBJ_FILE_LIST), )
- ifeq ($$($1_LINK_OBJS_RELATIVE), true)
- $$(eval $$(call ListPathsSafely, $1_ALL_OBJS_RELATIVE, $$($1_OBJ_FILE_LIST)))
- else
- $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
- endif
- endif
- # Keep as much as possible on one execution line for best performance
- # on Windows
- $$(call LogInfo, Linking $$($1_BASENAME))
- $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
- ifeq ($(call isTargetOs, windows), true)
-
- $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
- $$($1_LD) $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) $$($1_SYSROOT_LDFLAGS) \
- $(LD_OUT_OPTION)$$($1_TARGET) $$($1_LD_OBJ_ARG) $$($1_RES) \
- $$($1_LIBS) $$($1_EXTRA_LIBS)) \
- | $(GREP) -v "^ Creating library .*\.lib and object .*\.exp" || \
- test "$$$$?" = "1" ; \
- $$($1_CREATE_DEBUGINFO_CMDS)
- $$($1_STRIP_CMD)
- $$($1_CREATE_DEBUGLINK_CMDS)
- ifeq ($(call isBuildOsEnv, windows.wsl2), true)
- $$(CHMOD) +x $$($1_TARGET)
- endif
- else
- $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
- $$(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
- $$($1_LD) $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) $$($1_SYSROOT_LDFLAGS) \
- $(LD_OUT_OPTION)$$($1_TARGET) $$($1_LD_OBJ_ARG) $$($1_RES) \
- $$($1_LIBS) $$($1_EXTRA_LIBS)) ; \
- $$($1_CREATE_DEBUGINFO_CMDS)
- $$($1_STRIP_CMD)
- $$($1_CREATE_DEBUGLINK_CMDS)
- endif
- ifeq ($(call isTargetOs, windows), true)
- ifneq ($$($1_MANIFEST), )
- $$($1_MT) -nologo -manifest $$($1_MANIFEST) -identity:"$$($1_NAME).exe, version=$$($1_MANIFEST_VERSION)" -outputresource:$$@;#1
- endif
- endif
- # On macosx, optionally run codesign on every binary.
- # Remove signature explicitly first to avoid warnings if the linker
- # added a default adhoc signature.
- ifeq ($(MACOSX_CODESIGN_MODE), hardened)
- $(CODESIGN) --remove-signature $$@
- $(CODESIGN) -f -s "$(MACOSX_CODESIGN_IDENTITY)" --timestamp --options runtime \
- --entitlements $$(call GetEntitlementsFile, $$@) $$@
- else ifeq ($(MACOSX_CODESIGN_MODE), debug)
- $(CODESIGN) --remove-signature $$@
- $(CODESIGN) -f -s - --entitlements $$(call GetEntitlementsFile, $$@) $$@
- endif
- endif
-
- ifeq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
- $1 := $$($1_ALL_OBJS_JSON)
- endif
endef
endif # _NATIVE_COMPILATION_GMK
diff --git a/make/common/TestFilesCompilation.gmk b/make/common/TestFilesCompilation.gmk
index d97d0e6c697e7..626eb058f0a1b 100644
--- a/make/common/TestFilesCompilation.gmk
+++ b/make/common/TestFilesCompilation.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -103,6 +103,7 @@ define SetupTestFilesCompilationBody
$$(eval $$(call SetupNativeCompilation, BUILD_TEST_$$(name), \
NAME := $$(unprefixed_name), \
TYPE := $$($1_COMPILATION_TYPE), \
+ LINK_TYPE := $(if $$(filter %.cpp, $$(file)), C++, C), \
EXTRA_FILES := $$(file) $$($1_EXTRA_FILES), \
OBJECT_DIR := $$($1_OUTPUT_DIR)/support/$$(name), \
OUTPUT_DIR := $$($1_OUTPUT_DIR)/$$($1_OUTPUT_SUBDIR), \
@@ -113,7 +114,6 @@ define SetupTestFilesCompilationBody
DISABLED_WARNINGS_clang := undef format-nonliteral \
missing-field-initializers sometimes-uninitialized, \
LIBS := $$($1_LIBS_$$(name)), \
- TOOLCHAIN := $(if $$(filter %.cpp, $$(file)), TOOLCHAIN_LINK_CXX, TOOLCHAIN_DEFAULT), \
OPTIMIZATION := $$(if $$($1_OPTIMIZATION_$$(name)),$$($1_OPTIMIZATION_$$(name)),LOW), \
COPY_DEBUG_SYMBOLS := $$($1_COPY_DEBUG_SYMBOLS), \
STRIP_SYMBOLS := $$(if $$($1_STRIP_SYMBOLS_$$(name)),$$($1_STRIP_SYMBOLS_$$(name)),false), \
diff --git a/make/common/native/CompileFile.gmk b/make/common/native/CompileFile.gmk
new file mode 100644
index 0000000000000..a9384fb0cf509
--- /dev/null
+++ b/make/common/native/CompileFile.gmk
@@ -0,0 +1,351 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to compiling a single native source
+# file (C, C++ or Objective-C) into an object file. It also harbours related
+# functionality for generating PCH (precompiled headers) and Windows resource
+# files.
+
+################################################################################
+# Creates a recipe that creates a compile_commands.json fragment. Remove any
+# occurrences of FIXPATH programs from the command to show the actual invocation.
+#
+# Param 1: Name of file to create
+# Param 2: Working directory
+# Param 3: Source file
+# Param 4: Compile command
+################################################################################
+define WriteCompileCommandsFragment
+ $(call LogInfo, Creating compile commands fragment for $(notdir $3))
+ $(call MakeDir, $(dir $1))
+ $(call WriteFile,{ \
+ "directory": "$(strip $(call FixPath, $2))"$(COMMA) \
+ "file": "$(strip $(call FixPath, $3))"$(COMMA) \
+ "command": "$(strip $(subst $(DQUOTE),\$(DQUOTE),$(subst \,\\,\
+ $(subst $(FIXPATH),,$(call FixPath, $4)))))" \
+ }$(COMMA), \
+ $1)
+endef
+
+################################################################################
+# Extensions of files handled by this macro.
+NATIVE_SOURCE_EXTENSIONS := %.S %.c %.cpp %.cc %.m %.mm
+
+# Replaces native source extensions with the object file extension in a string.
+# Param 1: the string containing source file names with extensions
+# The surrounding strip is needed to keep additional whitespace out
+define replace_with_obj_extension
+$(strip \
+ $(foreach extension, $(NATIVE_SOURCE_EXTENSIONS), \
+ $(patsubst $(extension),%$(OBJ_SUFFIX), $(filter $(extension), $1))) \
+)
+endef
+
+################################################################################
+# This pattern is used to transform the output of the microsoft CL compiler
+# into a make syntax dependency file (.d)
+WINDOWS_SHOWINCLUDE_SED_PATTERN := \
+ -e '/^Note: including file:/!d' \
+ -e 's|Note: including file: *||' \
+ -e 's|\r||g' \
+ -e 's|\\|/|g' \
+ -e 's|^\([a-zA-Z]\):|$(WINENV_PREFIX)/\1|g' \
+ -e '\|$(TOPDIR)|I !d' \
+ -e 's|$$$$| \\|g' \
+ #
+
+################################################################################
+# This pattern is used to transform a dependency file (.d) to a list
+# of make targets for dependent files (.d.targets)
+DEPENDENCY_TARGET_SED_PATTERN := \
+ -e 's/\#.*//' \
+ -e 's/^[^:]*: *//' \
+ -e 's/ *\\$$$$//' \
+ -e 's/^[ ]*//' \
+ -e '/^$$$$/ d' \
+ -e 's/$$$$/ :/' \
+ #
+
+################################################################################
+# Create the recipe needed to compile a single native source file.
+#
+# Parameter 1 is the name of the rule, based on the name of the library/
+# program being build and the name of the source code file, e.g.
+# BUILD_LIBFOO_fooMain.cpp.
+#
+# Remaining parameters are named arguments:
+# FILE - The full path of the source file to compiler
+# BASE - The name of the rule for the entire binary to build ($1)
+#
+CreateCompiledNativeFile = $(NamedParamsMacroTemplate)
+define CreateCompiledNativeFileBody
+ $1_FILENAME := $$(notdir $$($1_FILE))
+
+ # The target file to be generated.
+ $1_OBJ := $$($$($1_BASE)_OBJECT_DIR)/$$(call replace_with_obj_extension, \
+ $$($1_FILENAME))
+
+ # Generate the corresponding compile_commands.json fragment.
+ $1_OBJ_JSON = $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
+ $$(OUTPUTDIR)/,,$$($1_OBJ))).json
+ $$($1_BASE)_ALL_OBJS_JSON += $$($1_OBJ_JSON)
+
+ # Only continue if this object file hasn't been processed already. This lets
+ # the first found source file override any other with the same name.
+ ifeq ($$($1_OBJ_PROCESSED), )
+ $1_OBJ_PROCESSED := true
+ # This is the definite source file to use for $1_FILENAME.
+ $1_SRC_FILE := $$($1_FILE)
+
+ $$(eval $$(call SetupCompileFileFlags,$1,$$($1_BASE)))
+
+ ifneq ($$(filter %.c, $$($1_FILENAME)), )
+ # Compile as a C file
+ $1_CFLAGS += $$($1_WARNINGS_FLAGS)
+ $1_FLAGS := $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) $$($1_BASE_CFLAGS) \
+ $$($1_OPT_CFLAGS) $$($1_CFLAGS) -c
+ $1_COMPILER := $$($$($1_BASE)_CC)
+ else ifneq ($$(filter %.m, $$($1_FILENAME)), )
+ # Compile as an Objective-C file
+ $1_CFLAGS += $$($1_WARNINGS_FLAGS)
+ $1_FLAGS := -x objective-c $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) \
+ $$($1_BASE_CFLAGS) $$($1_OPT_CFLAGS) $$($1_CFLAGS) -c
+ $1_COMPILER := $$($$($1_BASE)_CC)
+ else ifneq ($$(filter %.S, $$($1_FILENAME)), )
+ # Compile as preprocessed assembler file
+ $1_FLAGS := $(BASIC_ASFLAGS) $$($1_BASE_ASFLAGS)
+ $1_COMPILER := $(AS)
+
+ # gcc or clang assembly files must contain an appropriate relative .file
+ # path for reproducible builds.
+ ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
+ # If no absolute paths allowed, work out relative source file path
+ # for assembly .file substitution, otherwise use full file path
+ ifeq ($(ALLOW_ABSOLUTE_PATHS_IN_OUTPUT), false)
+ $1_REL_ASM_SRC := $$(call RelativePath, $$($1_FILE), $(WORKSPACE_ROOT))
+ else
+ $1_REL_ASM_SRC := $$($1_FILE)
+ endif
+ $1_FLAGS := $$($1_FLAGS) -DASSEMBLY_SRC_FILE='"$$($1_REL_ASM_SRC)"' \
+ -include $(TOPDIR)/make/data/autoheaders/assemblyprefix.h
+ endif
+ else ifneq ($$(filter %.cpp %.cc %.mm, $$($1_FILENAME)), )
+ # Compile as a C++ or Objective-C++ file
+ $1_CXXFLAGS += $$($1_WARNINGS_FLAGS)
+ $1_FLAGS := $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) $$($1_BASE_CXXFLAGS) \
+ $$($1_OPT_CXXFLAGS) $$($1_CXXFLAGS) -c
+ $1_COMPILER := $$($$($1_BASE)_CXX)
+ else
+ $$(error Internal error in NativeCompilation.gmk: no compiler for file $$($1_FILENAME))
+ endif
+
+ # And this is the dependency file for this obj file.
+ $1_DEPS_FILE := $$(patsubst %$(OBJ_SUFFIX),%.d,$$($1_OBJ))
+ # The dependency target file lists all dependencies as empty targets to
+ # avoid make error "No rule to make target" for removed files
+ $1_DEPS_TARGETS_FILE := $$(patsubst %$(OBJ_SUFFIX),%.d.targets,$$($1_OBJ))
+
+ # Only try to load individual dependency information files if the global
+ # file hasn't been loaded (could happen if make was interrupted).
+ ifneq ($$($$($1_BASE)_DEPS_FILE_LOADED), true)
+ # Include previously generated dependency information. (if it exists)
+ -include $$($1_DEPS_FILE)
+ -include $$($1_DEPS_TARGETS_FILE)
+ endif
+
+ ifneq ($$(strip $$($1_CFLAGS) $$($1_CXXFLAGS) $$($1_OPTIMIZATION)), )
+ $1_VARDEPS := $$($1_CFLAGS) $$($1_CXXFLAGS) $$($1_OPTIMIZATION)
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, $$($1_OBJ).vardeps)
+ endif
+
+ $1_OBJ_DEPS := $$($1_SRC_FILE) $$($$($1_BASE)_COMPILE_VARDEPS_FILE) \
+ $$($$($1_BASE)_EXTRA_DEPS) $$($1_VARDEPS_FILE)
+ $1_COMPILE_OPTIONS := $$($1_FLAGS) $(CC_OUT_OPTION)$$($1_OBJ) $$($1_SRC_FILE)
+ # For reproducible builds with gcc and clang ensure random symbol generation is
+ # seeded deterministically
+ ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
+ $1_COMPILE_OPTIONS += -frandom-seed="$$($1_FILENAME)"
+ endif
+
+ $$($1_OBJ_JSON): $$($1_OBJ_DEPS)
+ $$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$($1_SRC_FILE), \
+ $$($1_COMPILER) $$($1_COMPILE_OPTIONS))
+
+ $$($1_OBJ): $$($1_OBJ_DEPS) | $$($$($1_BASE)_BUILD_INFO)
+ $$(call LogInfo, Compiling $$($1_FILENAME) (for $$($$($1_BASE)_BASENAME)))
+ $$(call MakeDir, $$(@D))
+ ifneq ($(TOOLCHAIN_TYPE), microsoft)
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_COMPILER) $$(GENDEPS_FLAGS) \
+ $$(addsuffix .tmp, $$($1_DEPS_FILE)) \
+ $$($1_COMPILE_OPTIONS)))
+ ifneq ($$($1_DEPS_FILE), )
+ $$(call fix-deps-file, $$($1_DEPS_FILE))
+ # Create a dependency target file from the dependency file.
+ # Solution suggested by:
+ # http://make.mad-scientist.net/papers/advanced-auto-dependency-generation/
+ $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_DEPS_FILE) \
+ > $$($1_DEPS_TARGETS_FILE)
+ endif
+ else
+ # The Visual Studio compiler lacks a feature for generating make
+ # dependencies, but by setting -showIncludes, all included files are
+ # printed. These are filtered out and parsed into make dependences.
+ #
+ # Keep as much as possible on one execution line for best performance
+ # on Windows. No need to save exit code from compilation since
+ # pipefail is always active on Windows.
+ ifeq ($$(filter %.S, $$($1_FILENAME)), )
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_COMPILER) -showIncludes $$($1_COMPILE_OPTIONS))) \
+ | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
+ -e "^$$($1_FILENAME)$$$$" || test "$$$$?" = "1" ; \
+ $(ECHO) $$@: \\ > $$($1_DEPS_FILE) ; \
+ $(SED) $(WINDOWS_SHOWINCLUDE_SED_PATTERN) $$($1_OBJ).log \
+ | $(SORT) -u >> $$($1_DEPS_FILE) ; \
+ $(ECHO) >> $$($1_DEPS_FILE) ; \
+ $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_DEPS_FILE) > $$($1_DEPS_TARGETS_FILE)
+ else
+ # For assembler calls just create empty dependency lists
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_COMPILER) $$($1_FLAGS) \
+ $(CC_OUT_OPTION)$$($1_OBJ) -Ta $$($1_SRC_FILE))) \
+ | $(TR) -d '\r' | $(GREP) -v -e "Assembling:" || test "$$$$?" = "1" ; \
+ $(ECHO) > $$($1_DEPS_FILE) ; \
+ $(ECHO) > $$($1_DEPS_TARGETS_FILE)
+ endif
+ endif
+ endif
+endef
+
+################################################################################
+define CreatePrecompiledHeader
+ ifneq ($$($1_PRECOMPILED_HEADER), )
+ ifeq ($(USE_PRECOMPILED_HEADER), true)
+ ifeq ($(TOOLCHAIN_TYPE), microsoft)
+ $1_PCH_FILE := $$($1_OBJECT_DIR)/$1.pch
+ $1_GENERATED_PCH_SRC := $$($1_OBJECT_DIR)/$1_pch.cpp
+ $1_GENERATED_PCH_OBJ := $$($1_OBJECT_DIR)/$1_pch$(OBJ_SUFFIX)
+
+ $$(eval $$(call CreateCompiledNativeFile, $1_$$(notdir $$($1_GENERATED_PCH_SRC)), \
+ FILE := $$($1_GENERATED_PCH_SRC), \
+ BASE := $1, \
+ EXTRA_CXXFLAGS := -Fp$$($1_PCH_FILE) -Yc$$(notdir $$($1_PRECOMPILED_HEADER)), \
+ ))
+
+ $1_USE_PCH_FLAGS := \
+ -Fp$$($1_PCH_FILE) -Yu$$(notdir $$($1_PRECOMPILED_HEADER))
+
+ $$($1_ALL_OBJS): $$($1_GENERATED_PCH_OBJ)
+
+ # Explicitly add the pch obj file first to ease comparing to old
+ # hotspot build.
+ $1_ALL_OBJS := $$($1_GENERATED_PCH_OBJ) $$($1_ALL_OBJS)
+
+ $$($1_GENERATED_PCH_SRC):
+ $(ECHO) "#include \"$$(notdir $$($1_PRECOMPILED_HEADER))\"" > $$@
+
+ else ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
+ ifeq ($(TOOLCHAIN_TYPE), gcc)
+ $1_PCH_FILE := $$($1_OBJECT_DIR)/precompiled/$$(notdir $$($1_PRECOMPILED_HEADER)).gch
+ $1_USE_PCH_FLAGS := -I$$($1_OBJECT_DIR)/precompiled
+ else ifeq ($(TOOLCHAIN_TYPE), clang)
+ $1_PCH_FILE := $$($1_OBJECT_DIR)/precompiled/$$(notdir $$($1_PRECOMPILED_HEADER)).pch
+ $1_USE_PCH_FLAGS := -include-pch $$($1_PCH_FILE)
+ endif
+ $1_PCH_DEPS_FILE := $$($1_PCH_FILE).d
+ $1_PCH_DEPS_TARGETS_FILE := $$($1_PCH_FILE).d.targets
+
+ -include $$($1_PCH_DEPS_FILE)
+ -include $$($1_PCH_DEPS_TARGETS_FILE)
+
+ $1_PCH_COMMAND := $$($1_CC) $$($1_CFLAGS) $$($1_EXTRA_CFLAGS) $$($1_SYSROOT_CFLAGS) \
+ $$($1_OPT_CFLAGS) -x c++-header -c $(GENDEPS_FLAGS) \
+ $$(addsuffix .tmp, $$($1_PCH_DEPS_FILE))
+
+ $$($1_PCH_FILE): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
+ $$(call LogInfo, Generating precompiled header)
+ $$(call MakeDir, $$(@D))
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_PCH_COMMAND) $$< -o $$@))
+ $$(call fix-deps-file, $$($1_PCH_DEPS_FILE))
+ $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_PCH_DEPS_FILE) \
+ > $$($1_PCH_DEPS_TARGETS_FILE)
+
+ $$($1_ALL_OBJS): $$($1_PCH_FILE)
+
+ # Generate the corresponding compile_commands.json fragment.
+ $1_PCH_FILE_JSON := $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
+ $$(OUTPUTDIR)/,,$$($1_PCH_FILE))).json
+ $1_ALL_OBJS_JSON += $$($1_PCH_FILE_JSON)
+
+ $$($1_PCH_FILE_JSON): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
+ $$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$<, \
+ $$($1_PCH_COMMAND) $$< -o $$($1_PCH_FILE))
+ endif
+ endif
+ endif
+endef
+
+################################################################################
+define CreateWindowsResourceFile
+ ifneq ($$($1_VERSIONINFO_RESOURCE), )
+ $1_RES := $$($1_OBJECT_DIR)/$$($1_BASENAME).res
+ $1_RES_DEPS_FILE := $$($1_RES).d
+ $1_RES_DEPS_TARGETS_FILE := $$($1_RES).d.targets
+ -include $$($1_RES_DEPS_FILE)
+ -include $$($1_RES_DEPS_TARGETS_FILE)
+
+ $1_RES_VARDEPS := $$($1_RC) $$($1_RCFLAGS)
+ $1_RES_VARDEPS_FILE := $$(call DependOnVariable, $1_RES_VARDEPS, \
+ $$($1_RES).vardeps)
+
+ $$($1_RES): $$($1_VERSIONINFO_RESOURCE) $$($1_RES_VARDEPS_FILE)
+ $$(call LogInfo, Compiling resource $$(notdir $$($1_VERSIONINFO_RESOURCE)) (for $$($1_BASENAME)))
+ $$(call MakeDir, $$(@D) $$($1_OBJECT_DIR))
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_RC) $$($1_RCFLAGS) $$($1_SYSROOT_CFLAGS) $(CC_OUT_OPTION)$$@ \
+ $$($1_VERSIONINFO_RESOURCE) 2>&1 ))
+ # Windows RC compiler does not support -showIncludes, so we mis-use CL
+ # for this. Filter out RC specific arguments that are unknown to CL.
+ # For some unknown reason, in this case CL actually outputs the show
+ # includes to stderr so need to redirect it to hide the output from the
+ # main log.
+ $$(call ExecuteWithLog, $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX), \
+ $$($1_CC) $$(filter-out -l%, $$($1_RCFLAGS)) \
+ $$($1_SYSROOT_CFLAGS) -showIncludes -nologo -TC \
+ $(CC_OUT_OPTION)$$($1_RES_DEPS_FILE)$(OBJ_SUFFIX) -P -Fi$$($1_RES_DEPS_FILE).pp \
+ $$($1_VERSIONINFO_RESOURCE)) 2>&1 \
+ | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
+ -e "^$$(notdir $$($1_VERSIONINFO_RESOURCE))$$$$" || test "$$$$?" = "1" ; \
+ $(ECHO) $$($1_RES): \\ > $$($1_RES_DEPS_FILE) ; \
+ $(SED) $(WINDOWS_SHOWINCLUDE_SED_PATTERN) $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX).log \
+ >> $$($1_RES_DEPS_FILE) ; \
+ $(ECHO) >> $$($1_RES_DEPS_FILE) ;\
+ $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_RES_DEPS_FILE) \
+ > $$($1_RES_DEPS_TARGETS_FILE)
+ endif
+endef
diff --git a/make/common/native/DebugSymbols.gmk b/make/common/native/DebugSymbols.gmk
new file mode 100644
index 0000000000000..9f49f5e1d5292
--- /dev/null
+++ b/make/common/native/DebugSymbols.gmk
@@ -0,0 +1,118 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to native debug symbol handling.
+
+################################################################################
+define CreateDebugSymbols
+ ifneq ($$($1_COPY_DEBUG_SYMBOLS), false)
+ $1_COPY_DEBUG_SYMBOLS := $(COPY_DEBUG_SYMBOLS)
+ endif
+
+ ifneq ($$($1_ZIP_EXTERNAL_DEBUG_SYMBOLS), false)
+ $1_ZIP_EXTERNAL_DEBUG_SYMBOLS := $(ZIP_EXTERNAL_DEBUG_SYMBOLS)
+ endif
+
+ ifeq ($$($1_COPY_DEBUG_SYMBOLS), true)
+ ifneq ($$($1_DEBUG_SYMBOLS), false)
+ $$(call SetIfEmpty, $1_SYMBOLS_DIR, $$($1_OUTPUT_DIR))
+ # Only copy debug symbols for dynamic libraries and programs.
+ ifneq ($$($1_TYPE), STATIC_LIBRARY)
+ # Generate debuginfo files.
+ ifeq ($(call isTargetOs, windows), true)
+ $1_EXTRA_LDFLAGS += -debug "-pdb:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).pdb" \
+ "-map:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).map"
+ ifeq ($(SHIP_DEBUG_SYMBOLS), public)
+ $1_EXTRA_LDFLAGS += "-pdbstripped:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).stripped.pdb"
+ endif
+ $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_BASENAME).pdb \
+ $$($1_SYMBOLS_DIR)/$$($1_BASENAME).map
+
+ else ifeq ($(call isTargetOs, linux), true)
+ $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).debuginfo
+ # Setup the command line creating debuginfo files, to be run after linking.
+ # It cannot be run separately since it updates the original target file
+ # Creating the debuglink is done in another command rather than all at once
+ # so we can run it after strip is called, since strip can sometimes mangle the
+ # embedded debuglink, which we want to avoid.
+ $1_CREATE_DEBUGINFO_CMDS := \
+ $$($1_OBJCOPY) --only-keep-debug $$($1_TARGET) $$($1_DEBUGINFO_FILES) && \
+ $$(CHMOD) -x $$($1_DEBUGINFO_FILES)
+ $1_CREATE_DEBUGLINK_CMDS := $(CD) $$($1_SYMBOLS_DIR) && \
+ $$($1_OBJCOPY) --add-gnu-debuglink=$$($1_DEBUGINFO_FILES) $$($1_TARGET)
+
+ else ifeq ($(call isTargetOs, aix), true)
+ # AIX does not provide the equivalent of OBJCOPY to extract debug symbols,
+ # so we copy the compiled object with symbols to the .debuginfo file, which
+ # happens prior to the STRIP_CMD on the original target object file.
+ $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).debuginfo
+ $1_CREATE_DEBUGINFO_CMDS := $(CP) $$($1_TARGET) $$($1_DEBUGINFO_FILES)
+
+ else ifeq ($(call isTargetOs, macosx), true)
+ $1_DEBUGINFO_FILES := \
+ $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM/Contents/Info.plist \
+ $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM/Contents/Resources/DWARF/$$($1_BASENAME)
+ $1_CREATE_DEBUGINFO_CMDS := \
+ $(DSYMUTIL) --out $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM $$($1_TARGET)
+ endif
+
+ # Since the link rule creates more than one file that we want to track,
+ # we have to use some tricks to get make to cooperate. To properly
+ # trigger downstream dependants of $$($1_DEBUGINFO_FILES), we must have
+ # a recipe in the rule below. To avoid rerunning the recipe every time
+ # have it touch the target. If a debuginfo file is deleted by something
+ # external, explicitly delete the TARGET to trigger a rebuild of both.
+ ifneq ($$(wildcard $$($1_DEBUGINFO_FILES)), $$($1_DEBUGINFO_FILES))
+ $$(call LogDebug, Deleting $$($1_BASENAME) because debuginfo files are missing)
+ $$(shell $(RM) $$($1_TARGET))
+ endif
+ $$($1_DEBUGINFO_FILES): $$($1_TARGET)
+ $$(if $$(CORRECT_FUNCTION_IN_RECIPE_EVALUATION), \
+ $$(if $$(wildcard $$@), , $$(error $$@ was not created for $$<)) \
+ )
+ $(TOUCH) $$@
+
+ $1 += $$($1_DEBUGINFO_FILES)
+
+ ifeq ($$($1_ZIP_EXTERNAL_DEBUG_SYMBOLS), true)
+ ifeq ($(call isTargetOs, windows), true)
+ $1_DEBUGINFO_ZIP := $$($1_SYMBOLS_DIR)/$$($1_BASENAME).diz
+ else
+ $1_DEBUGINFO_ZIP := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).diz
+ endif
+ $1 += $$($1_DEBUGINFO_ZIP)
+
+ # The dependency on TARGET is needed for debuginfo files
+ # to be rebuilt properly.
+ $$($1_DEBUGINFO_ZIP): $$($1_DEBUGINFO_FILES) $$($1_TARGET)
+ $(CD) $$($1_SYMBOLS_DIR) && \
+ $(ZIPEXE) -q -r $$@ $$(subst $$($1_SYMBOLS_DIR)/,, $$($1_DEBUGINFO_FILES))
+
+ endif
+ endif # !STATIC_LIBRARY
+ endif # $1_DEBUG_SYMBOLS != false
+ endif # COPY_DEBUG_SYMBOLS
+endef
diff --git a/make/common/native/Flags.gmk b/make/common/native/Flags.gmk
new file mode 100644
index 0000000000000..213312047a4ff
--- /dev/null
+++ b/make/common/native/Flags.gmk
@@ -0,0 +1,225 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to setting up compiler and linker
+# flags, based on various more abstract sources of compilation description,
+# like optimization level.
+
+################################################################################
+# $1 is the prefix of the file to be compiled
+# $2 is the prefix of the library, i.e. $$($1_BASE)
+define SetupCompileFileFlags
+ ifeq ($$($1_OPTIMIZATION), )
+ $1_OPT_CFLAGS := $$($2_OPT_CFLAGS)
+ $1_OPT_CXXFLAGS := $$($2_OPT_CXXFLAGS)
+ else
+ ifeq ($$($1_OPTIMIZATION), NONE)
+ $1_OPT_CFLAGS := $(C_O_FLAG_NONE)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NONE)
+ else ifeq ($$($1_OPTIMIZATION), LOW)
+ $1_OPT_CFLAGS := $(C_O_FLAG_NORM)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NORM)
+ else ifeq ($$($1_OPTIMIZATION), HIGH)
+ $1_OPT_CFLAGS := $(C_O_FLAG_HI)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HI)
+ else ifeq ($$($1_OPTIMIZATION), HIGHEST)
+ $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST)
+ else ifeq ($$($1_OPTIMIZATION), HIGHEST_JVM)
+ $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST_JVM)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST_JVM)
+ else ifeq ($$($1_OPTIMIZATION), SIZE)
+ $1_OPT_CFLAGS := $(C_O_FLAG_SIZE)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_SIZE)
+ else
+ $$(error Unknown value for file OPTIMIZATION: $$($1_OPTIMIZATION))
+ endif
+ endif
+
+ ifneq ($$($2_PRECOMPILED_HEADER), )
+ ifeq ($$(filter $$($1_FILENAME), $$($2_PRECOMPILED_HEADER_EXCLUDE)), )
+ $1_USE_PCH_FLAGS := $$($2_USE_PCH_FLAGS)
+ endif
+ endif
+
+ ifneq ($(DISABLE_WARNING_PREFIX), )
+ $1_WARNINGS_FLAGS := $$(addprefix $(DISABLE_WARNING_PREFIX), \
+ $$($2_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$$($1_FILENAME)) \
+ $$($2_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)_$$($1_FILENAME)))
+ endif
+
+ $1_BASE_CFLAGS := $$($2_CFLAGS) $$($2_EXTRA_CFLAGS) \
+ $$($2_SYSROOT_CFLAGS)
+ $1_BASE_CXXFLAGS := $$($2_CXXFLAGS) $$($2_EXTRA_CXXFLAGS) \
+ $$($2_SYSROOT_CFLAGS) $$($1_EXTRA_CXXFLAGS)
+ $1_BASE_ASFLAGS := $$($2_ASFLAGS) $$($2_EXTRA_ASFLAGS)
+endef
+
+################################################################################
+define SetupCompilerFlags
+ # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS, TOOLCHAIN_TYPE and
+ # OPENJDK_TARGET_OS plus OPENJDK_TARGET_CPU pair dependent variables for CFLAGS.
+ $1_EXTRA_CFLAGS := $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_CFLAGS_$(OPENJDK_TARGET_OS)) \
+ $$($1_CFLAGS_$(TOOLCHAIN_TYPE)) \
+ $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU))
+
+ ifneq ($(DEBUG_LEVEL), release)
+ # Pickup extra debug dependent variables for CFLAGS
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_debug)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)_debug)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_debug)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU)_debug)
+ else
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_release)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)_release)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_release)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU)_release)
+ endif
+ ifeq ($(STATIC_LIBS), true)
+ $1_EXTRA_CFLAGS += $$(STATIC_LIBS_CFLAGS)
+ endif
+
+ # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS and/or TOOLCHAIN_TYPE
+ # dependent variables for CXXFLAGS.
+ $1_EXTRA_CXXFLAGS := $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)) \
+ $$($1_CXXFLAGS_$(TOOLCHAIN_TYPE))
+
+ ifneq ($(DEBUG_LEVEL), release)
+ # Pickup extra debug dependent variables for CXXFLAGS
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_debug)
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)_debug)
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)_debug)
+ else
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_release)
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)_release)
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)_release)
+ endif
+ ifeq ($(STATIC_LIBS), true)
+ $1_EXTRA_CXXFLAGS += $$(STATIC_LIB_CFLAGS)
+ endif
+
+ # If no C++ flags are explicitly set, default to using the C flags.
+ # After that, we can set additional C++ flags that should not interfere
+ # with the mechanism for copying the C flags by default.
+ ifeq ($$($1_CXXFLAGS), )
+ $1_CXXFLAGS := $$($1_CFLAGS)
+ endif
+ ifeq ($$(strip $$($1_EXTRA_CXXFLAGS)), )
+ $1_EXTRA_CXXFLAGS := $$($1_EXTRA_CFLAGS)
+ endif
+
+ $$(call SetIfEmpty, $1_COMPILE_WITH_DEBUG_SYMBOLS, $$(COMPILE_WITH_DEBUG_SYMBOLS))
+
+ ifeq ($(STATIC_LIBS), true)
+ # For release builds where debug symbols are configured to be moved to
+ # separate debuginfo files, disable debug symbols for static libs instead.
+ # We don't currently support this configuration and we don't want symbol
+ # information in release builds unless explicitly asked to provide it.
+ ifeq ($(DEBUG_LEVEL), release)
+ ifeq ($(COPY_DEBUG_SYMBOLS), true)
+ $1_COMPILE_WITH_DEBUG_SYMBOLS := false
+ endif
+ endif
+ endif
+
+ ifeq ($$($1_COMPILE_WITH_DEBUG_SYMBOLS), true)
+ $1_EXTRA_CFLAGS += $$(CFLAGS_DEBUG_SYMBOLS)
+ $1_EXTRA_CXXFLAGS += $$(CFLAGS_DEBUG_SYMBOLS)
+ $1_EXTRA_ASFLAGS += $$(ASFLAGS_DEBUG_SYMBOLS)
+ endif
+
+ # Pass the library name for static JNI library naming
+ ifeq ($$($1_TYPE), STATIC_LIBRARY)
+ $1_EXTRA_CFLAGS += -DLIBRARY_NAME=$$($1_NAME)
+ $1_EXTRA_CXXFLAGS += -DLIBRARY_NAME=$$($1_NAME)
+ endif
+
+ # Pick up disabled warnings, if possible on this platform.
+ ifneq ($(DISABLE_WARNING_PREFIX), )
+ $1_EXTRA_CFLAGS += $$(addprefix $(DISABLE_WARNING_PREFIX), \
+ $$(DISABLED_WARNINGS) \
+ $$(DISABLED_WARNINGS_C) \
+ $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)) \
+ $$($1_DISABLED_WARNINGS_C_$(TOOLCHAIN_TYPE)) \
+ $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)) \
+ $$($1_DISABLED_WARNINGS_C_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)))
+ $1_EXTRA_CXXFLAGS += $$(addprefix $(DISABLE_WARNING_PREFIX), \
+ $$(DISABLED_WARNINGS) \
+ $$(DISABLED_WARNINGS_CXX) \
+ $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)) \
+ $$($1_DISABLED_WARNINGS_CXX_$(TOOLCHAIN_TYPE)) \
+ $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)) \
+ $$($1_DISABLED_WARNINGS_CXX_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)))
+ endif
+
+ # Check if warnings should be considered errors.
+ # Pick first binary and toolchain specific, then binary specific, then general setting.
+ ifeq ($$($1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE)), )
+ ifeq ($$($1_WARNINGS_AS_ERRORS), )
+ $1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE) := $$(WARNINGS_AS_ERRORS)
+ else
+ $1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE) := $$($1_WARNINGS_AS_ERRORS)
+ endif
+ endif
+
+ ifeq ($$($1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE)), true)
+ $1_EXTRA_CFLAGS += $(CFLAGS_WARNINGS_ARE_ERRORS)
+ $1_EXTRA_CXXFLAGS += $(CFLAGS_WARNINGS_ARE_ERRORS)
+ endif
+
+ ifeq (NONE, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_NONE)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NONE)
+ else ifeq (LOW, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_NORM)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NORM)
+ else ifeq (HIGH, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_HI)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HI)
+ else ifeq (HIGHEST, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST)
+ else ifeq (HIGHEST_JVM, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST_JVM)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST_JVM)
+ else ifeq (SIZE, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_SIZE)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_SIZE)
+ else ifneq (, $$($1_OPTIMIZATION))
+ $$(error Unknown value for OPTIMIZATION: $$($1_OPTIMIZATION))
+ endif
+endef
+
+################################################################################
+define SetupLinkerFlags
+ # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS and TOOLCHAIN_TYPE
+ # dependent variables for LDFLAGS and LIBS, and additionally the pair dependent
+ # TOOLCHAIN_TYPE plus OPENJDK_TARGET_OS
+ $1_EXTRA_LDFLAGS += $$($1_LDFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_LDFLAGS_$(OPENJDK_TARGET_OS)) \
+ $$($1_LDFLAGS_$(TOOLCHAIN_TYPE)) $$($1_LDFLAGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS))
+ $1_EXTRA_LIBS += $$($1_LIBS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_LIBS_$(OPENJDK_TARGET_OS)) \
+ $$($1_LIBS_$(TOOLCHAIN_TYPE)) $$($1_LIBS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS))
+endef
diff --git a/make/common/native/Link.gmk b/make/common/native/Link.gmk
new file mode 100644
index 0000000000000..fb23152d4fb9b
--- /dev/null
+++ b/make/common/native/Link.gmk
@@ -0,0 +1,182 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to linking a native binary;
+# creating either a dynamic library, a static library or an executable.
+
+################################################################################
+# GetEntitlementsFile
+# Find entitlements file for executable when signing on macosx. If no
+# specialized file is found, returns the default file.
+# This macro might be called from custom makefiles.
+# $1 Executable to find entitlements file for.
+ENTITLEMENTS_DIR := $(TOPDIR)/make/data/macosxsigning
+ifeq ($(MACOSX_CODESIGN_MODE), debug)
+ CODESIGN_PLIST_SUFFIX := -debug
+else
+ CODESIGN_PLIST_SUFFIX :=
+endif
+DEFAULT_ENTITLEMENTS_FILE := $(ENTITLEMENTS_DIR)/default$(CODESIGN_PLIST_SUFFIX).plist
+
+GetEntitlementsFile = \
+ $(foreach f, $(ENTITLEMENTS_DIR)/$(strip $(notdir $1))$(CODESIGN_PLIST_SUFFIX).plist, \
+ $(if $(wildcard $f), $f, $(DEFAULT_ENTITLEMENTS_FILE)) \
+ )
+
+################################################################################
+define SetupLinking
+ # Unless specifically set, stripping should only happen if symbols are also
+ # being copied.
+ $$(call SetIfEmpty, $1_STRIP_SYMBOLS, $$($1_COPY_DEBUG_SYMBOLS))
+
+ ifneq ($$($1_STRIP_SYMBOLS), false)
+ # Default to using the global STRIPFLAGS. Allow for overriding with an
+ # empty value
+ $1_STRIPFLAGS ?= $(STRIPFLAGS)
+ $1_STRIP_CMD := $$($1_STRIP) $$($1_STRIPFLAGS) $$($1_TARGET)
+ endif
+endef
+
+################################################################################
+define CreateLinkedResult
+ ifeq ($$($1_TYPE), STATIC_LIBRARY)
+ $$(eval $$(call CreateStaticLibrary,$1))
+ else
+ $$(eval $$(call CreateDynamicLibraryOrExecutable,$1))
+ endif
+endef
+
+################################################################################
+define CreateStaticLibrary
+ # Include partial linking when building the static library with clang on linux
+ ifeq ($(call isTargetOs, linux), true)
+ ifneq ($(findstring $(TOOLCHAIN_TYPE), clang), )
+ $1_ENABLE_PARTIAL_LINKING := true
+ endif
+ endif
+
+ $1_VARDEPS := $$($1_AR) $$(ARFLAGS) $$($1_ARFLAGS) $$($1_LIBS) \
+ $$($1_EXTRA_LIBS)
+ ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
+ $1_VARDEPS += $$($1_LD) $$($1_SYSROOT_LDFLAGS)
+ endif
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
+ $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
+
+ $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_VARDEPS_FILE)
+
+ $1_AR_OBJ_ARG := $$($1_LD_OBJ_ARG)
+ # With clang on linux, partial linking is enabled and 'AR' takes the output
+ # object from the partial linking step.
+ ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
+ $1_TARGET_RELOCATABLE := $$($1_OBJECT_DIR)/$$($1_PREFIX)$$($1_NAME)_relocatable$(OBJ_SUFFIX)
+ $1_AR_OBJ_ARG := $$($1_TARGET_RELOCATABLE)
+ endif
+
+ $$($1_TARGET): $$($1_TARGET_DEPS)
+ ifneq ($$($1_OBJ_FILE_LIST), )
+ ifeq ($$($1_LINK_OBJS_RELATIVE), true)
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS_RELATIVE, $$($1_OBJ_FILE_LIST)))
+ else
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
+ endif
+ endif
+ $$(call LogInfo, Building static library $$($1_BASENAME))
+ $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
+ # Do partial linking.
+ ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_partial_link, \
+ $(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
+ $$($1_LD) $(LDFLAGS_CXX_PARTIAL_LINKING) $$($1_SYSROOT_LDFLAGS) \
+ -o $$($1_TARGET_RELOCATABLE) \
+ $$($1_LD_OBJ_ARG))
+ endif
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
+ $(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
+ $$($1_AR) $$(ARFLAGS) $$($1_ARFLAGS) -r -cs $$($1_TARGET) \
+ $$($1_AR_OBJ_ARG) $$($1_RES))
+ ifeq ($(STATIC_BUILD), true)
+ $(RM) $$(@D)/$$(basename $$(@F)).symbols; \
+ $(ECHO) "Getting symbols from nm"; \
+ $(NM) $(NMFLAGS) -m $$($1_TARGET) | $(GREP) "__TEXT" | \
+ $(EGREP) -v "non-external|private extern|__TEXT,__eh_frame" | \
+ $(SED) -e 's/.* //' > $$(@D)/$$(basename $$(@F)).symbols
+ endif
+endef
+
+################################################################################
+define CreateDynamicLibraryOrExecutable
+ # A shared dynamic library or an executable binary has been specified
+ ifeq ($$($1_TYPE), LIBRARY)
+ # Generating a dynamic library.
+ $1_EXTRA_LDFLAGS += $$(call SET_SHARED_LIBRARY_NAME,$$($1_BASENAME))
+
+ # Create loadmap on AIX. Helps in diagnosing some problems.
+ ifneq ($(COMPILER_BINDCMD_FILE_FLAG), )
+ $1_EXTRA_LDFLAGS += $(COMPILER_BINDCMD_FILE_FLAG)$$($1_OBJECT_DIR)/$$($1_NOSUFFIX).loadmap
+ endif
+ endif
+
+ $1_VARDEPS := $$($1_LD) $$($1_SYSROOT_LDFLAGS) $$($1_LDFLAGS) \
+ $$($1_EXTRA_LDFLAGS) $$($1_LIBS) $$($1_EXTRA_LIBS) \
+ $$($1_CREATE_DEBUGINFO_CMDS) $$($1_STRIP_CMD) $$($1_CREATE_DEBUGLINK_CMDS)
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
+ $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
+
+ $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_VARDEPS_FILE)
+
+ $$($1_TARGET): $$($1_TARGET_DEPS)
+ ifneq ($$($1_OBJ_FILE_LIST), )
+ ifeq ($$($1_LINK_OBJS_RELATIVE), true)
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS_RELATIVE, $$($1_OBJ_FILE_LIST)))
+ else
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
+ endif
+ endif
+ $$(call LogInfo, Linking $$($1_BASENAME))
+ $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
+ $$(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
+ $$($1_LD) $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) \
+ $$($1_SYSROOT_LDFLAGS) -o $$($1_TARGET) $$($1_LD_OBJ_ARG) \
+ $$($1_LIBS) $$($1_EXTRA_LIBS))
+ $$($1_CREATE_DEBUGINFO_CMDS)
+ $$($1_STRIP_CMD)
+ $$($1_CREATE_DEBUGLINK_CMDS)
+ # On macosx, optionally run codesign on every binary.
+ # Remove signature explicitly first to avoid warnings if the linker
+ # added a default adhoc signature.
+ ifeq ($(MACOSX_CODESIGN_MODE), hardened)
+ $(CODESIGN) --remove-signature $$@
+ $(CODESIGN) -f -s "$(MACOSX_CODESIGN_IDENTITY)" --timestamp \
+ --options runtime --entitlements \
+ $$(call GetEntitlementsFile, $$@) $$@
+ else ifeq ($(MACOSX_CODESIGN_MODE), debug)
+ $(CODESIGN) --remove-signature $$@
+ $(CODESIGN) -f -s - --entitlements \
+ $$(call GetEntitlementsFile, $$@) $$@
+ endif
+endef
diff --git a/make/common/native/LinkMicrosoft.gmk b/make/common/native/LinkMicrosoft.gmk
new file mode 100644
index 0000000000000..f998bf3d117f1
--- /dev/null
+++ b/make/common/native/LinkMicrosoft.gmk
@@ -0,0 +1,112 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to linking a native binary;
+# creating either a dynamic library, a static library or an executable.
+
+################################################################################
+define CreateLinkedResultMicrosoft
+ ifeq ($$($1_TYPE), STATIC_LIBRARY)
+ $$(eval $$(call CreateStaticLibraryMicrosoft,$1))
+ else
+ $$(eval $$(call CreateDynamicLibraryOrExecutableMicrosoft,$1))
+ endif
+endef
+
+################################################################################
+define CreateStaticLibraryMicrosoft
+ $1_VARDEPS := $$($1_LIB) $$(LIBFLAGS) $$($1_LIBFLAGS) $$($1_LIBS) \
+ $$($1_EXTRA_LIBS)
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
+ $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
+
+ $$($1_TARGET): $$($1_ALL_OBJS) $$($1_RES) $$($1_VARDEPS_FILE)
+ ifneq ($$($1_OBJ_FILE_LIST), )
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
+ endif
+ $$(call LogInfo, Building static library $$($1_BASENAME))
+ $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
+ $$($1_LIB) -nologo $$(LIBFLAGS) $$($1_LIBFLAGS) -out:$$($1_TARGET) \
+ $$($1_LD_OBJ_ARG) $$($1_RES))
+endef
+
+################################################################################
+define CreateDynamicLibraryOrExecutableMicrosoft
+ ifeq ($$($1_EMBED_MANIFEST), true)
+ $1_EXTRA_LDFLAGS += -manifest:embed
+ endif
+
+ $1_IMPORT_LIBRARY := $$($1_OBJECT_DIR)/$$($1_NAME).lib
+ $1_EXTRA_LDFLAGS += "-implib:$$($1_IMPORT_LIBRARY)"
+
+ ifeq ($$($1_TYPE), LIBRARY)
+ # To properly trigger downstream dependants of the import library, just as
+ # for debug files, we must have a recipe in the rule. To avoid rerunning
+ # the recipe every time have it touch the target. If an import library
+ # file is deleted by something external, explicitly delete the target to
+ # trigger a rebuild of both.
+ ifneq ($$(wildcard $$($1_IMPORT_LIBRARY)), $$($1_IMPORT_LIBRARY))
+ $$(call LogDebug, Deleting $$($1_BASENAME) because import library is missing)
+ $$(shell $(RM) $$($1_TARGET))
+ endif
+ $$($1_IMPORT_LIBRARY): $$($1_TARGET)
+ $(TOUCH) $$@
+
+ $1 += $$($1_IMPORT_LIBRARY)
+ endif
+
+ $1_VARDEPS := $$($1_LD) $$($1_SYSROOT_LDFLAGS) $$($1_LDFLAGS) \
+ $$($1_EXTRA_LDFLAGS) $$($1_LIBS) $$($1_EXTRA_LIBS) $$($1_MT) \
+ $$($1_MANIFEST_VERSION)
+
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
+ $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
+
+ $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_RES) $$($1_MANIFEST) \
+ $$($1_VARDEPS_FILE)
+
+ $$($1_TARGET): $$($1_TARGET_DEPS)
+ ifneq ($$($1_OBJ_FILE_LIST), )
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
+ endif
+ $$(call LogInfo, Linking $$($1_BASENAME))
+ $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
+ $$($1_LD) -nologo $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) \
+ $$($1_SYSROOT_LDFLAGS) -out:$$($1_TARGET) $$($1_LD_OBJ_ARG) \
+ $$($1_RES) $$($1_LIBS) $$($1_EXTRA_LIBS)) \
+ | $(GREP) -v "^ Creating library .*\.lib and object .*\.exp" || \
+ test "$$$$?" = "1"
+ ifeq ($(call isBuildOsEnv, windows.wsl2), true)
+ $$(CHMOD) +x $$($1_TARGET)
+ endif
+ ifneq ($$($1_MANIFEST), )
+ $$($1_MT) -nologo -manifest $$($1_MANIFEST) \
+ -identity:"$$($1_NAME).exe, version=$$($1_MANIFEST_VERSION)" \
+ -outputresource:$$@;#1
+ endif
+endef
diff --git a/make/common/native/Paths.gmk b/make/common/native/Paths.gmk
new file mode 100644
index 0000000000000..67aa61d86e968
--- /dev/null
+++ b/make/common/native/Paths.gmk
@@ -0,0 +1,247 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to handling paths for source files
+# and object files. This is complicated by the fact that we usually, but not
+# always, use absolute instead of relative paths. It is further complicated
+# by the fact that not all tools allow inputting large lists of files as
+# "@-files", which we normally use to avoid hitting command line length limits.
+# Finally this file contains functionality for locating all source code files
+# that should be included in the compilation.
+
+################################################################################
+# When absolute paths are not allowed in the output, and the compiler does not
+# support any options to avoid it, we need to rewrite compile commands to use
+# relative paths. By doing this, the __FILE__ macro will resolve to relative
+# paths. The relevant input paths on the command line are the -I flags and the
+# path to the source file itself.
+#
+# The macro MakeCommandRelative is used to rewrite the command line like this:
+# 'CD $(WORKSPACE_ROOT) && '
+# and changes all paths in cmd to be relative to the workspace root. This only
+# works properly if the build dir is inside the workspace root. If it's not,
+# relative paths are still calculated, but depending on the distance between the
+# dirs, paths in the build dir may end up as essentially absolute anyway.
+#
+# The fix-deps-file macro is used to adjust the contents of the generated make
+# dependency files to contain paths compatible with make.
+#
+REWRITE_PATHS_RELATIVE = false
+ifeq ($(ALLOW_ABSOLUTE_PATHS_IN_OUTPUT)-$(FILE_MACRO_CFLAGS), false-)
+ REWRITE_PATHS_RELATIVE = true
+endif
+
+# CCACHE_BASEDIR needs fix-deps-file as makefiles use absolute filenames for
+# object files while CCACHE_BASEDIR will make ccache relativize all paths for
+# its compiler. The compiler then produces relative dependency files.
+# make does not know a relative and absolute filename is the same so it will
+# ignore such dependencies. This only applies when the OUTPUTDIR is inside
+# the WORKSPACE_ROOT.
+ifneq ($(CCACHE), )
+ ifneq ($(filter $(WORKSPACE_ROOT)/%, $(OUTPUTDIR)), )
+ REWRITE_PATHS_RELATIVE = true
+ endif
+endif
+
+ifeq ($(REWRITE_PATHS_RELATIVE), true)
+ # Need to handle -I flags as both '-Ifoo' and '-I foo'.
+ MakeCommandRelative = \
+ $(CD) $(WORKSPACE_ROOT) && \
+ $(foreach o, $1, \
+ $(if $(filter $(WORKSPACE_ROOT)/% $(OUTPUTDIR)/%, $o), \
+ $(call RelativePath, $o, $(WORKSPACE_ROOT)) \
+ , \
+ $(if $(filter -I$(WORKSPACE_ROOT)/%, $o), \
+ -I$(call RelativePath, $(patsubst -I%, %, $o), $(WORKSPACE_ROOT)) \
+ , \
+ $o \
+ ) \
+ ) \
+ )
+
+ # When compiling with relative paths, the deps file may come out with relative
+ # paths, and that path may start with './'. First remove any leading ./, then
+ # add WORKSPACE_ROOT to any line not starting with /, while allowing for
+ # leading spaces. There may also be multiple entries on the same line, so start
+ # with splitting such lines.
+ # Non GNU sed (BSD on macosx) cannot substitute in literal \n using regex.
+ # Instead use a bash escaped literal newline. To avoid having unmatched quotes
+ # ruin the ability for an editor to properly syntax highlight this file, define
+ # that newline sequence as a separate variable and add the closing quote behind
+ # a comment.
+ sed_newline := \'$$'\n''#'
+ define fix-deps-file
+ $(SED) \
+ -e 's|\([^ ]\) \{1,\}\([^\\:]\)|\1 \\$(sed_newline) \2|g' \
+ $1.tmp \
+ | $(SED) \
+ -e 's|^\([ ]*\)\./|\1|' \
+ -e '/^[ ]*[^/ ]/s|^\([ ]*\)|\1$(WORKSPACE_ROOT)/|' \
+ > $1
+ endef
+else
+ # By default the MakeCommandRelative macro does nothing.
+ MakeCommandRelative = $1
+
+ # No adjustment is needed.
+ define fix-deps-file
+ $(MV) $1.tmp $1
+ endef
+endif
+
+################################################################################
+define SetupSourceFiles
+ $$(foreach d, $$($1_SRC), $$(if $$(wildcard $$d), , \
+ $$(error SRC specified to SetupNativeCompilation $1 contains missing directory $$d)))
+
+ $1_SRCS_RAW := $$(call FindFiles, $$($1_SRC))
+ # Order src files according to the order of the src dirs
+ $1_SRCS := $$(foreach d, $$($1_SRC), $$(filter $$d%, $$($1_SRCS_RAW)))
+ $1_SRCS := $$(filter $$(NATIVE_SOURCE_EXTENSIONS), $$($1_SRCS))
+ # Extract the C/C++ files.
+ ifneq ($$($1_EXCLUDE_PATTERNS), )
+ # We must not match the exclude pattern against the src root(s).
+ $1_SRCS_WITHOUT_ROOTS := $$($1_SRCS)
+ $$(foreach i, $$($1_SRC), $$(eval $1_SRCS_WITHOUT_ROOTS := $$(patsubst \
+ $$i/%,%, $$($1_SRCS_WITHOUT_ROOTS))))
+ $1_ALL_EXCLUDE_FILES := $$(call containing, $$($1_EXCLUDE_PATTERNS), \
+ $$($1_SRCS_WITHOUT_ROOTS))
+ endif
+ ifneq ($$($1_EXCLUDE_FILES), )
+ $1_ALL_EXCLUDE_FILES += $$($1_EXCLUDE_FILES)
+ endif
+ ifneq ($$($1_ALL_EXCLUDE_FILES), )
+ $1_EXCLUDE_FILES_PAT := $$($1_ALL_EXCLUDE_FILES) \
+ $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$($1_ALL_EXCLUDE_FILES)))
+ $1_EXCLUDE_FILES_PAT := $$(addprefix %, $$($1_EXCLUDE_FILES_PAT))
+ $1_SRCS := $$(filter-out $$($1_EXCLUDE_FILES_PAT), $$($1_SRCS))
+ endif
+ ifneq ($$($1_INCLUDE_FILES), )
+ $1_INCLUDE_FILES_PAT := $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$($1_INCLUDE_FILES)))
+ $1_SRCS := $$(filter $$($1_INCLUDE_FILES_PAT), $$($1_SRCS))
+ endif
+ # Now we have a list of all c/c++ files to compile: $$($1_SRCS)
+
+ # Prepend the source/bin path to the filter expressions. Then do the filtering.
+ ifneq ($$($1_INCLUDES), )
+ $1_SRC_INCLUDES := $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$(addsuffix /%, $$($1_INCLUDES))))
+ $1_SRCS := $$(filter $$($1_SRC_INCLUDES), $$($1_SRCS))
+ endif
+ ifneq ($$($1_EXCLUDES), )
+ $1_SRC_EXCLUDES := $$(addsuffix /%, $$($1_EXCLUDES))
+ $1_SRC_EXCLUDES += $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$(addsuffix /%, $$($1_EXCLUDES))))
+ $1_SRCS := $$(filter-out $$($1_SRC_EXCLUDES), $$($1_SRCS))
+ endif
+
+ $1_SRCS += $$($1_EXTRA_FILES)
+
+ ifeq ($$($1_SRCS), )
+ $$(error No sources found for $1 when looking inside the dirs $$($1_SRC))
+ endif
+
+ ifeq ($$($1_TYPE), EXECUTABLE)
+ ifeq ($(UBSAN_ENABLED), true)
+ # We need to set the default options for UBSan. This needs to be included in every executable.
+ # Rather than copy and paste code to everything with a main function, we add an additional
+ # source file to every executable that exports __ubsan_default_options.
+ ifneq ($$(filter %.cpp %.cc, $$($1_SRCS)), )
+ $1_SRCS += $(TOPDIR)/make/data/ubsan/ubsan_default_options.cpp
+ else
+ $1_SRCS += $(TOPDIR)/make/data/ubsan/ubsan_default_options.c
+ endif
+ endif
+ endif
+endef
+
+################################################################################
+define SetupOutputFiles
+ # Calculate the expected output from compiling the sources
+ $1_EXPECTED_OBJS_FILENAMES := $$(call replace_with_obj_extension, $$(notdir $$($1_SRCS)))
+ $1_EXPECTED_OBJS := $$(addprefix $$($1_OBJECT_DIR)/, $$($1_EXPECTED_OBJS_FILENAMES))
+ # Sort to remove duplicates and provide a reproducible order on the input files to the linker.
+ $1_ALL_OBJS := $$(sort $$($1_EXPECTED_OBJS) $$($1_EXTRA_OBJECT_FILES))
+ ifeq ($(STATIC_LIBS), true)
+ # Exclude the object files that match with $1_STATIC_LIB_EXCLUDE_OBJS.
+ ifneq ($$($1_STATIC_LIB_EXCLUDE_OBJS), )
+ $1_ALL_OBJS := $$(call not-containing, $$($1_STATIC_LIB_EXCLUDE_OBJS), $$($1_ALL_OBJS))
+ endif
+ endif
+endef
+
+################################################################################
+define RemoveSuperfluousOutputFiles
+ # Are there too many object files on disk? Perhaps because some source file was removed?
+ $1_BINS := $$(wildcard $$($1_OBJECT_DIR)/*$(OBJ_SUFFIX))
+ $1_SUPERFLOUS_OBJS := $$(sort $$(filter-out $$($1_EXPECTED_OBJS), $$($1_BINS)))
+ # Clean out the superfluous object files.
+ ifneq ($$($1_SUPERFLUOUS_OBJS), )
+ $$(shell $(RM) -f $$($1_SUPERFLUOUS_OBJS))
+ endif
+endef
+
+################################################################################
+define SetupObjectFileList
+ $1_LD_OBJ_ARG := $$($1_ALL_OBJS)
+
+ # If there are many object files, use an @-file...
+ ifneq ($$(word 17, $$($1_ALL_OBJS)), )
+ $1_OBJ_FILE_LIST := $$($1_OBJECT_DIR)/_$1_objectfilenames.txt
+ ifneq ($(COMPILER_COMMAND_FILE_FLAG), )
+ $1_LD_OBJ_ARG := $(COMPILER_COMMAND_FILE_FLAG)$$($1_OBJ_FILE_LIST)
+ else
+ # ...except for toolchains which don't support them.
+ $1_LD_OBJ_ARG := `cat $$($1_OBJ_FILE_LIST)`
+ endif
+
+ # If we are building static library, 'AR' on macosx/aix may not support @-file.
+ ifeq ($$($1_TYPE), STATIC_LIBRARY)
+ ifeq ($(call isTargetOs, macosx aix), true)
+ $1_LD_OBJ_ARG := `cat $$($1_OBJ_FILE_LIST)`
+ endif
+ endif
+ endif
+
+ # Unfortunately the @-file trick does not work reliably when using clang.
+ # Clang does not propagate the @-file parameter to the ld sub process, but
+ # instead puts the full content on the command line. At least the llvm ld
+ # does not even support an @-file.
+ #
+ # When linking a large amount of object files, we risk hitting the limit
+ # of the command line length even on posix systems if the path length of
+ # the output dir is very long due to our use of absolute paths. To
+ # mitigate this, use paths relative to the output dir when linking over
+ # 500 files with clang and the output dir path is deep.
+ ifneq ($$(word 500, $$($1_ALL_OBJS)), )
+ ifeq ($$(TOOLCHAIN_TYPE), clang)
+ # There is no strlen function in make, but checking path depth is a
+ # reasonable approximation.
+ ifneq ($$(word 10, $$(subst /, ,$$(OUTPUTDIR))), )
+ $1_LINK_OBJS_RELATIVE := true
+ $1_ALL_OBJS_RELATIVE := $$(patsubst $$(OUTPUTDIR)/%, %, $$($1_ALL_OBJS))
+ endif
+ endif
+ endif
+endef
diff --git a/make/conf/github-actions.conf b/make/conf/github-actions.conf
index daa191fc70e3d..38e0f563aca75 100644
--- a/make/conf/github-actions.conf
+++ b/make/conf/github-actions.conf
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -36,6 +36,10 @@ MACOS_X64_BOOT_JDK_EXT=tar.gz
MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk21/fd2272bbf8e04c3dbaee13770090416c/35/GPL/openjdk-21_macos-x64_bin.tar.gz
MACOS_X64_BOOT_JDK_SHA256=af32e84c11009f72f783fdcdc9917efc277893988f097e198e2576875d1e88c1
+MACOS_AARCH64_BOOT_JDK_EXT=tar.gz
+MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk21/fd2272bbf8e04c3dbaee13770090416c/35/GPL/openjdk-21_macos-aarch64_bin.tar.gz
+MACOS_AARCH64_BOOT_JDK_SHA256=f12e1e0a2dffc847951598f597c8ee60fb0913932f24b2b09c62cfd2f0f4dfb9
+
WINDOWS_X64_BOOT_JDK_EXT=zip
WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk21/fd2272bbf8e04c3dbaee13770090416c/35/GPL/openjdk-21_windows-x64_bin.zip
WINDOWS_X64_BOOT_JDK_SHA256=5434faaf029e66e7ce6e75770ca384de476750984a7d2881ef7686894c4b4944
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index d8bca3551217b..af164624877a4 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -441,7 +441,7 @@ var getJibProfilesProfiles = function (input, common, data) {
"macosx-x64": {
target_os: "macosx",
target_cpu: "x64",
- dependencies: ["devkit", "gtest", "pandoc"],
+ dependencies: ["devkit", "gtest", "graphviz", "pandoc"],
configure_args: concat(common.configure_args_64bit, "--with-zlib=system",
"--with-macosx-version-max=11.00.00",
"--enable-compatible-cds-alignment",
@@ -453,7 +453,7 @@ var getJibProfilesProfiles = function (input, common, data) {
"macosx-aarch64": {
target_os: "macosx",
target_cpu: "aarch64",
- dependencies: ["devkit", "gtest", "pandoc"],
+ dependencies: ["devkit", "gtest", "graphviz", "pandoc"],
configure_args: concat(common.configure_args_64bit,
"--with-macosx-version-max=11.00.00"),
},
@@ -486,7 +486,7 @@ var getJibProfilesProfiles = function (input, common, data) {
"linux-aarch64": {
target_os: "linux",
target_cpu: "aarch64",
- dependencies: ["devkit", "gtest", "build_devkit", "pandoc"],
+ dependencies: ["devkit", "gtest", "build_devkit", "graphviz", "pandoc"],
configure_args: [
"--with-zlib=system",
"--disable-dtrace",
@@ -1181,12 +1181,6 @@ var getJibProfilesDependencies = function (input, common) {
revision: (input.build_cpu == "x64" ? "Xcode11.3.1-MacOSX10.15+1.2" : devkit_platform_revisions[devkit_platform])
},
- cups: {
- organization: common.organization,
- ext: "tar.gz",
- revision: "1.0118+1.0"
- },
-
jtreg: {
server: "jpg",
product: "jtreg",
@@ -1237,7 +1231,7 @@ var getJibProfilesDependencies = function (input, common) {
graphviz: {
organization: common.organization,
ext: "tar.gz",
- revision: "2.38.0-1+1.1",
+ revision: "9.0.0+1.0",
module: "graphviz-" + input.target_platform,
configure_args: "DOT=" + input.get("graphviz", "install_path") + "/dot",
environment_path: input.get("graphviz", "install_path")
diff --git a/make/data/hotspot-symbols/symbols-aix b/make/data/hotspot-symbols/symbols-aix
deleted file mode 100644
index 1d32104e8a1ed..0000000000000
--- a/make/data/hotspot-symbols/symbols-aix
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_handle_aix_signal
-numa_error
-numa_warn
diff --git a/make/data/hotspot-symbols/symbols-aix-debug b/make/data/hotspot-symbols/symbols-aix-debug
deleted file mode 100644
index 10887ab2b61fb..0000000000000
--- a/make/data/hotspot-symbols/symbols-aix-debug
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_AccessVMBooleanFlag
-JVM_AccessVMIntFlag
-JVM_VMBreakPoint
diff --git a/make/data/hotspot-symbols/symbols-linux b/make/data/hotspot-symbols/symbols-linux
deleted file mode 100644
index d1f258297d82d..0000000000000
--- a/make/data/hotspot-symbols/symbols-linux
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_handle_linux_signal
-JVM_IsUseContainerSupport
-numa_error
-numa_warn
diff --git a/make/data/hotspot-symbols/symbols-macosx b/make/data/hotspot-symbols/symbols-macosx
deleted file mode 100644
index d0243562b67f9..0000000000000
--- a/make/data/hotspot-symbols/symbols-macosx
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_handle_bsd_signal
diff --git a/make/data/hotspot-symbols/symbols-shared b/make/data/hotspot-symbols/symbols-shared
deleted file mode 100644
index c5b13ef1ee867..0000000000000
--- a/make/data/hotspot-symbols/symbols-shared
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-AsyncGetCallTrace
-jio_fprintf
-jio_printf
-jio_snprintf
-jio_vfprintf
-jio_vsnprintf
-JNI_CreateJavaVM
-JNI_GetCreatedJavaVMs
-JNI_GetDefaultJavaVMInitArgs
-JVM_IsForeignLinkerSupported
-JVM_FindClassFromBootLoader
-JVM_InitAgentProperties
diff --git a/make/data/hotspot-symbols/symbols-unix b/make/data/hotspot-symbols/symbols-unix
deleted file mode 100644
index fbb82a11facb7..0000000000000
--- a/make/data/hotspot-symbols/symbols-unix
+++ /dev/null
@@ -1,233 +0,0 @@
-#
-# Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_ActiveProcessorCount
-JVM_AreNestMates
-JVM_ArrayCopy
-JVM_AssertionStatusDirectives
-JVM_BeforeHalt
-JVM_CallStackWalk
-JVM_Clone
-JVM_ConstantPoolGetClassAt
-JVM_ConstantPoolGetClassAtIfLoaded
-JVM_ConstantPoolGetClassRefIndexAt
-JVM_ConstantPoolGetDoubleAt
-JVM_ConstantPoolGetFieldAt
-JVM_ConstantPoolGetFieldAtIfLoaded
-JVM_ConstantPoolGetFloatAt
-JVM_ConstantPoolGetIntAt
-JVM_ConstantPoolGetLongAt
-JVM_ConstantPoolGetMemberRefInfoAt
-JVM_ConstantPoolGetMethodAt
-JVM_ConstantPoolGetMethodAtIfLoaded
-JVM_ConstantPoolGetNameAndTypeRefIndexAt
-JVM_ConstantPoolGetNameAndTypeRefInfoAt
-JVM_ConstantPoolGetSize
-JVM_ConstantPoolGetStringAt
-JVM_ConstantPoolGetTagAt
-JVM_ConstantPoolGetUTF8At
-JVM_CurrentCarrierThread
-JVM_CurrentThread
-JVM_SetCurrentThread
-JVM_CurrentTimeMillis
-JVM_DefineClass
-JVM_DefineClassWithSource
-JVM_DesiredAssertionStatus
-JVM_DumpAllStacks
-JVM_DumpClassListToFile
-JVM_DumpDynamicArchive
-JVM_DumpThreads
-JVM_ExpandStackFrameInfo
-JVM_FillInStackTrace
-JVM_FindClassFromCaller
-JVM_FindClassFromClass
-JVM_FindLibraryEntry
-JVM_FindLoadedClass
-JVM_FindPrimitiveClass
-JVM_FindSignal
-JVM_FreeMemory
-JVM_GC
-JVM_GetAllThreads
-JVM_GetAndClearReferencePendingList
-JVM_GetArrayElement
-JVM_GetArrayLength
-JVM_GetCallerClass
-JVM_GetClassAccessFlags
-JVM_GetClassAnnotations
-JVM_GetClassConstantPool
-JVM_GetClassContext
-JVM_GetClassCPEntriesCount
-JVM_GetClassCPTypes
-JVM_GetClassDeclaredConstructors
-JVM_GetClassDeclaredFields
-JVM_GetClassDeclaredMethods
-JVM_GetClassFieldsCount
-JVM_GetClassFileVersion
-JVM_GetClassInterfaces
-JVM_GetClassMethodsCount
-JVM_GetClassModifiers
-JVM_GetClassNameUTF
-JVM_GetClassSignature
-JVM_GetClassSigners
-JVM_GetClassTypeAnnotations
-JVM_GetCPClassNameUTF
-JVM_GetCPFieldClassNameUTF
-JVM_GetCPFieldModifiers
-JVM_GetCPFieldNameUTF
-JVM_GetCPFieldSignatureUTF
-JVM_GetCPMethodClassNameUTF
-JVM_GetCPMethodModifiers
-JVM_GetCPMethodNameUTF
-JVM_GetCPMethodSignatureUTF
-JVM_GetDeclaredClasses
-JVM_GetDeclaringClass
-JVM_GetEnclosingMethodInfo
-JVM_GetExtendedNPEMessage
-JVM_GetFieldIxModifiers
-JVM_GetFieldTypeAnnotations
-JVM_GetInheritedAccessControlContext
-JVM_GetManagement
-JVM_GetMethodIxArgsSize
-JVM_GetMethodIxByteCode
-JVM_GetMethodIxByteCodeLength
-JVM_GetMethodIxExceptionIndexes
-JVM_GetMethodIxExceptionsCount
-JVM_GetMethodIxExceptionTableEntry
-JVM_GetMethodIxExceptionTableLength
-JVM_GetMethodIxLocalsCount
-JVM_GetMethodIxMaxStack
-JVM_GetMethodIxModifiers
-JVM_GetMethodIxNameUTF
-JVM_GetMethodIxSignatureUTF
-JVM_GetMethodParameters
-JVM_GetMethodTypeAnnotations
-JVM_GetNanoTimeAdjustment
-JVM_GetNestHost
-JVM_GetNestMembers
-JVM_GetNextThreadIdOffset
-JVM_GetPermittedSubclasses
-JVM_GetPrimitiveArrayElement
-JVM_GetProperties
-JVM_GetProtectionDomain
-JVM_GetRandomSeedForDumping
-JVM_GetRecordComponents
-JVM_GetSimpleBinaryName
-JVM_GetStackAccessControlContext
-JVM_GetSystemPackage
-JVM_GetSystemPackages
-JVM_GetTemporaryDirectory
-JVM_GetVmArguments
-JVM_Halt
-JVM_HasReferencePendingList
-JVM_HoldsLock
-JVM_GetStackTrace
-JVM_IHashCode
-JVM_InitClassName
-JVM_InitStackTraceElement
-JVM_InitStackTraceElementArray
-JVM_InitializeFromArchive
-JVM_InternString
-JVM_Interrupt
-JVM_InvokeMethod
-JVM_IsArrayClass
-JVM_IsCDSDumpingEnabled
-JVM_IsConstructorIx
-JVM_IsDumpingClassList
-JVM_IsFinalizationEnabled
-JVM_IsHiddenClass
-JVM_IsInterface
-JVM_IsPreviewEnabled
-JVM_IsContinuationsSupported
-JVM_IsPrimitiveClass
-JVM_IsRecord
-JVM_IsSameClassPackage
-JVM_IsSharingEnabled
-JVM_IsSupportedJNIVersion
-JVM_IsVMGeneratedMethodIx
-JVM_LatestUserDefinedLoader
-JVM_LoadZipLibrary
-JVM_LoadLibrary
-JVM_LookupDefineClass
-JVM_LookupLambdaProxyClassFromArchive
-JVM_LogLambdaFormInvoker
-JVM_MaxMemory
-JVM_MaxObjectInspectionAge
-JVM_MonitorNotify
-JVM_MonitorNotifyAll
-JVM_MonitorWait
-JVM_MoreStackWalk
-JVM_NanoTime
-JVM_NativePath
-JVM_NewArray
-JVM_NewInstanceFromConstructor
-JVM_NewMultiArray
-JVM_PhantomReferenceRefersTo
-JVM_PrintWarningAtDynamicAgentLoad
-JVM_RaiseSignal
-JVM_RawMonitorCreate
-JVM_RawMonitorDestroy
-JVM_RawMonitorEnter
-JVM_RawMonitorExit
-JVM_ReferenceClear
-JVM_ReferenceRefersTo
-JVM_RegisterContinuationMethods
-JVM_RegisterLambdaProxyClassForArchiving
-JVM_RegisterSignal
-JVM_ReleaseUTF
-JVM_ReportFinalizationComplete
-JVM_SetArrayElement
-JVM_SetClassSigners
-JVM_SetNativeThreadName
-JVM_SetPrimitiveArrayElement
-JVM_SetStackWalkContinuation
-JVM_SetThreadPriority
-JVM_SleepNanos
-JVM_StartThread
-JVM_TotalMemory
-JVM_UnloadLibrary
-JVM_WaitForReferencePendingList
-JVM_Yield
-
-# Module related API's
-JVM_AddModuleExports
-JVM_AddModuleExportsToAll
-JVM_AddModuleExportsToAllUnnamed
-JVM_AddReadsModule
-JVM_DefineArchivedModules
-JVM_DefineModule
-JVM_SetBootLoaderUnnamedModule
-
-# Virtual thread notifications for JVMTI
-JVM_VirtualThreadStart
-JVM_VirtualThreadEnd
-JVM_VirtualThreadMount
-JVM_VirtualThreadUnmount
-JVM_VirtualThreadHideFrames
-JVM_VirtualThreadDisableSuspend
-
-# Scoped values
-JVM_EnsureMaterializedForStackWalk_func
-JVM_FindScopedValueBindings
-JVM_ScopedValueCache
-JVM_SetScopedValueCache
-#
diff --git a/make/data/hotspot-symbols/version-script.txt b/make/data/hotspot-symbols/version-script.txt
new file mode 100644
index 0000000000000..29578bf7cb2e1
--- /dev/null
+++ b/make/data/hotspot-symbols/version-script.txt
@@ -0,0 +1,11 @@
+SUNWprivate_1.1 {
+ global:
+ *;
+
+ local:
+ __bss_start;
+ _edata;
+ _end;
+ _fini;
+ _init;
+};
diff --git a/make/devkit/Tools.gmk b/make/devkit/Tools.gmk
index a964045eb5225..5c29ee4db7587 100644
--- a/make/devkit/Tools.gmk
+++ b/make/devkit/Tools.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -548,6 +548,7 @@ $(BUILDDIR)/$(gcc_ver)/Makefile \
$(PATHPRE) $(ENVS) $(GCC_CFG) $(EXTRA_CFLAGS) \
$(CONFIG) \
--with-sysroot=$(SYSROOT) \
+ --with-debug-prefix-map=$(OUTPUT_ROOT)=devkit \
--enable-languages=c,c++ \
--enable-shared \
--disable-nls \
diff --git a/make/devkit/createGraphvizBundle.sh b/make/devkit/createGraphvizBundle.sh
index 290e68c382c4c..1b890838761ba 100644
--- a/make/devkit/createGraphvizBundle.sh
+++ b/make/devkit/createGraphvizBundle.sh
@@ -1,6 +1,6 @@
-#!/bin/bash -e
+#!/bin/bash
#
-# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -26,38 +26,106 @@
# Create a bundle in the current directory, containing what's needed to run
# the 'dot' program from the graphviz suite by the OpenJDK build.
-TMPDIR=`mktemp -d -t graphvizbundle-XXXX`
-trap "rm -rf \"$TMPDIR\"" EXIT
-
-ORIG_DIR=`pwd`
-cd "$TMPDIR"
-GRAPHVIZ_VERSION=2.38.0-1
-PACKAGE_VERSION=1.1
-TARGET_PLATFORM=linux_x64
-BUNDLE_NAME=graphviz-$TARGET_PLATFORM-$GRAPHVIZ_VERSION+$PACKAGE_VERSION.tar.gz
-wget http://www.graphviz.org/pub/graphviz/stable/redhat/el6/x86_64/os/graphviz-$GRAPHVIZ_VERSION.el6.x86_64.rpm
-wget http://www.graphviz.org/pub/graphviz/stable/redhat/el6/x86_64/os/graphviz-libs-$GRAPHVIZ_VERSION.el6.x86_64.rpm
-wget http://www.graphviz.org/pub/graphviz/stable/redhat/el6/x86_64/os/graphviz-plugins-core-$GRAPHVIZ_VERSION.el6.x86_64.rpm
-wget http://www.graphviz.org/pub/graphviz/stable/redhat/el6/x86_64/os/graphviz-plugins-x-$GRAPHVIZ_VERSION.el6.x86_64.rpm
-wget http://public-yum.oracle.com/repo/OracleLinux/OL6/latest/x86_64/getPackage/libtool-ltdl-2.2.6-15.5.el6.x86_64.rpm
-
-mkdir graphviz
-cd graphviz
-for rpm in ../*.rpm; do
- rpm2cpio $rpm | cpio --extract --make-directories
-done
-
-cat > dot << EOF
+set -eux
+
+mydir="$(cd -- $(dirname ${BASH_SOURCE[0]}) && pwd)"
+me="${mydir}/$(basename ${BASH_SOURCE[0]})"
+
+EXPAT_VERSION="2.6.0"
+EXPAT_URL="https://github.com/libexpat/libexpat/releases/download/R_${EXPAT_VERSION//./_}/expat-${EXPAT_VERSION}.tar.gz"
+EXPAT_SHA256="a13447b9aa67d7c860783fdf6820f33ebdea996900d6d8bbc50a628f55f099f7"
+
+GRAPHVIZ_VERSION="9.0.0"
+GRAPHVIZ_URL="https://gitlab.com/api/v4/projects/4207231/packages/generic/graphviz-releases/${GRAPHVIZ_VERSION}/graphviz-${GRAPHVIZ_VERSION}.tar.xz"
+GRAPHVIZ_SHA256="6c9afda06a732af7658c2619ee713d2545818c3ff19b7b8fd48effcd06d57bf6"
+
+uname_s="$(uname -s)"
+case ${uname_s} in
+ Linux)
+ bundle_os="linux"
+ shacmd="sha256sum --strict --check -"
+ lib_path_var="LD_LIBRARY_PATH"
+ ;;
+ Darwin)
+ bundle_os="macosx"
+ shacmd="shasum -a 256 --strict --check -"
+ lib_path_var="DYLD_LIBRARY_PATH"
+ ;;
+ *)
+ echo "Unknown OS: ${uname_s}"
+ exit 1
+ ;;
+esac
+uname_m="$(uname -m)"
+case ${uname_m} in
+ aarch64|arm64)
+ bundle_cpu="aarch64"
+ ;;
+ x86_64)
+ bundle_cpu="x64"
+ ;;
+esac
+bundle_platform="${bundle_os}_${bundle_cpu}"
+
+build_dir="${mydir}/../../build/graphviz"
+download_dir="${build_dir}/download"
+install_dir="${build_dir}/result/graphviz-${bundle_platform}-${GRAPHVIZ_VERSION}"
+bundle_file="${install_dir}.tar.gz"
+
+expat_dir="${build_dir}/expat"
+expat_src_dir="${expat_dir}/src"
+
+graphviz_dir="${build_dir}/graphviz"
+graphviz_src_dir="${graphviz_dir}/src"
+graphviz_doc_dir="${install_dir}/doc"
+
+mkdir -p "${build_dir}"
+cd "${build_dir}"
+
+download_and_unpack() {
+ local url="$1"
+ local sha256="$2"
+ local file="$3"
+ local dir="$4"
+
+ mkdir -p "$(dirname "${file}")"
+ if [ ! -f "${file}" ]; then
+ curl -L -o "${file}" "${url}"
+ fi
+ echo "${sha256} ${file}" | ${shacmd}
+ if [ ! -d "${dir}" ]; then
+ mkdir -p "${dir}"
+ tar --extract --file "${file}" --directory "${dir}" --strip-components 1
+ fi
+}
+
+download_and_unpack "${EXPAT_URL}" "${EXPAT_SHA256}" "${download_dir}/expat.tar.gz" "${expat_src_dir}"
+download_and_unpack "${GRAPHVIZ_URL}" "${GRAPHVIZ_SHA256}" "${download_dir}/graphviz.tar.gz" "${graphviz_src_dir}"
+
+(
+ cd "${expat_src_dir}"
+ ./configure --prefix="${install_dir}"
+ make -j install
+)
+
+(
+ cd "${graphviz_src_dir}"
+ ./configure --prefix="${install_dir}" EXPAT_CFLAGS="-I${install_dir}/include" EXPAT_LIBS="-L${install_dir}/lib -lexpat"
+ make -j install
+)
+
+cat > "${install_dir}/dot" << EOF
#!/bin/bash
# Get an absolute path to this script
-this_script_dir=\`dirname \$0\`
-this_script_dir=\`cd \$this_script_dir > /dev/null && pwd\`
-export LD_LIBRARY_PATH="\$this_script_dir/usr/lib64:\$LD_LIBRARY_PATH"
-exec \$this_script_dir/usr/bin/dot "\$@"
+this_script_dir="\$(dirname \$0)"
+this_script_dir="\$(cd \${this_script_dir} > /dev/null && pwd)"
+export ${lib_path_var}="\${this_script_dir}/lib:\${this_script_dir}/lib/graphviz"
+exec "\${this_script_dir}/bin/dot" "\$@"
EOF
-chmod +x dot
-export LD_LIBRARY_PATH="$TMPDIR/graphviz/usr/lib64:$LD_LIBRARY_PATH"
+chmod +x "${install_dir}/dot"
# create config file
-./dot -c
-tar -cvzf ../$BUNDLE_NAME *
-cp ../$BUNDLE_NAME "$ORIG_DIR"
+"${install_dir}/dot" -c
+
+cp "${me}" "${install_dir}"
+
+tar --create --gzip --file "${bundle_file}" -C "${install_dir}" .
diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk
index 0898d91e1c2a0..f9e09706141fd 100644
--- a/make/hotspot/gensrc/GensrcAdlc.gmk
+++ b/make/hotspot/gensrc/GensrcAdlc.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -45,13 +45,12 @@ ifeq ($(call check-jvm-feature, compiler2), true)
ADLC_CFLAGS := -qnortti -qeh -q64 -DAIX
endif
else ifeq ($(call isBuildOs, windows), true)
- ADLC_LDFLAGS += -nologo
ADLC_CFLAGS := -nologo -EHsc
ADLC_CFLAGS_WARNINGS := -W3 -D_CRT_SECURE_NO_WARNINGS
endif
# Set the C++ standard
- ADLC_CFLAGS += $(ADLC_LANGSTD_CXXFLAG)
+ ADLC_CFLAGS += $(ADLC_LANGSTD_CXXFLAGS)
# NOTE: The old build didn't set -DASSERT for windows but it doesn't seem to
# hurt.
@@ -72,7 +71,8 @@ ifeq ($(call check-jvm-feature, compiler2), true)
$(eval $(call SetupNativeCompilation, BUILD_ADLC, \
NAME := adlc, \
TYPE := EXECUTABLE, \
- TOOLCHAIN := TOOLCHAIN_BUILD_LINK_CXX, \
+ TARGET_TYPE := BUILD, \
+ LINK_TYPE := C++, \
SRC := $(TOPDIR)/src/hotspot/share/adlc, \
EXTRA_FILES := $(TOPDIR)/src/hotspot/share/opto/opcodes.cpp, \
CFLAGS := $(ADLC_CFLAGS) $(ADLC_CFLAGS_WARNINGS), \
diff --git a/make/hotspot/lib/CompileGtest.gmk b/make/hotspot/lib/CompileGtest.gmk
index 0d17f7a3be562..a50d1ffac9eb4 100644
--- a/make/hotspot/lib/CompileGtest.gmk
+++ b/make/hotspot/lib/CompileGtest.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -50,7 +50,7 @@ endif
$(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
NAME := gtest, \
TYPE := STATIC_LIBRARY, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OUTPUT_DIR := $(JVM_OUTPUTDIR)/libgtest, \
OBJECT_DIR := $(JVM_OUTPUTDIR)/libgtest/objs, \
SRC := \
@@ -75,25 +75,11 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
TARGETS += $(BUILD_GTEST_LIBGTEST)
################################################################################
-
-ifeq ($(call isTargetOs, windows), true)
- GTEST_JVM_MAPFILE := $(JVM_MAPFILE)
-else
- GTEST_JVM_MAPFILE := $(JVM_OUTPUTDIR)/gtest/mapfile
-
- $(JVM_OUTPUTDIR)/gtest/symbols: $(JVM_OUTPUTDIR)/symbols
- $(call MakeDir, $(@D))
- ( $(CAT) $< ; echo "runUnitTests" ) > $@
-
- $(GTEST_JVM_MAPFILE): $(JVM_OUTPUTDIR)/gtest/symbols
- $(call create-mapfile)
-endif
-
# Additional disabled warnings are due to code in the test source.
$(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
NAME := jvm, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OUTPUT_DIR := $(JVM_OUTPUTDIR)/gtest, \
OBJECT_DIR := $(JVM_OUTPUTDIR)/gtest/objs, \
SRC := $(GTEST_TEST_SRC), \
@@ -123,8 +109,6 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
LIBS_unix := -lgtest, \
LIBS_windows := $(JVM_OUTPUTDIR)/libgtest/gtest.lib, \
OPTIMIZATION := $(JVM_OPTIMIZATION), \
- MAPFILE := $(GTEST_JVM_MAPFILE), \
- USE_MAPFILE_FOR_SYMBOLS := true, \
COPY_DEBUG_SYMBOLS := $(GTEST_COPY_DEBUG_SYMBOLS), \
ZIP_EXTERNAL_DEBUG_SYMBOLS := false, \
STRIP_SYMBOLS := false, \
@@ -134,14 +118,19 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
$(BUILD_GTEST_LIBJVM) : $(BUILD_GTEST_LIBGTEST)
+ifeq ($(call isTargetOs, windows), true)
+ $(BUILD_GTEST_LIBJVM_TARGET): $(WIN_EXPORT_FILE)
+endif
+
+
TARGETS += $(BUILD_GTEST_LIBJVM)
################################################################################
$(eval $(call SetupJdkExecutable, BUILD_GTEST_LAUNCHER, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
NAME := gtestLauncher, \
TYPE := EXECUTABLE, \
+ LINK_TYPE := C++, \
OUTPUT_DIR := $(JVM_OUTPUTDIR)/gtest, \
EXTRA_FILES := $(GTEST_LAUNCHER_SRC), \
OBJECT_DIR := $(JVM_OUTPUTDIR)/gtest/launcher-objs, \
diff --git a/make/hotspot/lib/CompileJvm.gmk b/make/hotspot/lib/CompileJvm.gmk
index 8a461e647ab73..69cd80f5171c8 100644
--- a/make/hotspot/lib/CompileJvm.gmk
+++ b/make/hotspot/lib/CompileJvm.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -33,7 +33,6 @@ include lib/JvmFlags.gmk
# Setup compilation of the main Hotspot native library (libjvm).
JVM_OUTPUTDIR := $(JVM_VARIANT_OUTPUTDIR)/libjvm
-JVM_MAPFILE := $(JVM_OUTPUTDIR)/mapfile
################################################################################
# Platform independent setup
@@ -85,7 +84,7 @@ CFLAGS_VM_VERSION := \
DISABLED_WARNINGS_gcc := array-bounds comment delete-non-virtual-dtor \
empty-body implicit-fallthrough int-in-bool-context \
- maybe-uninitialized missing-field-initializers parentheses \
+ maybe-uninitialized missing-field-initializers \
shift-negative-value unknown-pragmas
DISABLED_WARNINGS_clang := sometimes-uninitialized \
@@ -146,12 +145,28 @@ $(call FillFindCache, $(JVM_SRC_DIRS))
# operator new.
LIBJVM_STATIC_EXCLUDE_OBJS := operator_new.o
+ifeq ($(call isTargetOs, windows), true)
+ ifeq ($(STATIC_LIBS), true)
+ WIN_EXPORT_FILE := $(JVM_OUTPUTDIR)/static-win-exports.def
+ else
+ WIN_EXPORT_FILE := $(JVM_OUTPUTDIR)/win-exports.def
+ endif
+
+ JVM_LDFLAGS += -def:$(WIN_EXPORT_FILE)
+endif
+
+ifeq ($(call isTargetOs, linux), true)
+ HOTSPOT_VERSION_SCRIPT := $(TOPDIR)/make/data/hotspot-symbols/version-script.txt
+
+ JVM_LDFLAGS += -Wl,--exclude-libs,ALL -Wl,-version-script=$(HOTSPOT_VERSION_SCRIPT)
+endif
+
################################################################################
# Now set up the actual compilation of the main hotspot native library
$(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
NAME := jvm, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OUTPUT_DIR := $(JVM_LIB_OUTPUTDIR), \
SRC := $(JVM_SRC_DIRS), \
EXCLUDES := $(JVM_EXCLUDES), \
@@ -169,6 +184,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
DISABLED_WARNINGS_gcc_jvmciCodeInstaller.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_jvmtiTagMap.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_postaloc.cpp := address, \
+ DISABLED_WARNINGS_gcc_shenandoahLock.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_synchronizer.cpp := stringop-overflow, \
DISABLED_WARNINGS_clang := $(DISABLED_WARNINGS_clang), \
DISABLED_WARNINGS_clang_arguments.cpp := missing-field-initializers, \
@@ -194,8 +210,6 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
LIBS := $(JVM_LIBS), \
OPTIMIZATION := $(JVM_OPTIMIZATION), \
OBJECT_DIR := $(JVM_OUTPUTDIR)/objs, \
- MAPFILE := $(JVM_MAPFILE), \
- USE_MAPFILE_FOR_SYMBOLS := true, \
STRIPFLAGS := $(JVM_STRIPFLAGS), \
EMBED_MANIFEST := true, \
RC_FILEDESC := $(HOTSPOT_VM_DISTRO) $(OPENJDK_TARGET_CPU_BITS)-Bit $(JVM_VARIANT) VM, \
@@ -204,11 +218,47 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
STATIC_LIB_EXCLUDE_OBJS := $(LIBJVM_STATIC_EXCLUDE_OBJS), \
))
+ifeq ($(call isTargetOs, windows), true)
+ # The following lines create a list of vftable symbols to be filtered out of
+ # the symbol file. Removing this line causes the linker to complain about too
+ # many (> 64K) symbols, so the _guess_ is that this line is here to keep down
+ # the number of exported symbols below that limit.
+ #
+ # Some usages of C++ lambdas require the vftable symbol of classes that use
+ # the lambda type as a template parameter. The usage of those classes won't
+ # link if their vftable symbols are removed. That's why there's an exception
+ # for vftable symbols containing the string 'lambda'.
+ #
+ # A very simple example of a lambda usage that fails if the lambda vftable
+ # symbols are missing in the symbol file:
+ #
+ # #include
+ # std::function f = [](){}
+ FILTER_SYMBOLS_AWK_SCRIPT := \
+ '{ \
+ if ($$7 ~ /\?\?_7.*@@6B@/ && $$7 !~ /type_info/ && $$7 !~ /lambda/) print " " $$7; \
+ }'
+
+ # A more correct solution would be to send BUILD_LIBJVM_ALL_OBJS instead of
+ # cd && *.obj, but this will result in very long command lines, which could be
+ # problematic.
+ $(WIN_EXPORT_FILE): $(BUILD_LIBJVM_ALL_OBJS)
+ $(call LogInfo, Generating list of symbols to export from object files)
+ $(call MakeDir, $(@D))
+ $(ECHO) "EXPORTS" > $@.tmp
+ $(CD) $(BUILD_LIBJVM_OBJECT_DIR) && \
+ $(DUMPBIN) -symbols *$(OBJ_SUFFIX) | $(AWK) $(FILTER_SYMBOLS_AWK_SCRIPT) | $(SORT) -u >> $@.tmp
+ $(RM) $@
+ $(MV) $@.tmp $@
+
+ $(BUILD_LIBJVM_TARGET): $(WIN_EXPORT_FILE)
+endif
+
# Always recompile abstract_vm_version.cpp if libjvm needs to be relinked. This ensures
# that the internal vm version is updated as it relies on __DATE__ and __TIME__
# macros.
ABSTRACT_VM_VERSION_OBJ := $(JVM_OUTPUTDIR)/objs/abstract_vm_version$(OBJ_SUFFIX)
-$(ABSTRACT_VM_VERSION_OBJ): $(filter-out $(ABSTRACT_VM_VERSION_OBJ) $(JVM_MAPFILE), \
+$(ABSTRACT_VM_VERSION_OBJ): $(filter-out $(ABSTRACT_VM_VERSION_OBJ), \
$(BUILD_LIBJVM_TARGET_DEPS))
ifneq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
@@ -236,11 +286,6 @@ endif
# 1540-1090 : (I) The destructor of "..." might not be called.
# 1540-1639 : (I) The behavior of long type bit fields has changed ...
-# Include mapfile generation. It relies on BUILD_LIBJVM_ALL_OBJS which is only
-# defined after the above call to BUILD_LIBJVM. Mapfile will be generated
-# after all object files are built, but before the jvm library is linked.
-include lib/JvmMapfile.gmk
-
TARGETS += $(BUILD_LIBJVM)
################################################################################
diff --git a/make/hotspot/lib/JvmMapfile.gmk b/make/hotspot/lib/JvmMapfile.gmk
deleted file mode 100644
index b2199e7d17c6f..0000000000000
--- a/make/hotspot/lib/JvmMapfile.gmk
+++ /dev/null
@@ -1,176 +0,0 @@
-#
-# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation. Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-$(eval $(call IncludeCustomExtension, hotspot/lib/JvmMapfile.gmk))
-
-################################################################################
-# Combine a list of static symbols
-
-ifeq ($(call And, $(call isTargetOs, windows) $(call isTargetCpu, x86_64)), false)
- # On Windows x86_64, we should not have any symbols at all, since that
- # results in duplicate warnings from the linker (JDK-8043491).
- SYMBOLS_SRC += $(TOPDIR)/make/data/hotspot-symbols/symbols-shared
-endif
-
-ifeq ($(call isTargetOsType, unix), true)
- SYMBOLS_SRC += $(TOPDIR)/make/data/hotspot-symbols/symbols-unix
-endif
-
-ifneq ($(wildcard $(TOPDIR)/make/data/hotspot-symbols/symbols-$(OPENJDK_TARGET_OS)), )
- SYMBOLS_SRC += $(TOPDIR)/make/data/hotspot-symbols/symbols-$(OPENJDK_TARGET_OS)
-endif
-
-ifneq ($(findstring debug, $(DEBUG_LEVEL)), )
- ifneq ($(wildcard $(TOPDIR)/make/data/hotspot-symbols/symbols-$(OPENJDK_TARGET_OS)-debug), )
- SYMBOLS_SRC += $(TOPDIR)/make/data/hotspot-symbols/symbols-$(OPENJDK_TARGET_OS)-debug
- endif
-endif
-
-################################################################################
-# Create a dynamic list of symbols from the built object files. This is highly
-# platform dependent.
-
-ifeq ($(call isTargetOs, linux), true)
- DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) --defined-only *$(OBJ_SUFFIX)
- ifneq ($(FILTER_SYMBOLS_PATTERN), )
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)|
- endif
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)^_ZTV|^gHotSpotVM|^UseSharedSpaces$$
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)|^_ZN9Arguments17SharedArchivePathE$$
- FILTER_SYMBOLS_AWK_SCRIPT := \
- '{ \
- if ($$3 ~ /$(FILTER_SYMBOLS_PATTERN)/) print $$3; \
- }'
-
-else ifeq ($(call isTargetOs, macosx), true)
- # nm on macosx prints out "warning: nm: no name list" to stderr for
- # files without symbols. Hide this, even at the expense of hiding real errors.
- DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) -Uj *$(OBJ_SUFFIX) 2> /dev/null
- ifneq ($(FILTER_SYMBOLS_PATTERN), )
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)|
- endif
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)^_ZTV|^gHotSpotVM
- FILTER_SYMBOLS_AWK_SCRIPT := \
- '{ \
- if ($$3 ~ /$(FILTER_SYMBOLS_PATTERN)/) print $$3; \
- }'
-
-# NOTE: The script is from the old build. It is broken and finds no symbols.
-# The script below might be what was intended, but it fails to link with tons
-# of 'cannot export hidden symbol vtable for X'.
-# '{ if ($$1 ~ /^__ZTV/ || $$1 ~ /^_gHotSpotVM/) print substr($$1, 2) }'
-else ifeq ($(call isTargetOs, aix), true)
- # NOTE: The old build had the solution below. This should to be fixed in
- # configure instead.
-
- # On AIX we have to prevent that we pick up the 'nm' version from the GNU binutils
- # which may be installed under /opt/freeware/bin. So better use an absolute path here!
- # NM=/usr/bin/nm
-
- DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) -B -C *$(OBJ_SUFFIX)
- FILTER_SYMBOLS_AWK_SCRIPT := \
- '{ \
- if (($$2="d" || $$2="D") && ($$3 ~ /^__vft/ || $$3 ~ /^gHotSpotVM/)) print $$3; \
- if ($$3 ~ /^UseSharedSpaces$$/) print $$3; \
- if ($$3 ~ /^SharedArchivePath__9Arguments$$/) print $$3; \
- }'
-
-else ifeq ($(call isTargetOs, windows), true)
- DUMP_SYMBOLS_CMD := $(DUMPBIN) -symbols *$(OBJ_SUFFIX)
-
- # The following lines create a list of vftable symbols to be filtered out of
- # the mapfile. Removing this line causes the linker to complain about too many
- # (> 64K) symbols, so the _guess_ is that this line is here to keep down the
- # number of exported symbols below that limit.
- #
- # Some usages of C++ lambdas require the vftable symbol of classes that use
- # the lambda type as a template parameter. The usage of those classes won't
- # link if their vftable symbols are removed. That's why there's an exception
- # for vftable symbols containing the string 'lambda'.
- #
- # A very simple example of a lambda usage that fails if the lambda vftable
- # symbols are missing in the mapfile:
- #
- # #include
- # std::function f = [](){}
-
- FILTER_SYMBOLS_AWK_SCRIPT := \
- '{ \
- if ($$7 ~ /\?\?_7.*@@6B@/ && $$7 !~ /type_info/ && $$7 !~ /lambda/) print $$7; \
- }'
-
-else
- $(error Unknown target OS $(OPENJDK_TARGET_OS) in JvmMapfile.gmk)
-endif
-
-# A more correct solution would be to send BUILD_LIBJVM_ALL_OBJS instead of
-# cd && *.o, but this will result in very long command lines, which is
-# problematic on some platforms.
-$(JVM_OUTPUTDIR)/symbols-objects: $(BUILD_LIBJVM_ALL_OBJS)
- $(call LogInfo, Generating symbol list from object files)
- $(CD) $(JVM_OUTPUTDIR)/objs && \
- $(DUMP_SYMBOLS_CMD) | $(AWK) $(FILTER_SYMBOLS_AWK_SCRIPT) | $(SORT) -u > $@
-
-SYMBOLS_SRC += $(JVM_OUTPUTDIR)/symbols-objects
-
-################################################################################
-# Now concatenate all symbol lists into a single file and remove comments.
-
-$(JVM_OUTPUTDIR)/symbols: $(SYMBOLS_SRC)
- $(SED) -e '/^#/d' $^ > $@
-
-################################################################################
-# Finally convert the symbol list into a platform-specific mapfile
-
-ifeq ($(call isTargetOs, macosx), true)
- # On macosx, we need to add a leading underscore
- define create-mapfile-work
- $(AWK) '{ if ($$0 ~ ".") { print " _" $$0 } }' < $^ > $@.tmp
- endef
-else ifeq ($(call isTargetOs, windows), true)
- # On windows, add an 'EXPORTS' header
- define create-mapfile-work
- $(ECHO) "EXPORTS" > $@.tmp
- $(AWK) '{ if ($$0 ~ ".") { print " " $$0 } }' < $^ >> $@.tmp
- endef
-else
- # Assume standard linker script
- define create-mapfile-work
- $(PRINTF) "SUNWprivate_1.1 { \n global: \n" > $@.tmp
- $(AWK) '{ if ($$0 ~ ".") { print " " $$0 ";" } }' < $^ >> $@.tmp
- $(PRINTF) " local: \n *; \n }; \n" >> $@.tmp
- endef
-endif
-
-define create-mapfile
- $(call LogInfo, Creating mapfile)
- $(call MakeDir, $(@D))
- $(call create-mapfile-work)
- $(RM) $@
- $(MV) $@.tmp $@
-endef
-
-$(JVM_MAPFILE): $(JVM_OUTPUTDIR)/symbols
- $(call create-mapfile)
diff --git a/make/hotspot/lib/JvmOverrideFiles.gmk b/make/hotspot/lib/JvmOverrideFiles.gmk
index b50d6f8bb36d5..ffb98b9bb9a2b 100644
--- a/make/hotspot/lib/JvmOverrideFiles.gmk
+++ b/make/hotspot/lib/JvmOverrideFiles.gmk
@@ -48,9 +48,6 @@ ifneq ($(FDLIBM_CFLAGS), )
endif
ifeq ($(call isTargetOs, linux), true)
- BUILD_LIBJVM_ostream.cpp_CXXFLAGS := -D_FILE_OFFSET_BITS=64
- BUILD_LIBJVM_logFileOutput.cpp_CXXFLAGS := -D_FILE_OFFSET_BITS=64
-
BUILD_LIBJVM_sharedRuntimeTrig.cpp_CXXFLAGS := -DNO_PCH $(FDLIBM_CFLAGS) $(LIBJVM_FDLIBM_COPY_OPT_FLAG)
BUILD_LIBJVM_sharedRuntimeTrans.cpp_CXXFLAGS := -DNO_PCH $(FDLIBM_CFLAGS) $(LIBJVM_FDLIBM_COPY_OPT_FLAG)
diff --git a/make/jdk/src/classes/build/tools/classlist/HelloClasslist.java b/make/jdk/src/classes/build/tools/classlist/HelloClasslist.java
index d24aee5eee803..921eaeee764da 100644
--- a/make/jdk/src/classes/build/tools/classlist/HelloClasslist.java
+++ b/make/jdk/src/classes/build/tools/classlist/HelloClasslist.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,6 +35,7 @@
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.net.InetAddress;
+import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -132,6 +133,8 @@ public static void main(String ... args) throws Throwable {
String oldDate = String.format("%s%n",
DateFormat.getDateInstance(DateFormat.DEFAULT, Locale.ROOT)
.format(new Date()));
+ StandardCharsets.US_ASCII.encode("");
+ StandardCharsets.UTF_8.encode("");
// A selection of trivial and common reflection operations
var instance = HelloClasslist.class.getConstructor().newInstance();
diff --git a/make/modules/java.base/Java.gmk b/make/modules/java.base/Java.gmk
index 8621ff945ccf6..5820c280fe9d1 100644
--- a/make/modules/java.base/Java.gmk
+++ b/make/modules/java.base/Java.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,11 @@
# questions.
#
-DISABLED_WARNINGS_java += this-escape restricted
+# The base module should be built with all warnings enabled. When a
+# new warning is added to javac, it can be temporarily added to the
+# disabled warnings list.
+#
+# DISABLED_WARNINGS_java +=
DOCLINT += -Xdoclint:all/protected \
'-Xdoclint/package:java.*,javax.*'
@@ -37,7 +41,8 @@ EXCLUDE_FILES += \
EXCLUDES += java/lang/doc-files \
java/lang/classfile/snippet-files \
- java/lang/classfile/components/snippet-files
+ java/lang/classfile/components/snippet-files \
+ java/lang/foreign/snippet-files
# Exclude BreakIterator classes that are just used in compile process to generate
# data files and shouldn't go in the product
diff --git a/make/modules/java.base/Lib.gmk b/make/modules/java.base/Lib.gmk
index 924cb8aae268b..54050d0798626 100644
--- a/make/modules/java.base/Lib.gmk
+++ b/make/modules/java.base/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -242,7 +242,7 @@ endif
ifeq ($(call isTargetOs, linux)+$(call isTargetCpu, x86_64)+$(INCLUDE_COMPILER2)+$(filter $(TOOLCHAIN_TYPE), gcc), true+true+true+gcc)
$(eval $(call SetupJdkLibrary, BUILD_LIB_SIMD_SORT, \
NAME := simdsort, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OPTIMIZATION := HIGH, \
CFLAGS := $(CFLAGS_JDKLIB), \
CXXFLAGS := $(CXXFLAGS_JDKLIB) -std=c++17, \
diff --git a/make/modules/java.base/lib/CoreLibraries.gmk b/make/modules/java.base/lib/CoreLibraries.gmk
index 8904c39449e29..b27013536f8e3 100644
--- a/make/modules/java.base/lib/CoreLibraries.gmk
+++ b/make/modules/java.base/lib/CoreLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -115,7 +115,7 @@ TARGETS += $(BUILD_LIBZIP)
$(eval $(call SetupJdkLibrary, BUILD_LIBJIMAGE, \
NAME := jimage, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OPTIMIZATION := LOW, \
CFLAGS := $(CFLAGS_JDKLIB), \
CXXFLAGS := $(CXXFLAGS_JDKLIB), \
diff --git a/make/modules/java.compiler/Java.gmk b/make/modules/java.compiler/Java.gmk
index e2d5ac264b8b6..04f31a9bc66ee 100644
--- a/make/modules/java.compiler/Java.gmk
+++ b/make/modules/java.compiler/Java.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,10 @@
# questions.
#
+# To the extent technically possible, this module should be built with
+# -Werror and all lint warnings enabled. In particular,
+# DISABLED_WARNINGS_java should not be augmented.
+
DOCLINT += -Xdoclint:all/protected \
'-Xdoclint/package:java.*,javax.*'
diff --git a/make/modules/java.desktop/Lib.gmk b/make/modules/java.desktop/Lib.gmk
index be1ac3f1fb8a2..cb831faebf62c 100644
--- a/make/modules/java.desktop/Lib.gmk
+++ b/make/modules/java.desktop/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -51,13 +51,14 @@ ifeq ($(call isTargetOs, aix), false)
-DUSE_PLATFORM_MIDI_IN=TRUE \
#
+ LIBJSOUND_LINK_TYPE := C
ifeq ($(call isTargetOs, macosx), true)
- LIBJSOUND_TOOLCHAIN := TOOLCHAIN_LINK_CXX
+ LIBJSOUND_LINK_TYPE := C++
endif
$(eval $(call SetupJdkLibrary, BUILD_LIBJSOUND, \
NAME := jsound, \
- TOOLCHAIN := $(LIBJSOUND_TOOLCHAIN), \
+ LINK_TYPE := $(LIBJSOUND_LINK_TYPE), \
OPTIMIZATION := LOW, \
CFLAGS := $(CFLAGS_JDKLIB) \
$(LIBJSOUND_CFLAGS), \
diff --git a/make/modules/java.desktop/lib/Awt2dLibraries.gmk b/make/modules/java.desktop/lib/Awt2dLibraries.gmk
index 4d5ff751981ce..aaf98d088fdad 100644
--- a/make/modules/java.desktop/lib/Awt2dLibraries.gmk
+++ b/make/modules/java.desktop/lib/Awt2dLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -449,7 +449,6 @@ else
LIBFREETYPE_LIBS := -lfreetype
endif
- # gcc_ftobjs.c := maybe-uninitialized required for GCC 7 builds.
$(eval $(call SetupJdkLibrary, BUILD_LIBFREETYPE, \
NAME := freetype, \
OPTIMIZATION := HIGHEST, \
@@ -458,7 +457,6 @@ else
EXTRA_HEADER_DIRS := $(BUILD_LIBFREETYPE_HEADER_DIRS), \
DISABLED_WARNINGS_microsoft := 4267 4244 4996, \
DISABLED_WARNINGS_gcc := dangling-pointer stringop-overflow, \
- DISABLED_WARNINGS_gcc_ftobjs.c := maybe-uninitialized, \
LDFLAGS := $(LDFLAGS_JDKLIB) \
$(call SET_SHARED_LIBRARY_ORIGIN), \
))
@@ -508,8 +506,10 @@ else
# noexcept-type required for GCC 7 builds. Not required for GCC 8+.
# expansion-to-defined required for GCC 9 builds. Not required for GCC 10+.
# maybe-uninitialized required for GCC 8 builds. Not required for GCC 9+.
+ # calloc-transposed-args required for GCC 14 builds. (fixed upstream in Harfbuzz 032c931e1c0cfb20f18e5acb8ba005775242bd92)
HARFBUZZ_DISABLED_WARNINGS_CXX_gcc := class-memaccess noexcept-type \
- expansion-to-defined dangling-reference maybe-uninitialized
+ expansion-to-defined dangling-reference maybe-uninitialized \
+ calloc-transposed-args
HARFBUZZ_DISABLED_WARNINGS_clang := missing-field-initializers range-loop-analysis
HARFBUZZ_DISABLED_WARNINGS_microsoft := 4267 4244
@@ -562,9 +562,9 @@ LIBFONTMANAGER_CFLAGS += $(X_CFLAGS) -DLE_STANDALONE -DHEADLESS
# libawt_xawt). See JDK-8196516 for details.
$(eval $(call SetupJdkLibrary, BUILD_LIBFONTMANAGER, \
NAME := fontmanager, \
+ LINK_TYPE := C++, \
EXCLUDE_FILES := $(LIBFONTMANAGER_EXCLUDE_FILES) \
AccelGlyphCache.c, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
CFLAGS := $(CFLAGS_JDKLIB) $(LIBFONTMANAGER_CFLAGS), \
CXXFLAGS := $(CXXFLAGS_JDKLIB) $(LIBFONTMANAGER_CFLAGS), \
OPTIMIZATION := $(LIBFONTMANAGER_OPTIMIZATION), \
diff --git a/make/modules/jdk.compiler/Java.gmk b/make/modules/jdk.compiler/Java.gmk
index 7e6793fc6379b..a2dd4f60fa681 100644
--- a/make/modules/jdk.compiler/Java.gmk
+++ b/make/modules/jdk.compiler/Java.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,10 @@
# questions.
#
+# To the extent technically possible, this module should be built with
+# -Werror and all lint warnings enabled. In particular,
+# DISABLED_WARNINGS_java should not be augmented.
+
DOCLINT += -Xdoclint:all/protected \
'-Xdoclint/package:-com.sun.tools.*,-jdk.internal.*,sun.tools.serialver.resources.*'
JAVAC_FLAGS += -XDstringConcat=inline
diff --git a/make/modules/jdk.hotspot.agent/Lib.gmk b/make/modules/jdk.hotspot.agent/Lib.gmk
index d21c969c18813..6d85061583781 100644
--- a/make/modules/jdk.hotspot.agent/Lib.gmk
+++ b/make/modules/jdk.hotspot.agent/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -27,10 +27,7 @@ include LibCommon.gmk
################################################################################
-ifeq ($(call isTargetOs, linux), true)
- SA_CFLAGS := -D_FILE_OFFSET_BITS=64
-
-else ifeq ($(call isTargetOs, macosx), true)
+ifeq ($(call isTargetOs, macosx), true)
SA_CFLAGS := -D_GNU_SOURCE -mno-omit-leaf-frame-pointer \
-mstack-alignment=16 -fPIC
LIBSA_EXTRA_SRC := $(SUPPORT_OUTPUTDIR)/gensrc/jdk.hotspot.agent
@@ -48,16 +45,16 @@ else ifeq ($(call isTargetOs, windows), true)
endif
endif
-SA_TOOLCHAIN := $(TOOLCHAIN_DEFAULT)
+SA_LINK_TYPE := C
ifeq ($(call isTargetOs, linux), true)
- SA_TOOLCHAIN := TOOLCHAIN_LINK_CXX
+ SA_LINK_TYPE := C++
endif
################################################################################
$(eval $(call SetupJdkLibrary, BUILD_LIBSA, \
NAME := saproc, \
- TOOLCHAIN := $(SA_TOOLCHAIN), \
+ LINK_TYPE := $(SA_LINK_TYPE), \
OPTIMIZATION := HIGH, \
DISABLED_WARNINGS_gcc := sign-compare, \
DISABLED_WARNINGS_gcc_ps_core.c := pointer-arith, \
diff --git a/make/modules/jdk.internal.le/Lib.gmk b/make/modules/jdk.internal.le/Lib.gmk
index 75a2446cc5a18..85550e3cc1d8e 100644
--- a/make/modules/jdk.internal.le/Lib.gmk
+++ b/make/modules/jdk.internal.le/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -31,7 +31,7 @@ ifeq ($(call isTargetOs, linux macosx windows), true)
$(eval $(call SetupJdkLibrary, BUILD_LIBLE, \
NAME := le, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OPTIMIZATION := LOW, \
CFLAGS := $(CXXFLAGS_JDKLIB), \
LDFLAGS := $(LDFLAGS_JDKLIB), \
diff --git a/make/modules/jdk.javadoc/Java.gmk b/make/modules/jdk.javadoc/Java.gmk
index 3035864ecbacb..a8b1f2c117333 100644
--- a/make/modules/jdk.javadoc/Java.gmk
+++ b/make/modules/jdk.javadoc/Java.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,8 @@
# questions.
#
-DISABLED_WARNINGS_java += this-escape
+# To the extent technically possible, this module should be built with
+# -Werror and all lint warnings enabled. In particular,
+# DISABLED_WARNINGS_java should not be augmented.
COPY += .xml .css .svg .js .js.template .png .txt
diff --git a/make/modules/jdk.jpackage/Lib.gmk b/make/modules/jdk.jpackage/Lib.gmk
index 1d3e27e8a6b77..58e40d772e135 100644
--- a/make/modules/jdk.jpackage/Lib.gmk
+++ b/make/modules/jdk.jpackage/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -32,13 +32,13 @@ include LauncherCommon.gmk
ifeq ($(call isTargetOs, linux), true)
JPACKAGE_APPLAUNCHER_SRC := \
$(call FindSrcDirsForComponent, jdk.jpackage, applauncher)
- JPACKAGE_APPLAUNCHER_TOOLCHAIN := TOOLCHAIN_DEFAULT
+ JPACKAGE_APPLAUNCHER_LINK_TYPE := C
JPACKAGE_APPLAUNCHER_INCLUDE_FILES := %.c
else
JPACKAGE_APPLAUNCHER_SRC := \
$(call FindSrcDirsForComponent, jdk.jpackage, applauncher) \
$(call FindSrcDirsForComponent, jdk.jpackage, common)
- JPACKAGE_APPLAUNCHER_TOOLCHAIN := TOOLCHAIN_LINK_CXX
+ JPACKAGE_APPLAUNCHER_LINK_TYPE := C++
endif
@@ -59,11 +59,11 @@ JPACKAGE_APPLAUNCHER_INCLUDES := $(addprefix -I, $(JPACKAGE_APPLAUNCHER_SRC))
# Output app launcher executable in resources dir, and symbols in the object dir
$(eval $(call SetupJdkExecutable, BUILD_JPACKAGE_APPLAUNCHEREXE, \
NAME := jpackageapplauncher, \
+ LINK_TYPE := $(JPACKAGE_APPLAUNCHER_LINK_TYPE), \
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/jpackageapplauncher, \
SRC := $(JPACKAGE_APPLAUNCHER_SRC), \
INCLUDE_FILES := $(JPACKAGE_APPLAUNCHER_INCLUDE_FILES), \
- TOOLCHAIN := $(JPACKAGE_APPLAUNCHER_TOOLCHAIN), \
OPTIMIZATION := LOW, \
DISABLED_WARNINGS_clang_LinuxPackage.c := format-nonliteral, \
DISABLED_WARNINGS_clang_JvmLauncherLib.c := format-nonliteral, \
@@ -103,7 +103,7 @@ ifeq ($(call isTargetOs, linux), true)
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjpackageapplauncheraux, \
SRC := $(JPACKAGE_LIBAPPLAUNCHER_SRC), \
EXCLUDE_FILES := LinuxLauncher.c LinuxPackage.c, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OPTIMIZATION := LOW, \
DISABLED_WARNINGS_clang_JvmLauncherLib.c := format-nonliteral, \
DISABLED_WARNINGS_clang_tstrings.cpp := format-nonliteral, \
@@ -177,10 +177,10 @@ ifeq ($(call isTargetOs, windows), true)
# Build non-console version of launcher
$(eval $(call SetupJdkExecutable, BUILD_JPACKAGE_APPLAUNCHERWEXE, \
NAME := jpackageapplauncherw, \
+ LINK_TYPE := $(BUILD_JPACKAGE_APPLAUNCHEREXE_LINK_TYPE), \
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/jpackageapplauncherw, \
SRC := $(BUILD_JPACKAGE_APPLAUNCHEREXE_SRC), \
- TOOLCHAIN := $(BUILD_JPACKAGE_APPLAUNCHEREXE_TOOLCHAIN), \
OPTIMIZATION := $(BUILD_JPACKAGE_APPLAUNCHEREXE_OPTIMIZATION), \
CXXFLAGS := $(BUILD_JPACKAGE_APPLAUNCHEREXE_CXXFLAGS), \
CXXFLAGS_windows := $(BUILD_JPACKAGE_APPLAUNCHEREXE_CXXFLAGS_windows) -DJP_LAUNCHERW, \
diff --git a/make/scripts/compare.sh b/make/scripts/compare.sh
index a395e0cd850ca..44c700c48957e 100644
--- a/make/scripts/compare.sh
+++ b/make/scripts/compare.sh
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -60,13 +60,15 @@ else
STAT_PRINT_SIZE="-c %s"
fi
-COMPARE_EXCEPTIONS_INCLUDE="$TOPDIR/make/scripts/compare_exceptions.sh.incl"
-if [ ! -e "$COMPARE_EXCEPTIONS_INCLUDE" ]; then
- echo "Error: Cannot locate the exceptions file, it should have been here: $COMPARE_EXCEPTIONS_INCLUDE"
- exit 1
+
+if [ "$OPENJDK_TARGET_OS" = "windows" ]; then
+ # We ship a pdb file inside a published zip. Such files can never be built
+ # reproducibly, so ignore it.
+ ACCEPTED_JARZIP_CONTENTS="/modules_libs/java.security.jgss/w2k_lsa_auth.dll.pdb"
+elif [ "$OPENJDK_TARGET_OS" = "macosx" ]; then
+ # Due to signing, we can never get a byte-by-byte identical build on macOS
+ STRIP_TESTS_BEFORE_COMPARE="true"
fi
-# Include exception definitions
-. "$COMPARE_EXCEPTIONS_INCLUDE"
################################################################################
#
@@ -117,35 +119,6 @@ diff_text() {
TMP=$($DIFF $THIS_FILE $OTHER_FILE)
- if test "x$SUFFIX" = "xclass"; then
- if [ "$NAME" = "SystemModules\$all.class" ] \
- || [ "$NAME" = "SystemModules\$default.class" ]; then
- # The SystemModules\$*.classes are not comparable as they contain the
- # module hashes which would require a whole other level of
- # reproducible builds to get reproducible. There is also random
- # order of map initialization.
- TMP=""
- elif [ "$NAME" = "module-info.class" ]; then
- # The module-info.class have several issues with random ordering of
- # elements in HashSets.
- MODULES_CLASS_FILTER="$SED \
- -e 's/,$//' \
- -e 's/;$//' \
- -e 's/^ *[0-9]*://' \
- -e 's/#[0-9]* */#/' \
- -e 's/ *\/\// \/\//' \
- -e 's/aload *[0-9]*/aload X/' \
- -e 's/ldc_w/ldc /' \
- | $SORT \
- "
- $JAVAP -c -constants -l -p "${OTHER_FILE}" \
- | eval "$MODULES_CLASS_FILTER" > ${OTHER_FILE}.javap &
- $JAVAP -c -constants -l -p "${THIS_FILE}" \
- | eval "$MODULES_CLASS_FILTER" > ${THIS_FILE}.javap &
- wait
- TMP=$($DIFF ${OTHER_FILE}.javap ${THIS_FILE}.javap)
- fi
- fi
if test -n "$TMP"; then
echo Files $OTHER_FILE and $THIS_FILE differ
@@ -312,75 +285,60 @@ compare_file_types() {
}
################################################################################
-# Compare the rest of the files
+# Find all files to compare and separate them into different categories
-compare_general_files() {
+locate_files() {
THIS_DIR=$1
- OTHER_DIR=$2
- WORK_DIR=$3
+ TEMP_DIR=$COMPARE_ROOT/support
+ $MKDIR -p $TEMP_DIR
- GENERAL_FILES=$(cd $THIS_DIR && $FIND . -type f ! -name "*.so" ! -name "*.jar" \
- ! -name "*.zip" ! -name "*.debuginfo" ! -name "*.dylib" ! -name "jexec" \
- ! -name "modules" ! -name "ct.sym" ! -name "*.diz" ! -name "*.dll" \
- ! -name "*.cpl" ! -name "*.pdb" ! -name "*.exp" ! -name "*.ilk" \
- ! -name "*.lib" ! -name "*.jmod" ! -name "*.exe" \
- ! -name "*.obj" ! -name "*.o" ! -name "jspawnhelper" ! -name "*.a" \
- ! -name "*.tar.gz" ! -name "gtestLauncher" \
- ! -name "*.map" \
- | $GREP -v "./bin/" | $SORT | $FILTER)
+ ALL_FILES_PATH=$TEMP_DIR/all_files.txt
+ cd $THIS_DIR && $FIND . -type f | $SORT | $FILTER > $ALL_FILES_PATH
- echo Other files with binary differences...
- for f in $GENERAL_FILES
- do
- # Skip all files in test/*/native
- if [[ "$f" == */native/* ]]; then
- continue
- fi
- if [ -e $OTHER_DIR/$f ]; then
- SUFFIX="${f##*.}"
- if [ "$(basename $f)" = "release" ]; then
- # In release file, ignore differences in source rev numbers
- OTHER_FILE=$WORK_DIR/$f.other
- THIS_FILE=$WORK_DIR/$f.this
- $MKDIR -p $(dirname $OTHER_FILE)
- $MKDIR -p $(dirname $THIS_FILE)
- RELEASE_FILTER="$SED -e 's/SOURCE=".*"/SOURCE=/g'"
- $CAT $OTHER_DIR/$f | eval "$RELEASE_FILTER" > $OTHER_FILE
- $CAT $THIS_DIR/$f | eval "$RELEASE_FILTER" > $THIS_FILE
- elif [ "$SUFFIX" = "svg" ]; then
- # GraphViz has non-determinism when generating svg files
- OTHER_FILE=$WORK_DIR/$f.other
- THIS_FILE=$WORK_DIR/$f.this
- $MKDIR -p $(dirname $OTHER_FILE) $(dirname $THIS_FILE)
- SVG_FILTER="$SED \
- -e 's/edge[0-9][0-9]*/edgeX/g'
- "
- $CAT $OTHER_DIR/$f | eval "$SVG_FILTER" > $OTHER_FILE
- $CAT $THIS_DIR/$f | eval "$SVG_FILTER" > $THIS_FILE
- elif [ "$SUFFIX" = "jar_contents" ]; then
- # The jar_contents files may have some lines in random order
- OTHER_FILE=$WORK_DIR/$f.other
- THIS_FILE=$WORK_DIR/$f.this
- $MKDIR -p $(dirname $OTHER_FILE) $(dirname $THIS_FILE)
- $RM $OTHER_FILE $THIS_FILE
- $CAT $OTHER_DIR/$f | $SORT > $OTHER_FILE
- $CAT $THIS_DIR/$f | $SORT > $THIS_FILE
- else
- OTHER_FILE=$OTHER_DIR/$f
- THIS_FILE=$THIS_DIR/$f
- fi
- DIFF_OUT=$($DIFF $OTHER_FILE $THIS_FILE 2>&1)
- if [ -n "$DIFF_OUT" ]; then
- echo $f
- REGRESSIONS=true
- if [ "$SHOW_DIFFS" = "true" ]; then
- echo "$DIFF_OUT"
- fi
- fi
- fi
- done
+ ZIP_FILES_PATH=$TEMP_DIR/zip_files.txt
+ ZIP_FILTER="-e '\.zip$' -e '\.tar.gz$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $ZIP_FILTER > $ZIP_FILES_PATH
+
+ JMOD_FILES_PATH=$TEMP_DIR/jmod_files.txt
+ JMOD_FILTER="-e '\.jmod$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $JMOD_FILTER > $JMOD_FILES_PATH
+
+ JAR_FILES_PATH=$TEMP_DIR/jar_files.txt
+ JAR_FILTER="-e '\.jar$' -e '\.war$' -e '/module$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $JAR_FILTER > $JAR_FILES_PATH
+
+ LIB_FILES_PATH=$TEMP_DIR/lib_files.txt
+ LIB_FILTER="-e '\.dylib$' -e '/lib.*\.so$' -e '\.dll$' -e '\.obj$' -e '\.o$' -e '\.a$' -e '\.cpl$'"
+ # On macos, filter out the dSYM debug symbols files. They are identically named .dylib files that reside
+ # under a *.dSYM directory
+ LIB_EXCLUDE="-e '/lib.*\.dSYM/'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $LIB_FILTER | eval $GREP -v $LIB_EXCLUDE > $LIB_FILES_PATH
+ DEBUG_FILES_PATH=$TEMP_DIR/debug_files.txt
+ DEBUG_FILTER="-e '\.dSYM/' -e '\.debuginfo$' -e '\.diz$' -e '\.pdb$' -e '\.map$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $DEBUG_FILTER > $DEBUG_FILES_PATH
+ EXEC_FILES_PATH=$TEMP_DIR/exec_files.txt
+ if [ "$OPENJDK_TARGET_OS" = "windows" ]; then
+ EXEC_FILTER="-e '\.exe$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $EXEC_FILTER > $EXEC_FILES_PATH
+ else
+ # Find all files with the executable bit set
+ cd $THIS_DIR && $FIND . -type f -perm -100 | $SORT | $FILTER > $EXEC_FILES_PATH
+ fi
+
+ OTHER_FILES_PATH=$TEMP_DIR/other_files.txt
+ ACCOUNTED_FILES_PATH=$TEMP_DIR/accounted_files.txt
+ $CAT $ZIP_FILES_PATH $JMOD_FILES_PATH $JAR_FILES_PATH $LIB_FILES_PATH $DEBUG_FILES_PATH $EXEC_FILES_PATH > $ACCOUNTED_FILES_PATH
+ $CAT $ACCOUNTED_FILES_PATH $ALL_FILES_PATH | $SORT | $UNIQ -u > $OTHER_FILES_PATH
+
+ ALL_ZIP_FILES=$($CAT $ZIP_FILES_PATH)
+ ALL_JMOD_FILES=$($CAT $JMOD_FILES_PATH)
+ ALL_JAR_FILES=$($CAT $JAR_FILES_PATH)
+ ALL_LIB_FILES=$($CAT $LIB_FILES_PATH)
+ ALL_DEBUG_FILES=$($CAT $DEBUG_FILES_PATH)
+ ALL_EXEC_FILES=$($CAT $EXEC_FILES_PATH)
+ ALL_OTHER_FILES=$($CAT $OTHER_FILES_PATH)
}
################################################################################
@@ -450,12 +408,14 @@ compare_zip_file() {
if [ -n "$ONLY_OTHER" ]; then
echo " Only OTHER $ZIP_FILE contains:"
echo "$ONLY_OTHER" | sed "s|Only in $OTHER_UNZIPDIR| |"g | sed 's|: |/|g'
+ REGRESSIONS=true
return_value=1
fi
if [ -n "$ONLY_THIS" ]; then
echo " Only THIS $ZIP_FILE contains:"
echo "$ONLY_THIS" | sed "s|Only in $THIS_UNZIPDIR| |"g | sed 's|: |/|g'
+ REGRESSIONS=true
return_value=1
fi
@@ -484,6 +444,7 @@ compare_zip_file() {
done
if [ -s "$WORK_DIR/$ZIP_FILE.diffs" ]; then
+ REGRESSIONS=true
return_value=1
echo " Differing files in $ZIP_FILE"
$CAT $WORK_DIR/$ZIP_FILE.diffs | $GREP 'differ$' | cut -f 2 -d ' ' | \
@@ -508,6 +469,7 @@ compare_zip_file() {
compare_bin_file $THIS_UNZIPDIR $OTHER_UNZIPDIR $WORK_DIR/$ZIP_FILE.bin \
$file
if [ "$?" != "0" ]; then
+ REGRESSIONS=true
return_value=1
fi
done
@@ -547,12 +509,14 @@ compare_jmod_file() {
if [ -n "$ONLY_OTHER" ]; then
echo " Only OTHER $JMOD_FILE contains:"
echo "$ONLY_OTHER" | sed "s|^>| |"g | sed 's|: |/|g'
+ REGRESSIONS=true
return_value=1
fi
if [ -n "$ONLY_THIS" ]; then
echo " Only THIS $JMOD_FILE contains:"
echo "$ONLY_THIS" | sed "s|^<| |"g | sed 's|: |/|g'
+ REGRESSIONS=true
return_value=1
fi
@@ -567,19 +531,18 @@ compare_all_zip_files() {
OTHER_DIR=$2
WORK_DIR=$3
- ZIPS=$(cd $THIS_DIR && $FIND . -type f -name "*.zip" -o -name "*.tar.gz" \
- | $SORT | $FILTER )
+ locate_files $THIS_DIR
- if [ -n "$ZIPS" ]; then
+ if [ -n "$ALL_ZIP_FILES" ]; then
echo Zip/tar.gz files...
return_value=0
- for f in $ZIPS; do
+ for f in $ALL_ZIP_FILES; do
if [ -f "$OTHER_DIR/$f" ]; then
compare_zip_file $THIS_DIR $OTHER_DIR $WORK_DIR $f
if [ "$?" != "0" ]; then
- return_value=1
REGRESSIONS=true
+ return_value=1
fi
fi
done
@@ -596,18 +559,18 @@ compare_all_jmod_files() {
OTHER_DIR=$2
WORK_DIR=$3
- JMODS=$(cd $THIS_DIR && $FIND . -type f -name "*.jmod" | $SORT | $FILTER )
+ locate_files $THIS_DIR
- if [ -n "$JMODS" ]; then
+ if [ -n "$ALL_JMOD_FILES" ]; then
echo Jmod files...
return_value=0
- for f in $JMODS; do
+ for f in $ALL_JMOD_FILES; do
if [ -f "$OTHER_DIR/$f" ]; then
compare_jmod_file $THIS_DIR $OTHER_DIR $WORK_DIR $f
if [ "$?" != "0" ]; then
- return_value=1
REGRESSIONS=true
+ return_value=1
fi
fi
done
@@ -624,20 +587,18 @@ compare_all_jar_files() {
OTHER_DIR=$2
WORK_DIR=$3
- # TODO filter?
- ZIPS=$(cd $THIS_DIR && $FIND . -type f -name "*.jar" -o -name "*.war" \
- -o -name "modules" | $SORT | $FILTER)
+ locate_files $THIS_DIR
- if [ -n "$ZIPS" ]; then
+ if [ -n "$ALL_JAR_FILES" ]; then
echo Jar files...
return_value=0
- for f in $ZIPS; do
+ for f in $ALL_JAR_FILES; do
if [ -f "$OTHER_DIR/$f" ]; then
compare_zip_file $THIS_DIR $OTHER_DIR $WORK_DIR $f
if [ "$?" != "0" ]; then
- return_value=1
REGRESSIONS=true
+ return_value=1
fi
fi
done
@@ -699,14 +660,16 @@ compare_bin_file() {
unset _NT_SYMBOL_PATH
if [ "$(uname -o)" = "Cygwin" ]; then
THIS=$(cygpath -msa $THIS)
- OTHER=$(cygpath -msa $OTHER)
+ if [ -n "$OTHER" ]; then
+ OTHER=$(cygpath -msa $OTHER)
+ fi
fi
# Build an _NT_SYMBOL_PATH that contains all known locations for
# pdb files.
PDB_DIRS="$(ls -d \
{$OTHER,$THIS}/support/modules_{cmds,libs}/{*,*/*} \
{$OTHER,$THIS}/support/native/jdk.jpackage/* \
- )"
+ 2> /dev/null )"
export _NT_SYMBOL_PATH="$(echo $PDB_DIRS | tr ' ' ';')"
fi
@@ -1047,23 +1010,16 @@ compare_all_libs() {
OTHER_DIR=$2
WORK_DIR=$3
- LIBS=$(cd $THIS_DIR && $FIND . -type f \( -name 'lib*.so' -o -name '*.dylib' \
- -o -name '*.dll' -o -name '*.obj' -o -name '*.o' -o -name '*.a' \
- -o -name '*.cpl' \) | $SORT | $FILTER)
-
- # On macos, filter out the dSYM debug symbols files as they are also
- # named *.dylib.
- if [ "$OPENJDK_TARGET_OS" = "macosx" ]; then
- LIBS=$(echo "$LIBS" | $GREP -v '\.dSYM/')
- fi
+ locate_files $THIS_DIR
- if [ -n "$LIBS" ]; then
+ if [ -n "$ALL_LIB_FILES" ]; then
echo Libraries...
print_binary_diff_header
- for l in $LIBS; do
+ for l in $ALL_LIB_FILES; do
if [ -f "$OTHER_DIR/$l" ]; then
compare_bin_file $THIS_DIR $OTHER_DIR $WORK_DIR $l
if [ "$?" != "0" ]; then
+ REGRESSIONS=true
return_value=1
fi
fi
@@ -1081,33 +1037,16 @@ compare_all_execs() {
OTHER_DIR=$2
WORK_DIR=$3
- if [ "$OPENJDK_TARGET_OS" = "windows" ]; then
- EXECS=$(cd $THIS_DIR && $FIND . -type f -name '*.exe' | $SORT | $FILTER)
- else
- EXECS=$(cd $THIS_DIR && $FIND . -name db -prune -o -type f -perm -100 \! \
- \( -name '*.so' -o -name '*.dylib' -o -name '*.dll' -o -name '*.cgi' \
- -o -name '*.jar' -o -name '*.diz' -o -name 'jcontrol' -o -name '*.properties' \
- -o -name '*.data' -o -name '*.bfc' -o -name '*.src' -o -name '*.txt' \
- -o -name '*.cfg' -o -name 'meta-index' -o -name '*.properties.ja' \
- -o -name '*.xml' -o -name '*.html' -o -name '*.png' -o -name 'README' \
- -o -name '*.zip' -o -name '*.jimage' -o -name '*.java' -o -name '*.mf' \
- -o -name '*.jpg' -o -name '*.wsdl' -o -name '*.js' -o -name '*.sh' \
- -o -name '*.bat' -o -name '*LICENSE' -o -name '*.d' -o -name '*store' \
- -o -name 'blocked' -o -name '*certs' -o -name '*.ttf' \
- -o -name '*.jfc' -o -name '*.dat' -o -name 'release' -o -name '*.dir'\
- -o -name '*.sym' -o -name '*.idl' -o -name '*.h' -o -name '*.access' \
- -o -name '*.template' -o -name '*.policy' -o -name '*.security' \
- -o -name 'COPYRIGHT' -o -name '*.1' -o -name '*.debuginfo' \
- -o -name 'classlist' \) | $SORT | $FILTER)
- fi
-
- if [ -n "$EXECS" ]; then
+ locate_files $THIS_DIR
+
+ if [ -n "$ALL_EXEC_FILES" ]; then
echo Executables...
print_binary_diff_header
- for e in $EXECS; do
+ for e in $ALL_EXEC_FILES; do
if [ -f "$OTHER_DIR/$e" ]; then
compare_bin_file $THIS_DIR $OTHER_DIR $WORK_DIR $e
if [ "$?" != "0" ]; then
+ REGRESSIONS=true
return_value=1
fi
fi
@@ -1117,6 +1056,95 @@ compare_all_execs() {
return $return_value
}
+################################################################################
+# Compare native debug symbol files
+
+compare_all_debug_files() {
+ THIS_DIR=$1
+ OTHER_DIR=$2
+ WORK_DIR=$3
+
+ locate_files $THIS_DIR
+
+ echo Debug symbol files with binary differences...
+ for f in $ALL_DEBUG_FILES
+ do
+ if [ -e $OTHER_DIR/$f ]; then
+ SUFFIX="${f##*.}"
+ if [ "$SUFFIX" = "pdb" ]; then
+ # pdb files are never reproducible
+ DIFF_OUT=""
+ else
+ OTHER_FILE=$OTHER_DIR/$f
+ THIS_FILE=$THIS_DIR/$f
+ DIFF_OUT=$($DIFF $OTHER_FILE $THIS_FILE 2>&1)
+ fi
+
+ if [ -n "$DIFF_OUT" ]; then
+ echo $f
+ REGRESSIONS=true
+ if [ "$SHOW_DIFFS" = "true" ]; then
+ echo "$DIFF_OUT"
+ fi
+ fi
+ fi
+ done
+}
+
+################################################################################
+# Compare the rest of the files
+
+compare_all_other_files() {
+ THIS_DIR=$1
+ OTHER_DIR=$2
+ WORK_DIR=$3
+
+ locate_files $THIS_DIR
+
+ echo Other files with binary differences...
+ for f in $ALL_OTHER_FILES
+ do
+ # Skip all files in test/*/native
+ if [[ "$f" == */native/* ]]; then
+ continue
+ fi
+ if [ -e $OTHER_DIR/$f ]; then
+ SUFFIX="${f##*.}"
+ if [ "$(basename $f)" = "release" ]; then
+ # In release file, ignore differences in source rev numbers
+ OTHER_FILE=$WORK_DIR/$f.other
+ THIS_FILE=$WORK_DIR/$f.this
+ $MKDIR -p $(dirname $OTHER_FILE)
+ $MKDIR -p $(dirname $THIS_FILE)
+ RELEASE_FILTER="$SED -e 's/SOURCE=".*"/SOURCE=/g'"
+ $CAT $OTHER_DIR/$f | eval "$RELEASE_FILTER" > $OTHER_FILE
+ $CAT $THIS_DIR/$f | eval "$RELEASE_FILTER" > $THIS_FILE
+ elif [ "$SUFFIX" = "jar_contents" ]; then
+ # The jar_contents files are generated by the build and may have
+ # some lines in random order. They are only included for demos,
+ # which they shouldn't really...
+ OTHER_FILE=$WORK_DIR/$f.other
+ THIS_FILE=$WORK_DIR/$f.this
+ $MKDIR -p $(dirname $OTHER_FILE) $(dirname $THIS_FILE)
+ $RM $OTHER_FILE $THIS_FILE
+ $CAT $OTHER_DIR/$f | $SORT > $OTHER_FILE
+ $CAT $THIS_DIR/$f | $SORT > $THIS_FILE
+ else
+ OTHER_FILE=$OTHER_DIR/$f
+ THIS_FILE=$THIS_DIR/$f
+ fi
+ DIFF_OUT=$($DIFF $OTHER_FILE $THIS_FILE 2>&1)
+ if [ -n "$DIFF_OUT" ]; then
+ echo $f
+ REGRESSIONS=true
+ if [ "$SHOW_DIFFS" = "true" ]; then
+ echo "$DIFF_OUT"
+ fi
+ fi
+ fi
+ done
+}
+
################################################################################
# Initiate configuration
@@ -1515,22 +1543,31 @@ fi
if [ "$CMP_GENERAL" = "true" ]; then
if [ -n "$THIS_JDK" ] && [ -n "$OTHER_JDK" ]; then
echo -n "JDK "
- compare_general_files $THIS_JDK $OTHER_JDK $COMPARE_ROOT/jdk
+ compare_all_other_files $THIS_JDK $OTHER_JDK $COMPARE_ROOT/jdk
+ echo -n "JDK "
+ compare_all_debug_files $THIS_JDK $OTHER_JDK $COMPARE_ROOT/jdk
fi
if [ -n "$THIS_JDK_BUNDLE" ] && [ -n "$OTHER_JDK_BUNDLE" ]; then
echo -n "JDK Bundle "
- compare_general_files $THIS_JDK_BUNDLE $OTHER_JDK_BUNDLE $COMPARE_ROOT/jdk-bundle
+ compare_all_other_files $THIS_JDK_BUNDLE $OTHER_JDK_BUNDLE $COMPARE_ROOT/jdk-bundle
+ echo -n "JDK Bundle "
+ compare_all_debug_files $THIS_JDK_BUNDLE $OTHER_JDK_BUNDLE $COMPARE_ROOT/jdk-bundle
fi
if [ -n "$THIS_DOCS" ] && [ -n "$OTHER_DOCS" ]; then
echo -n "Docs "
- compare_general_files $THIS_DOCS $OTHER_DOCS $COMPARE_ROOT/docs
+ compare_all_other_files $THIS_DOCS $OTHER_DOCS $COMPARE_ROOT/docs
+ echo -n "Docs "
+ compare_all_debug_files $THIS_DOCS $OTHER_DOCS $COMPARE_ROOT/docs
fi
if [ -n "$THIS_TEST" ] && [ -n "$OTHER_TEST" ]; then
echo -n "Test "
- compare_general_files $THIS_TEST $OTHER_TEST $COMPARE_ROOT/test
+ compare_all_other_files $THIS_TEST $OTHER_TEST $COMPARE_ROOT/test
+ echo -n "Test "
+ compare_all_debug_files $THIS_TEST $OTHER_TEST $COMPARE_ROOT/test
fi
if [ -n "$THIS_BASE_DIR" ] && [ -n "$OTHER_BASE_DIR" ]; then
- compare_general_files $THIS_BASE_DIR $OTHER_BASE_DIR $COMPARE_ROOT/base_dir
+ compare_all_other_files $THIS_BASE_DIR $OTHER_BASE_DIR $COMPARE_ROOT/base_dir
+ compare_all_debug_files $THIS_BASE_DIR $OTHER_BASE_DIR $COMPARE_ROOT/base_dir
fi
fi
diff --git a/make/scripts/compare_exceptions.sh.incl b/make/scripts/compare_exceptions.sh.incl
index cfbfeeb5be4f9..e69de29bb2d1d 100644
--- a/make/scripts/compare_exceptions.sh.incl
+++ b/make/scripts/compare_exceptions.sh.incl
@@ -1,65 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-# This script is not to be run as stand-alone, it should be included from
-# compare.sh.
-
-##########################################################################################
-# Check that we are run via inclusion from compare.sh and not as stand-alone.
-if [ -z "$COMPARE_EXCEPTIONS_INCLUDE" ]; then
- echo "Error: This script should not be run as stand-alone. It is included by compare.sh"
- exit 1
-fi
-
-##########################################################################################
-# Diff exceptions
-
-if [ "$OPENJDK_TARGET_OS" = "linux" ]; then
- if [ "$USE_PRECOMPILED_HEADER" = "true" ]; then
- ACCEPTED_BIN_DIFF="
- ./lib/server/libjvm.so
- ./hotspot/gtest/server/libjvm.so
- "
- STRIP_BEFORE_COMPARE="
- ./hotspot/gtest/server/libjvm.so
- "
- fi
-elif [ "$OPENJDK_TARGET_OS" = "windows" ]; then
- SKIP_BIN_DIFF="true"
- SKIP_FULLDUMP_DIFF="true"
- ACCEPTED_JARZIP_CONTENTS="
- /modules_libs/java.security.jgss/w2k_lsa_auth.dll.pdb
- /modules_libs/java.security.jgss/w2k_lsa_auth.dll.map
- /modules_libs/java.security.jgss/w2k_lsa_auth.dll
- "
-elif [ "$OPENJDK_TARGET_OS" = "macosx" ]; then
- ACCEPTED_BIN_DIFF="
- ./lib/libawt_lwawt.dylib
- ./lib/libosxapp.dylib
- ./lib/libosxui.dylib
- ./lib/server/libjvm.dylib
- ./hotspot/gtest/server/libjvm.dylib
- "
- STRIP_TESTS_BEFORE_COMPARE="true"
-fi
diff --git a/make/test/BuildMicrobenchmark.gmk b/make/test/BuildMicrobenchmark.gmk
index 628f385572829..ba502a5612870 100644
--- a/make/test/BuildMicrobenchmark.gmk
+++ b/make/test/BuildMicrobenchmark.gmk
@@ -107,6 +107,7 @@ $(eval $(call SetupJavaCompilation, BUILD_JDK_MICROBENCHMARK, \
--add-exports java.base/sun.invoke.util=ALL-UNNAMED \
--add-exports java.base/sun.security.util=ALL-UNNAMED \
--enable-preview \
+ -XDsuppressNotes \
-processor org.openjdk.jmh.generators.BenchmarkProcessor, \
JAVA_FLAGS := \
--add-exports java.base/jdk.internal.vm=ALL-UNNAMED \
diff --git a/src/demo/share/java2d/J2DBench/Makefile b/src/demo/share/java2d/J2DBench/Makefile
index 04b0818a2c35b..edc4494e131de 100644
--- a/src/demo/share/java2d/J2DBench/Makefile
+++ b/src/demo/share/java2d/J2DBench/Makefile
@@ -29,6 +29,23 @@
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
+
+ifndef SOURCE
+export SOURCE := 7
+endif
+ifndef TARGET
+export TARGET := 7
+endif
+ifndef JAVAC
+export JAVAC := javac
+endif
+ifndef JAVA
+export JAVA := java
+endif
+ifndef JAR
+export JAR := jar
+endif
+
SOURCEPATH=src
CLASSES=build
DIST=dist
@@ -80,18 +97,18 @@ SCM_DIRs = .hg .svn CVS RCS SCCS Codemgr_wsdata deleted_files
all: mkdirs J2DBench.jar J2DAnalyzer.jar
run: mkdirs J2DBench.jar
- java -jar $(DIST)/J2DBench.jar
+ $(JAVA) -jar $(DIST)/J2DBench.jar
analyze: mkdirs J2DAnalyzer.jar
- java -jar $(DIST)/J2DAnalyzer.jar
+ $(JAVA) -jar $(DIST)/J2DAnalyzer.jar
J2DBench.jar: \
$(J2DBENCH_CLASSES) $(J2DBENCH_RESOURCES) \
$(CLASSES)/j2dbench.manifest
- jar cvmf $(CLASSES)/j2dbench.manifest $(DIST)/J2DBench.jar -C $(CLASSES) j2dbench
+ $(JAR) cvmf $(CLASSES)/j2dbench.manifest $(DIST)/J2DBench.jar -C $(CLASSES) j2dbench
J2DAnalyzer.jar: $(J2DANALYZER_CLASSES) $(CLASSES)/j2danalyzer.manifest
- jar cvmf $(CLASSES)/j2danalyzer.manifest \
+ $(JAR) cvmf $(CLASSES)/j2danalyzer.manifest \
$(DIST)/J2DAnalyzer.jar -C $(CLASSES) j2dbench/report
$(CLASSES)/j2dbench/tests/iio/images: $(RESOURCES)/images
@@ -120,7 +137,7 @@ $(CLASSES):
mkdirs: $(DIST) $(CLASSES)
$(CLASSES)/j2dbench/%.class: $(SOURCEPATH)/j2dbench/%.java
- javac -g:none -source 1.7 -target 1.7 -d $(CLASSES) -sourcepath $(SOURCEPATH) $<
+ $(JAVAC) -g:none -source $(SOURCE) -target $(TARGET) -d $(CLASSES) -sourcepath $(SOURCEPATH) $<
clean:
rm -rf $(CLASSES)
diff --git a/src/demo/share/java2d/J2DBench/README b/src/demo/share/java2d/J2DBench/README
index 3b9f25c13f14e..513c984a6555f 100644
--- a/src/demo/share/java2d/J2DBench/README
+++ b/src/demo/share/java2d/J2DBench/README
@@ -23,6 +23,9 @@ The benchmark requires at least jdk1.4 to compile and run. Note that
source/target is set to 1.7 in the makefile and build.xml, because of
support in jdk 14 compiler. To check compatibility with jdk1.4 you can
use "-source 1.4 -target 1.4" options and jdk1.7.
+Yo can use TARGET/SOURCE of makefile and -Dtarget/surce to set them up for your convinience.
+Similarly you can set JAVA/JAVAC/JAR and -Djava/javac to select diffferent java/javac then is on yoru PATH
+Unluckily in ant, you can not set jar, but ant should honor JAVA_HOME
-----------------------------------------------------------------------
How To Compile
diff --git a/src/demo/share/java2d/J2DBench/build.xml b/src/demo/share/java2d/J2DBench/build.xml
index 7b202946cf145..415c315899eac 100644
--- a/src/demo/share/java2d/J2DBench/build.xml
+++ b/src/demo/share/java2d/J2DBench/build.xml
@@ -39,6 +39,27 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
@@ -64,6 +86,7 @@
description="run J2DAnalyzer" >
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index f637648b2279c..7b94897eaad61 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1237,7 +1237,7 @@ source %{
// r27 is not allocatable when compressed oops is on and heapbase is not
// zero, compressed klass pointers doesn't use r27 after JDK-8234794
- if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
+ if (UseCompressedOops && (CompressedOops::ptrs_base() != nullptr)) {
_NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
_NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
_NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
@@ -1581,7 +1581,7 @@ bool needs_releasing_store(const Node *n)
{
// assert n->is_Store();
StoreNode *st = n->as_Store();
- return st->trailing_membar() != NULL;
+ return st->trailing_membar() != nullptr;
}
// predicate controlling translation of CAS
@@ -1593,9 +1593,9 @@ bool needs_acquiring_load_exclusive(const Node *n)
assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
LoadStoreNode* ldst = n->as_LoadStore();
if (is_CAS(n->Opcode(), false)) {
- assert(ldst->trailing_membar() != NULL, "expected trailing membar");
+ assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
} else {
- return ldst->trailing_membar() != NULL;
+ return ldst->trailing_membar() != nullptr;
}
// so we can just return true here
@@ -1734,7 +1734,7 @@ void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
st->print("sub sp, sp, rscratch1");
}
- if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
+ if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
st->print("\n\t");
st->print("ldr rscratch1, [guard]\n\t");
st->print("dmb ishld\n\t");
@@ -1783,9 +1783,9 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
__ build_frame(framesize);
- if (C->stub_function() == NULL) {
+ if (C->stub_function() == nullptr) {
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
- if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
+ if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
// Dummy labels for just measuring the code size
Label dummy_slow_path;
Label dummy_continuation;
@@ -2153,12 +2153,12 @@ void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
if (!ra_)
st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
else
- implementation(NULL, ra_, false, st);
+ implementation(nullptr, ra_, false, st);
}
#endif
void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- implementation(&cbuf, ra_, false, NULL);
+ implementation(&cbuf, ra_, false, nullptr);
}
uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
@@ -2205,14 +2205,14 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
st->print_cr("# MachUEPNode");
if (UseCompressedClassPointers) {
- st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- if (CompressedKlassPointers::shift() != 0) {
- st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
- }
+ st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tcmpw rscratch1, r10");
} else {
- st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tcmp rscratch1, r10");
}
- st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
}
#endif
@@ -2221,14 +2221,7 @@ void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
{
// This is the unverified entry point.
C2_MacroAssembler _masm(&cbuf);
-
- __ cmp_klass(j_rarg0, rscratch2, rscratch1);
- Label skip;
- // TODO
- // can we avoid this skip and still use a reloc?
- __ br(Assembler::EQ, skip);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- __ bind(skip);
+ __ ic_check(InteriorEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc* ra_) const
@@ -2249,7 +2242,7 @@ int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
// That's why we must use the macroassembler to generate a handler.
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_exception_handler());
- if (base == NULL) {
+ if (base == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -2267,7 +2260,7 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
// That's why we must use the macroassembler to generate a handler.
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_deopt_handler());
- if (base == NULL) {
+ if (base == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -2377,7 +2370,7 @@ int Matcher::min_vector_size(const BasicType bt) {
return MIN2(size, max_size);
}
-int Matcher::superword_max_vector_size(const BasicType bt) {
+int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
return Matcher::max_vector_size(bt);
}
@@ -2410,7 +2403,7 @@ MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd,
case Op_VecX: return new vecXOper();
}
ShouldNotReachHere();
- return NULL;
+ return nullptr;
}
bool Matcher::is_reg2reg_move(MachNode* m) {
@@ -2582,8 +2575,8 @@ Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
}
// Binary src (Replicate con)
-bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
- if (n == NULL || m == NULL) {
+static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
+ if (n == nullptr || m == nullptr) {
return false;
}
@@ -2623,8 +2616,8 @@ bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
// (XorV src (Replicate m1))
// (XorVMask src (MaskAll m1))
-bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
- if (n != NULL && m != NULL) {
+static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
+ if (n != nullptr && m != nullptr) {
return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
VectorNode::is_all_ones_vector(m);
}
@@ -2736,10 +2729,6 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
{
Address addr = mem2address(opcode, base, index, scale, disp);
if (addr.getMode() == Address::base_plus_offset) {
- /* If we get an out-of-range offset it is a bug in the compiler,
- so we assert here. */
- assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
- "c2 compiler bug");
/* Fix up any out-of-range offsets. */
assert_different_registers(rscratch1, base);
assert_different_registers(rscratch1, reg);
@@ -3430,7 +3419,7 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Register dst_reg = as_Register($dst$$reg);
address con = (address)$src$$constant;
- if (con == NULL || con == (address)1) {
+ if (con == nullptr || con == (address)1) {
ShouldNotReachHere();
} else {
relocInfo::relocType rtype = $src->constant_reloc();
@@ -3473,7 +3462,7 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Register dst_reg = as_Register($dst$$reg);
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
ShouldNotReachHere();
} else {
relocInfo::relocType rtype = $src->constant_reloc();
@@ -3492,7 +3481,7 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Register dst_reg = as_Register($dst$$reg);
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
ShouldNotReachHere();
} else {
relocInfo::relocType rtype = $src->constant_reloc();
@@ -3675,7 +3664,7 @@ encode %{
Label miss;
C2_MacroAssembler _masm(&cbuf);
__ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
- NULL, &miss,
+ nullptr, &miss,
/*set_cond_codes:*/ true);
if ($primary) {
__ mov(result_reg, zr);
@@ -3691,7 +3680,7 @@ encode %{
if (!_method) {
// A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
- if (call == NULL) {
+ if (call == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -3705,7 +3694,7 @@ encode %{
RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
: static_call_Relocation::spec(method_index);
call = __ trampoline_call(Address(addr, rspec));
- if (call == NULL) {
+ if (call == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -3715,8 +3704,8 @@ encode %{
cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
} else {
// Emit stub for static call
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
- if (stub == NULL) {
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, call);
+ if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -3735,7 +3724,7 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
int method_index = resolved_method_index(cbuf);
address call = __ ic_call((address)$meth$$method, method_index);
- if (call == NULL) {
+ if (call == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -3764,7 +3753,7 @@ encode %{
CodeBlob *cb = CodeCache::find_blob(entry);
if (cb) {
address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
- if (call == NULL) {
+ if (call == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -4663,7 +4652,7 @@ operand immP()
interface(CONST_INTER);
%}
-// NULL Pointer Immediate
+// Null Pointer Immediate
operand immP0()
%{
predicate(n->get_ptr() == 0);
@@ -4795,7 +4784,7 @@ operand immN()
interface(CONST_INTER);
%}
-// Narrow NULL Pointer Immediate
+// Narrow Null Pointer Immediate
operand immN0()
%{
predicate(n->get_narrowcon() == 0);
@@ -7219,7 +7208,7 @@ instruct loadConP0(iRegPNoSp dst, immP0 con)
match(Set dst con);
ins_cost(INSN_COST);
- format %{ "mov $dst, $con\t# NULL ptr" %}
+ format %{ "mov $dst, $con\t# null pointer" %}
ins_encode(aarch64_enc_mov_p0(dst, con));
@@ -7233,7 +7222,7 @@ instruct loadConP1(iRegPNoSp dst, immP_1 con)
match(Set dst con);
ins_cost(INSN_COST);
- format %{ "mov $dst, $con\t# NULL ptr" %}
+ format %{ "mov $dst, $con\t# null pointer" %}
ins_encode(aarch64_enc_mov_p1(dst, con));
@@ -7275,7 +7264,7 @@ instruct loadConN0(iRegNNoSp dst, immN0 con)
match(Set dst con);
ins_cost(INSN_COST);
- format %{ "mov $dst, $con\t# compressed NULL ptr" %}
+ format %{ "mov $dst, $con\t# compressed null pointer" %}
ins_encode(aarch64_enc_mov_n0(dst, con));
@@ -15256,7 +15245,7 @@ instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlag
ins_encode %{
address tpc = __ zero_words($base$$Register, $cnt$$Register);
- if (tpc == NULL) {
+ if (tpc == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -15277,7 +15266,7 @@ instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe d
ins_encode %{
address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
- if (tpc == NULL) {
+ if (tpc == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -16440,13 +16429,12 @@ instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
%{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
- // TODO
- // identify correct cost
ins_cost(5 * INSN_COST);
- format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
+ format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
ins_encode %{
__ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
@@ -16457,6 +16445,7 @@ instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegP
instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
%{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastUnlock object box));
effect(TEMP tmp, TEMP tmp2);
@@ -16470,6 +16459,37 @@ instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRe
ins_pipe(pipe_serial);
%}
+instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
+%{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastLock object box));
+ effect(TEMP tmp, TEMP tmp2);
+
+ ins_cost(5 * INSN_COST);
+ format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
+
+ ins_encode %{
+ __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
+instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
+%{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastUnlock object box));
+ effect(TEMP tmp, TEMP tmp2);
+
+ ins_cost(5 * INSN_COST);
+ format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
+
+ ins_encode %{
+ __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
// ============================================================================
// Safepoint Instructions
@@ -17104,23 +17124,7 @@ instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
ins_encode %{
// Count is in 8-bit bytes; non-Compact chars are 16 bits.
__ string_equals($str1$$Register, $str2$$Register,
- $result$$Register, $cnt$$Register, 1);
- %}
- ins_pipe(pipe_class_memory);
-%}
-
-instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
- iRegI_R0 result, rFlagsReg cr)
-%{
- predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
- match(Set result (StrEquals (Binary str1 str2) cnt));
- effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
-
- format %{ "String Equals $str1,$str2,$cnt -> $result" %}
- ins_encode %{
- // Count is in 8-bit bytes; non-Compact chars are 16 bits.
- __ string_equals($str1$$Register, $str2$$Register,
- $result$$Register, $cnt$$Register, 2);
+ $result$$Register, $cnt$$Register);
%}
ins_pipe(pipe_class_memory);
%}
@@ -17142,7 +17146,7 @@ instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
$tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
$result$$Register, $tmp$$Register, 1);
- if (tpc == NULL) {
+ if (tpc == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -17167,7 +17171,7 @@ instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
$tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
$result$$Register, $tmp$$Register, 2);
- if (tpc == NULL) {
+ if (tpc == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -17182,7 +17186,7 @@ instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg
format %{ "count positives byte[] $ary1,$len -> $result" %}
ins_encode %{
address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
- if (tpc == NULL) {
+ if (tpc == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -17225,7 +17229,7 @@ instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len
address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
$vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
$vtmp2$$FloatRegister, $tmp$$Register);
- if (tpc == NULL) {
+ if (tpc == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad
index 210efa0b760cd..d611c14f403ab 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2023, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -126,7 +126,7 @@ source %{
}
}
- bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+ bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
if (UseSVE == 0) {
// These operations are not profitable to be vectorized on NEON, because no direct
// NEON instructions support them. But the match rule support for them is profitable for
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
index 3f4ed020f55c5..5c4e13d432f51 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2023, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -116,7 +116,7 @@ source %{
}
}
- bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+ bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
if (UseSVE == 0) {
// These operations are not profitable to be vectorized on NEON, because no direct
// NEON instructions support them. But the match rule support for them is profitable for
diff --git a/src/hotspot/cpu/aarch64/ad_encode.m4 b/src/hotspot/cpu/aarch64/ad_encode.m4
index e6c87cf5b0559..4897998d8709e 100644
--- a/src/hotspot/cpu/aarch64/ad_encode.m4
+++ b/src/hotspot/cpu/aarch64/ad_encode.m4
@@ -19,7 +19,7 @@ dnl Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
dnl or visit www.oracle.com if you need additional information or have any
dnl questions.
dnl
-dnl
+dnl
dnl Process this file with m4 ad_encode.m4 to generate the load/store
dnl patterns used in aarch64.ad.
dnl
@@ -90,4 +90,3 @@ STORE(vRegD,strd,Float,,8)
loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
%}
-
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.cpp b/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
index afeb19e906e99..76f88764416e3 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020 Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -118,10 +118,6 @@ extern "C" {
else
Disassembler::decode((address)start, (address)start + len);
}
-
- JNIEXPORT void das1(uintptr_t insn) {
- das(insn, 1);
- }
}
#define __ as->
@@ -187,6 +183,26 @@ void Address::lea(MacroAssembler *as, Register r) const {
zrf(Rd, 0);
}
+// This encoding is similar (but not quite identical) to the encoding used
+// by literal ld/st. see JDK-8324123.
+// PRFM does not support writeback or pre/post index.
+void Assembler::prfm(const Address &adr, prfop pfop) {
+ Address::mode mode = adr.getMode();
+ // PRFM does not support pre/post index
+ guarantee((mode != Address::pre) && (mode != Address::post), "prfm does not support pre/post indexing");
+ if (mode == Address::literal) {
+ starti;
+ f(0b11, 31, 30), f(0b011, 29, 27), f(0b000, 26, 24);
+ f(pfop, 4, 0);
+ int64_t offset = (adr.target() - pc()) >> 2;
+ sf(offset, 23, 5);
+ } else {
+ assert((mode == Address::base_plus_offset)
+ || (mode == Address::base_plus_offset_reg), "must be base_plus_offset/base_plus_offset_reg");
+ ld_st2(as_Register(pfop), adr, 0b11, 0b10);
+ }
+}
+
// An "all-purpose" add/subtract immediate, per ARM documentation:
// A "programmer-friendly" assembler may accept a negative immediate
// between -(2^24 -1) and -1 inclusive, causing it to convert a
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index 187c1209303a3..9c05c36706d50 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -797,6 +797,8 @@ class Assembler : public AbstractAssembler {
void adrp(Register Rd, const Address &dest, uint64_t &offset) = delete;
+ void prfm(const Address &adr, prfop pfop = PLDL1KEEP);
+
#undef INSN
void add_sub_immediate(Instruction_aarch64 ¤t_insn, Register Rd, Register Rn,
@@ -1574,17 +1576,6 @@ class Assembler : public AbstractAssembler {
#undef INSN
-#define INSN(NAME, size, op) \
- void NAME(const Address &adr, prfop pfop = PLDL1KEEP) { \
- ld_st2(as_Register(pfop), adr, size, op); \
- }
-
- INSN(prfm, 0b11, 0b10); // FIXME: PRFM should not be used with
- // writeback modes, but the assembler
- // doesn't enfore that.
-
-#undef INSN
-
#define INSN(NAME, size, op) \
void NAME(FloatRegister Rt, const Address &adr) { \
ld_st2(as_Register(Rt), adr, size, op, 1); \
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index b83d618506298..9bd8c6b8e9f88 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -53,7 +53,6 @@
#endif
NEEDS_CLEANUP // remove this definitions ?
-const Register IC_Klass = rscratch2; // where the IC klass is cached
const Register SYNC_header = r0; // synchronization header
const Register SHIFT_count = r0; // where count for shift operations must be
@@ -293,27 +292,7 @@ void LIR_Assembler::osr_entry() {
// inline cache check; done before the frame is built.
int LIR_Assembler::check_icache() {
- Register receiver = FrameMap::receiver_opr->as_register();
- Register ic_klass = IC_Klass;
- int start_offset = __ offset();
- __ inline_cache_check(receiver, ic_klass);
-
- // if icache check fails, then jump to runtime routine
- // Note: RECEIVER must still contain the receiver!
- Label dont;
- __ br(Assembler::EQ, dont);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- // We align the verified entry point unless the method body
- // (including its inline cache check) will fit in a single 64-byte
- // icache line.
- if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
- // force alignment after the cache check.
- __ align(CodeEntryAlignment);
- }
-
- __ bind(dont);
- return start_offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -1230,7 +1209,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -2042,7 +2021,7 @@ void LIR_Assembler::emit_static_call_stub() {
__ relocate(static_stub_Relocation::spec(call_pc));
__ emit_static_call_stub();
- assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
+ assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
<= call_stub_size(), "stub too big");
__ end_a_stub();
}
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
index 43ec189255f9c..ef1b5fe2703e6 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
@@ -71,8 +71,8 @@ friend class ArrayCopyStub;
void deoptimize_trap(CodeEmitInfo *info);
enum {
- // call stub: CompiledStaticCall::to_interp_stub_size() +
- // CompiledStaticCall::to_trampoline_stub_size()
+ // call stub: CompiledDirectCall::to_interp_stub_size() +
+ // CompiledDirectCall::to_trampoline_stub_size()
_call_stub_size = 13 * NativeInstruction::instruction_size,
_exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(175),
_deopt_handler_size = 7 * NativeInstruction::instruction_size
diff --git a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
index 952e060ed212a..fd31d2dde5be1 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -831,18 +831,12 @@ void LIRGenerator::do_LibmIntrinsic(Intrinsic* x) {
}
break;
case vmIntrinsics::_dlog:
- if (StubRoutines::dlog() != nullptr) {
- __ call_runtime_leaf(StubRoutines::dlog(), getThreadTemp(), result_reg, cc->args());
- } else {
- __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog), getThreadTemp(), result_reg, cc->args());
- }
+ // Math.log intrinsic is not implemented on AArch64 (see JDK-8210858),
+ // but we can still call the shared runtime.
+ __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog), getThreadTemp(), result_reg, cc->args());
break;
case vmIntrinsics::_dlog10:
- if (StubRoutines::dlog10() != nullptr) {
- __ call_runtime_leaf(StubRoutines::dlog10(), getThreadTemp(), result_reg, cc->args());
- } else {
- __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), getThreadTemp(), result_reg, cc->args());
- }
+ __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), getThreadTemp(), result_reg, cc->args());
break;
case vmIntrinsics::_dpow:
if (StubRoutines::dpow() != nullptr) {
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
index d3a746178f14e..e48d64d90696c 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -80,12 +80,12 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
br(Assembler::NE, slow_case);
}
- // Load object header
- ldr(hdr, Address(obj, hdr_offset));
if (LockingMode == LM_LIGHTWEIGHT) {
lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
} else if (LockingMode == LM_LEGACY) {
Label done;
+ // Load object header
+ ldr(hdr, Address(obj, hdr_offset));
// and mark it as unlocked
orr(hdr, hdr, markWord::unlocked_value);
// save unlocked object header into the displaced header location on the stack
@@ -144,11 +144,6 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
verify_oop(obj);
if (LockingMode == LM_LIGHTWEIGHT) {
- ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
- // We cannot use tbnz here, the target might be too far away and cannot
- // be encoded.
- tst(hdr, markWord::monitor_value);
- br(Assembler::NE, slow_case);
lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
} else if (LockingMode == LM_LEGACY) {
// test if object header is pointing to the displaced header, and if so, restore
@@ -193,6 +188,12 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
+ int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
+ if (!is_aligned(base_offset, BytesPerWord)) {
+ assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+ // Clear gap/first 4 bytes following the length field.
+ strw(zr, Address(obj, base_offset));
+ }
} else if (UseCompressedClassPointers) {
store_klass_gap(obj, zr);
}
@@ -271,7 +272,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, t1, t2, klass);
// determine alignment mask
@@ -284,7 +285,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
const Register arr_size = t2; // okay to be the same
// align object end
- mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ mov(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
add(arr_size, arr_size, len, ext::uxtw, f);
andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
@@ -292,8 +293,11 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
initialize_header(obj, klass, len, t1, t2);
+ // Align-up to word boundary, because we clear the 4 bytes potentially
+ // following the length field in initialize_header().
+ int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
// clear rest of allocated space
- initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
+ initialize_body(obj, arr_size, base_offset, t1, t2);
if (Compilation::current()->bailed_out()) {
return;
}
@@ -308,17 +312,6 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
verify_oop(obj);
}
-
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- verify_oop(receiver);
- // explicit null check not needed since load from [klass_offset] causes a trap
- // check against inline cache
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
-
- cmp_klass(receiver, iCache, rscratch1);
-}
-
-
void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
// Make sure there is enough stack space for this method's activation.
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
index 4aa6206aa6073..d210c21d12b8f 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -100,7 +100,7 @@ using MacroAssembler::null_check;
// header_size: size of object header in words
// f : element scale factor
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_array(Register obj, Register len, Register t, Register t2, int header_size, int f, Register klass, Label& slow_case);
+ void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
int rsp_offset() const { return _rsp_offset; }
void set_rsp_offset(int n) { _rsp_offset = n; }
diff --git a/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp
index d2f4744a04914..63a32e714e365 100644
--- a/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp
@@ -38,7 +38,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_aarch64.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_aarch64.hpp"
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index 7b9784ec47a24..7e3ceb1f02029 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "opto/output.hpp"
#include "opto/subnode.hpp"
#include "runtime/stubRoutines.hpp"
+#include "utilities/globalDefinitions.hpp"
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -55,6 +56,7 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
Label object_has_monitor;
Label count, no_count;
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
assert_different_registers(oop, box, tmp, disp_hdr);
// Load markWord from object into displaced_header.
@@ -73,7 +75,8 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
if (LockingMode == LM_MONITOR) {
tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
b(cont);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Set tmp to be (markWord of object | UNLOCK_VALUE).
orr(tmp, disp_hdr, markWord::unlocked_value);
@@ -102,31 +105,26 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
b(cont);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- lightweight_lock(oop, disp_hdr, tmp, tmp3Reg, no_count);
- b(count);
}
// Handle existing monitor.
bind(object_has_monitor);
- // The object's monitor m is unlocked iff m->owner == NULL,
+ // The object's monitor m is unlocked iff m->owner == nullptr,
// otherwise m->owner may contain a thread or a stack address.
//
- // Try to CAS m->owner from NULL to current thread.
+ // Try to CAS m->owner from null to current thread.
add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
/*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result
- if (LockingMode != LM_LIGHTWEIGHT) {
- // Store a non-null value into the box to avoid looking like a re-entrant
- // lock. The fast-path monitor unlock code checks for
- // markWord::monitor_value so use markWord::unused_mark which has the
- // relevant bit set, and also matches ObjectSynchronizer::enter.
- mov(tmp, (address)markWord::unused_mark().value());
- str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
- }
+ // Store a non-null value into the box to avoid looking like a re-entrant
+ // lock. The fast-path monitor unlock code checks for
+ // markWord::monitor_value so use markWord::unused_mark which has the
+ // relevant bit set, and also matches ObjectSynchronizer::enter.
+ mov(tmp, (address)markWord::unused_mark().value());
+ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
+
br(Assembler::EQ, cont); // CAS success means locking succeeded
cmp(tmp3Reg, rthread);
@@ -157,6 +155,7 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
Label object_has_monitor;
Label count, no_count;
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
assert_different_registers(oop, box, tmp, disp_hdr);
if (LockingMode == LM_LEGACY) {
@@ -175,7 +174,8 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
if (LockingMode == LM_MONITOR) {
tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
b(cont);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Check if it is still a light weight lock, this is is true if we
// see the stack address of the basicLock in the markWord of the
// object.
@@ -183,10 +183,6 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
/*release*/ true, /*weak*/ false, tmp);
b(cont);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- lightweight_unlock(oop, tmp, box, disp_hdr, no_count);
- b(count);
}
assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
@@ -196,19 +192,6 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
add(tmp, tmp, -(int)markWord::monitor_value); // monitor
- if (LockingMode == LM_LIGHTWEIGHT) {
- // If the owner is anonymous, we need to fix it -- in an outline stub.
- Register tmp2 = disp_hdr;
- ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
- // We cannot use tbnz here, the target might be too far away and cannot
- // be encoded.
- tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
- C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
- Compile::current()->output()->add_stub(stub);
- br(Assembler::NE, stub->entry());
- bind(stub->continuation());
- }
-
ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
Label notRecursive;
@@ -241,6 +224,262 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
bind(no_count);
}
+void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
+ Register t2, Register t3) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert_different_registers(obj, t1, t2, t3);
+
+ // Handle inflated monitor.
+ Label inflated;
+ // Finish fast lock successfully. MUST branch to with flag == EQ
+ Label locked;
+ // Finish fast lock unsuccessfully. MUST branch to with flag == NE
+ Label slow_path;
+
+ if (DiagnoseSyncOnValueBasedClasses != 0) {
+ load_klass(t1, obj);
+ ldrw(t1, Address(t1, Klass::access_flags_offset()));
+ tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
+ br(Assembler::NE, slow_path);
+ }
+
+ const Register t1_mark = t1;
+
+ { // Lightweight locking
+
+ // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
+ Label push;
+
+ const Register t2_top = t2;
+ const Register t3_t = t3;
+
+ // Check if lock-stack is full.
+ ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
+ br(Assembler::GT, slow_path);
+
+ // Check if recursive.
+ subw(t3_t, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t3_t));
+ cmp(obj, t3_t);
+ br(Assembler::EQ, push);
+
+ // Relaxed normal load to check for monitor. Optimization for monitor case.
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+
+ // Not inflated
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
+
+ // Try to lock. Transition lock-bits 0b01 => 0b00
+ orr(t1_mark, t1_mark, markWord::unlocked_value);
+ eor(t3_t, t1_mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
+ /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
+ br(Assembler::NE, slow_path);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ str(obj, Address(rthread, t2_top));
+ addw(t2_top, t2_top, oopSize);
+ strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(locked);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated);
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register t1_tagged_monitor = t1_mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+ const Register t2_owner_addr = t2;
+ const Register t3_owner = t3;
+
+ // Compute owner address.
+ lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
+
+ // CAS owner (null => current thread).
+ cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
+ /*release*/ false, /*weak*/ false, t3_owner);
+ br(Assembler::EQ, locked);
+
+ // Check if recursive.
+ cmp(t3_owner, rthread);
+ br(Assembler::NE, slow_path);
+
+ // Recursive.
+ increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
+ }
+
+ bind(locked);
+ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
+
+#ifdef ASSERT
+ // Check that locked label is reached with Flags == EQ.
+ Label flag_correct;
+ br(Assembler::EQ, flag_correct);
+ stop("Fast Lock Flag != EQ");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with Flags == NE.
+ br(Assembler::NE, flag_correct);
+ stop("Fast Lock Flag != NE");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
+}
+
+void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
+ Register t3) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert_different_registers(obj, t1, t2, t3);
+
+ // Handle inflated monitor.
+ Label inflated, inflated_load_monitor;
+ // Finish fast unlock successfully. MUST branch to with flag == EQ
+ Label unlocked;
+ // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
+ Label slow_path;
+
+ const Register t1_mark = t1;
+ const Register t2_top = t2;
+ const Register t3_t = t3;
+
+ { // Lightweight unlock
+
+ // Check if obj is top of lock-stack.
+ ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ subw(t2_top, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t2_top));
+ cmp(obj, t3_t);
+ // Top of lock stack was not obj. Must be monitor.
+ br(Assembler::NE, inflated_load_monitor);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
+ strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Check if recursive.
+ subw(t3_t, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t3_t));
+ cmp(obj, t3_t);
+ br(Assembler::EQ, unlocked);
+
+ // Not recursive.
+ // Load Mark.
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check header for monitor (0b10).
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(t3_t, t1_mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
+ /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
+ br(Assembler::EQ, unlocked);
+
+ // Compare and exchange failed.
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
+ addw(t2_top, t2_top, oopSize);
+ str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(slow_path);
+ }
+
+
+ { // Handle inflated monitor.
+ bind(inflated_load_monitor);
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+#ifdef ASSERT
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+ stop("Fast Unlock not monitor");
+#endif
+
+ bind(inflated);
+
+#ifdef ASSERT
+ Label check_done;
+ subw(t2_top, t2_top, oopSize);
+ cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
+ br(Assembler::LT, check_done);
+ ldr(t3_t, Address(rthread, t2_top));
+ cmp(obj, t3_t);
+ br(Assembler::NE, inflated);
+ stop("Fast Unlock lock on stack");
+ bind(check_done);
+#endif
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register t1_monitor = t1_mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+
+ // Untag the monitor.
+ sub(t1_monitor, t1_mark, monitor_tag);
+
+ const Register t2_recursions = t2;
+ Label not_recursive;
+
+ // Check if recursive.
+ ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
+ cbz(t2_recursions, not_recursive);
+
+ // Recursive unlock.
+ sub(t2_recursions, t2_recursions, 1u);
+ str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
+ // Set flag == EQ
+ cmp(t2_recursions, t2_recursions);
+ b(unlocked);
+
+ bind(not_recursive);
+
+ Label release;
+ const Register t2_owner_addr = t2;
+
+ // Compute owner address.
+ lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
+
+ // Check if the entry lists are empty.
+ ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
+ ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
+ orr(rscratch1, rscratch1, t3_t);
+ cmp(rscratch1, zr);
+ br(Assembler::EQ, release);
+
+ // The owner may be anonymous and we removed the last obj entry in
+ // the lock-stack. This loses the information about the owner.
+ // Write the thread to the owner field so the runtime knows the owner.
+ str(rthread, Address(t2_owner_addr));
+ b(slow_path);
+
+ bind(release);
+ // Set owner to null.
+ // Release to satisfy the JMM
+ stlr(zr, t2_owner_addr);
+ }
+
+ bind(unlocked);
+ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
+
+#ifdef ASSERT
+ // Check that unlocked label is reached with Flags == EQ.
+ Label flag_correct;
+ br(Assembler::EQ, flag_correct);
+ stop("Fast Unlock Flag != EQ");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with Flags == NE.
+ br(Assembler::NE, flag_correct);
+ stop("Fast Unlock Flag != NE");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
+}
+
// Search for str1 in str2 and return index or -1
// Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
void C2_MacroAssembler::string_indexof(Register str2, Register str1,
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
index dfa7d88cb93fe..1481f975020c9 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,9 +36,11 @@
public:
// Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
- // See full description in macroAssembler_aarch64.cpp.
void fast_lock(Register object, Register box, Register tmp, Register tmp2, Register tmp3);
void fast_unlock(Register object, Register box, Register tmp, Register tmp2);
+ // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in .ad file.
+ void fast_lock_lightweight(Register object, Register t1, Register t2, Register t3);
+ void fast_unlock_lightweight(Register object, Register t1, Register t2, Register t3);
void string_compare(Register str1, Register str2,
Register cnt1, Register cnt2, Register result,
diff --git a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
index c58ff8828bce6..23c08f11d1a8b 100644
--- a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
@@ -26,7 +26,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
@@ -36,7 +35,7 @@
// ----------------------------------------------------------------------------
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
precond(cbuf.stubs()->start() != badAddress);
precond(cbuf.stubs()->end() != badAddress);
@@ -71,11 +70,11 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
}
#undef __
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return MacroAssembler::static_call_stub_size();
}
-int CompiledStaticCall::to_trampoline_stub_size() {
+int CompiledDirectCall::to_trampoline_stub_size() {
// Somewhat pessimistically, we count 3 instructions here (although
// there are only two) because we sometimes emit an alignment nop.
// Trampoline stubs are always word aligned.
@@ -83,21 +82,14 @@ int CompiledStaticCall::to_trampoline_stub_size() {
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 4; // 3 in emit_to_interp_stub + 1 in emit_call
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder
= nativeMovConstReg_at(stub + NativeInstruction::instruction_size);
@@ -115,7 +107,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -132,7 +124,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp b/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp
index d035ab21093da..47b6f1f2f386a 100644
--- a/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2023, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -58,7 +58,7 @@ static char* reserve_at_eor_compatible_address(size_t size, bool aslr) {
0x7ffc, 0x7ffe, 0x7fff
};
static constexpr int num_immediates = sizeof(immediates) / sizeof(immediates[0]);
- const int start_index = aslr ? os::random() : 0;
+ const int start_index = aslr ? os::next_random((int)os::javaTimeNanos()) : 0;
constexpr int max_tries = 64;
for (int ntry = 0; result == nullptr && ntry < max_tries; ntry ++) {
// As in os::attempt_reserve_memory_between, we alternate between higher and lower
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.cpp b/src/hotspot/cpu/aarch64/frame_aarch64.cpp
index c5b2ff8a4c01c..8d0fa8895d15c 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -678,7 +678,7 @@ static void printbc(Method *m, intptr_t bcx) {
printf("%s : %s ==> %s\n", m->name_and_sig_as_C_string(), buf, name);
}
-void internal_pf(uintptr_t sp, uintptr_t fp, uintptr_t pc, uintptr_t bcx) {
+static void internal_pf(uintptr_t sp, uintptr_t fp, uintptr_t pc, uintptr_t bcx) {
if (! fp)
return;
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
index 42081d422c869..427987da97141 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
@@ -28,8 +28,8 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/javaThread.hpp"
diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
index b26eaa4bfcdd8..293cc6eb0d0c6 100644
--- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2019, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -117,7 +117,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Use prfm hint with specified distance in compiled code." \
"Value -1 means off.") \
range(-1, 4096) \
- product(ccstr, OnSpinWaitInst, "none", DIAGNOSTIC, \
+ product(ccstr, OnSpinWaitInst, "yield", DIAGNOSTIC, \
"The instruction to use to implement " \
"java.lang.Thread.onSpinWait()." \
"Options: none, nop, isb, yield.") \
diff --git a/src/hotspot/cpu/aarch64/icBuffer_aarch64.cpp b/src/hotspot/cpu/aarch64/icBuffer_aarch64.cpp
deleted file mode 100644
index bd8cfc42600e2..0000000000000
--- a/src/hotspot/cpu/aarch64/icBuffer_aarch64.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_aarch64.hpp"
-#include "oops/oop.inline.hpp"
-
-int InlineCacheBuffer::ic_stub_code_size() {
- return (MacroAssembler::far_branches() ? 6 : 4) * NativeInstruction::instruction_size;
-}
-
-#define __ masm->
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler* masm = new MacroAssembler(&code);
- // note: even though the code contains an embedded value, we do not need reloc info
- // because
- // (1) the value is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear
- // assert(cached_value == nullptr || cached_oop->is_perm(), "must be perm oop");
-
- address start = __ pc();
- Label l;
- __ ldr(rscratch2, l);
- int jump_code_size = __ far_jump(ExternalAddress(entry_point));
- // IC stub code size is not expected to vary depending on target address.
- // We use NOPs to make the [ldr + far_jump + nops + int64] stub size equal to ic_stub_code_size.
- for (int size = NativeInstruction::instruction_size + jump_code_size + 8;
- size < ic_stub_code_size(); size += NativeInstruction::instruction_size) {
- __ nop();
- }
- __ bind(l);
- assert((uintptr_t)__ pc() % wordSize == 0, "");
- __ emit_int64((int64_t)cached_value);
- // Only need to invalidate the 1st two instructions - not the whole ic stub
- ICache::invalidate_range(code_begin, InlineCacheBuffer::ic_stub_code_size());
- assert(__ pc() - start == ic_stub_code_size(), "must be");
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- NativeJump* jump = nativeJump_at(code_begin + 4);
- return jump->jump_destination();
-}
-
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- // The word containing the cached value is at the end of this IC buffer
- uintptr_t *p = (uintptr_t *)(code_begin + ic_stub_code_size() - wordSize);
- void* o = (void*)*p;
- return o;
-}
diff --git a/src/hotspot/cpu/aarch64/immediate_aarch64.cpp b/src/hotspot/cpu/aarch64/immediate_aarch64.cpp
index 3d87fde2b5bcd..7caafc19fbd31 100644
--- a/src/hotspot/cpu/aarch64/immediate_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/immediate_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -53,7 +53,7 @@ struct li_pair {
static struct li_pair InverseLITable[LI_TABLE_SIZE];
// comparator to sort entries in the inverse table
-int compare_immediate_pair(const void *i1, const void *i2)
+static int compare_immediate_pair(const void *i1, const void *i2)
{
struct li_pair *li1 = (struct li_pair *)i1;
struct li_pair *li2 = (struct li_pair *)i2;
@@ -142,7 +142,7 @@ static inline uint32_t uimm(uint32_t val, int hi, int lo)
// result
// a bit string containing count copies of input bit string
//
-uint64_t replicate(uint64_t bits, int nbits, int count)
+static uint64_t replicate(uint64_t bits, int nbits, int count)
{
assert(count > 0, "must be");
assert(nbits > 0, "must be");
@@ -231,8 +231,8 @@ uint64_t replicate(uint64_t bits, int nbits, int count)
// For historical reasons the implementation of this function is much
// more convoluted than is really necessary.
-int expandLogicalImmediate(uint32_t immN, uint32_t immr,
- uint32_t imms, uint64_t &bimm)
+static int expandLogicalImmediate(uint32_t immN, uint32_t immr,
+ uint32_t imms, uint64_t &bimm)
{
int len; // ought to be <= 6
uint32_t levels; // 6 bits
@@ -446,4 +446,3 @@ uint32_t encoding_for_fp_immediate(float immediate)
res = (s << 7) | (r << 4) | f;
return res;
}
-
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
index 69a61e281f352..b5625b7fc6134 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -701,7 +701,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
}
if (LockingMode == LM_LIGHTWEIGHT) {
- ldr(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
lightweight_lock(obj_reg, tmp, tmp2, tmp3, slow_case);
b(count);
} else if (LockingMode == LM_LEGACY) {
@@ -818,22 +817,6 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
if (LockingMode == LM_LIGHTWEIGHT) {
Label slow_case;
-
- // Check for non-symmetric locking. This is allowed by the spec and the interpreter
- // must handle it.
- Register tmp = rscratch1;
- // First check for lock-stack underflow.
- ldrw(tmp, Address(rthread, JavaThread::lock_stack_top_offset()));
- cmpw(tmp, (unsigned)LockStack::start_offset());
- br(Assembler::LE, slow_case);
- // Then check if the top of the lock-stack matches the unlocked object.
- subw(tmp, tmp, oopSize);
- ldr(tmp, Address(rthread, tmp));
- cmpoop(tmp, obj_reg);
- br(Assembler::NE, slow_case);
-
- ldr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- tbnz(header_reg, exact_log2(markWord::monitor_value), slow_case);
lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
b(count);
bind(slow_case);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index a3c560b28d3cf..369a50cc01fcc 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -23,12 +23,11 @@
*
*/
-#include
-
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
#include "ci/ciEnv.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/compileTask.hpp"
#include "compiler/disassembler.hpp"
#include "compiler/oopMap.hpp"
@@ -55,6 +54,7 @@
#include "runtime/jniHandles.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
@@ -66,6 +66,8 @@
#include "opto/output.hpp"
#endif
+#include
+
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
#else
@@ -390,13 +392,13 @@ static bool offset_for(uint32_t insn1, uint32_t insn2, ptrdiff_t &byte_offset) {
return false;
}
-class Decoder : public RelocActions {
- virtual reloc_insn adrpMem() { return &Decoder::adrpMem_impl; }
- virtual reloc_insn adrpAdd() { return &Decoder::adrpAdd_impl; }
- virtual reloc_insn adrpMovk() { return &Decoder::adrpMovk_impl; }
+class AArch64Decoder : public RelocActions {
+ virtual reloc_insn adrpMem() { return &AArch64Decoder::adrpMem_impl; }
+ virtual reloc_insn adrpAdd() { return &AArch64Decoder::adrpAdd_impl; }
+ virtual reloc_insn adrpMovk() { return &AArch64Decoder::adrpMovk_impl; }
public:
- Decoder(address insn_addr, uint32_t insn) : RelocActions(insn_addr, insn) {}
+ AArch64Decoder(address insn_addr, uint32_t insn) : RelocActions(insn_addr, insn) {}
virtual int loadStore(address insn_addr, address &target) {
intptr_t offset = Instruction_aarch64::sextract(_insn, 23, 5);
@@ -492,7 +494,7 @@ class Decoder : public RelocActions {
};
address MacroAssembler::target_addr_for_insn(address insn_addr, uint32_t insn) {
- Decoder decoder(insn_addr, insn);
+ AArch64Decoder decoder(insn_addr, insn);
address target;
decoder.run(insn_addr, target);
return target;
@@ -965,7 +967,7 @@ int MacroAssembler::max_trampoline_stub_size() {
}
void MacroAssembler::emit_static_call_stub() {
- // CompiledDirectStaticCall::set_to_interpreted knows the
+ // CompiledDirectCall::set_to_interpreted knows the
// exact layout of this stub.
isb();
@@ -995,10 +997,51 @@ address MacroAssembler::ic_call(address entry, jint method_index) {
// address const_ptr = long_constant((jlong)Universe::non_oop_word());
// uintptr_t offset;
// ldr_constant(rscratch2, const_ptr);
- movptr(rscratch2, (uintptr_t)Universe::non_oop_word());
+ movptr(rscratch2, (intptr_t)Universe::non_oop_word());
return trampoline_call(Address(entry, rh));
}
+int MacroAssembler::ic_check_size() {
+ if (target_needs_far_branch(CAST_FROM_FN_PTR(address, SharedRuntime::get_ic_miss_stub()))) {
+ return NativeInstruction::instruction_size * 7;
+ } else {
+ return NativeInstruction::instruction_size * 5;
+ }
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ Register receiver = j_rarg0;
+ Register data = rscratch2;
+ Register tmp1 = rscratch1;
+ Register tmp2 = r10;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, offset() + ic_check_size());
+
+ int uep_offset = offset();
+
+ if (UseCompressedClassPointers) {
+ ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+ cmpw(tmp1, tmp2);
+ } else {
+ ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+ cmp(tmp1, tmp2);
+ }
+
+ Label dont;
+ br(Assembler::EQ, dont);
+ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ bind(dont);
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
+
+ return uep_offset;
+}
+
// Implementation of call_VM versions
void MacroAssembler::call_VM(Register oop_result,
@@ -1100,7 +1143,14 @@ void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thr
}
void MacroAssembler::align(int modulus) {
- while (offset() % modulus != 0) nop();
+ align(modulus, offset());
+}
+
+// Ensure that the code at target bytes offset from the current offset() is aligned
+// according to modulus.
+void MacroAssembler::align(int modulus, int target) {
+ int delta = target - offset();
+ while ((offset() + delta) % modulus != 0) nop();
}
void MacroAssembler::post_call_nop() {
@@ -1197,7 +1247,7 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
}
// Look up the method for a megamorphic invokeinterface call in a single pass over itable:
-// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
// - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
// The target method is determined by .
// The receiver klass is in recv_klass.
@@ -4258,108 +4308,117 @@ void MacroAssembler::kernel_crc32_common_fold_using_crypto_pmull(Register crc, R
}
add(table, table, table_offset);
+ // Registers v0..v7 are used as data registers.
+ // Registers v16..v31 are used as tmp registers.
sub(buf, buf, 0x10);
- ldrq(v1, Address(buf, 0x10));
- ldrq(v2, Address(buf, 0x20));
- ldrq(v3, Address(buf, 0x30));
- ldrq(v4, Address(buf, 0x40));
- ldrq(v5, Address(buf, 0x50));
- ldrq(v6, Address(buf, 0x60));
- ldrq(v7, Address(buf, 0x70));
- ldrq(v8, Address(pre(buf, 0x80)));
-
- movi(v25, T4S, 0);
- mov(v25, S, 0, crc);
- eor(v1, T16B, v1, v25);
-
- ldrq(v0, Address(table));
+ ldrq(v0, Address(buf, 0x10));
+ ldrq(v1, Address(buf, 0x20));
+ ldrq(v2, Address(buf, 0x30));
+ ldrq(v3, Address(buf, 0x40));
+ ldrq(v4, Address(buf, 0x50));
+ ldrq(v5, Address(buf, 0x60));
+ ldrq(v6, Address(buf, 0x70));
+ ldrq(v7, Address(pre(buf, 0x80)));
+
+ movi(v31, T4S, 0);
+ mov(v31, S, 0, crc);
+ eor(v0, T16B, v0, v31);
+
+ // Register v16 contains constants from the crc table.
+ ldrq(v16, Address(table));
b(CRC_by128_loop);
align(OptoLoopAlignment);
BIND(CRC_by128_loop);
- pmull (v9, T1Q, v1, v0, T1D);
- pmull2(v10, T1Q, v1, v0, T2D);
- ldrq(v1, Address(buf, 0x10));
- eor3(v1, T16B, v9, v10, v1);
-
- pmull (v11, T1Q, v2, v0, T1D);
- pmull2(v12, T1Q, v2, v0, T2D);
- ldrq(v2, Address(buf, 0x20));
- eor3(v2, T16B, v11, v12, v2);
-
- pmull (v13, T1Q, v3, v0, T1D);
- pmull2(v14, T1Q, v3, v0, T2D);
- ldrq(v3, Address(buf, 0x30));
- eor3(v3, T16B, v13, v14, v3);
-
- pmull (v15, T1Q, v4, v0, T1D);
- pmull2(v16, T1Q, v4, v0, T2D);
- ldrq(v4, Address(buf, 0x40));
- eor3(v4, T16B, v15, v16, v4);
-
- pmull (v17, T1Q, v5, v0, T1D);
- pmull2(v18, T1Q, v5, v0, T2D);
- ldrq(v5, Address(buf, 0x50));
- eor3(v5, T16B, v17, v18, v5);
-
- pmull (v19, T1Q, v6, v0, T1D);
- pmull2(v20, T1Q, v6, v0, T2D);
- ldrq(v6, Address(buf, 0x60));
- eor3(v6, T16B, v19, v20, v6);
-
- pmull (v21, T1Q, v7, v0, T1D);
- pmull2(v22, T1Q, v7, v0, T2D);
- ldrq(v7, Address(buf, 0x70));
- eor3(v7, T16B, v21, v22, v7);
-
- pmull (v23, T1Q, v8, v0, T1D);
- pmull2(v24, T1Q, v8, v0, T2D);
- ldrq(v8, Address(pre(buf, 0x80)));
- eor3(v8, T16B, v23, v24, v8);
+ pmull (v17, T1Q, v0, v16, T1D);
+ pmull2(v18, T1Q, v0, v16, T2D);
+ ldrq(v0, Address(buf, 0x10));
+ eor3(v0, T16B, v17, v18, v0);
+
+ pmull (v19, T1Q, v1, v16, T1D);
+ pmull2(v20, T1Q, v1, v16, T2D);
+ ldrq(v1, Address(buf, 0x20));
+ eor3(v1, T16B, v19, v20, v1);
+
+ pmull (v21, T1Q, v2, v16, T1D);
+ pmull2(v22, T1Q, v2, v16, T2D);
+ ldrq(v2, Address(buf, 0x30));
+ eor3(v2, T16B, v21, v22, v2);
+
+ pmull (v23, T1Q, v3, v16, T1D);
+ pmull2(v24, T1Q, v3, v16, T2D);
+ ldrq(v3, Address(buf, 0x40));
+ eor3(v3, T16B, v23, v24, v3);
+
+ pmull (v25, T1Q, v4, v16, T1D);
+ pmull2(v26, T1Q, v4, v16, T2D);
+ ldrq(v4, Address(buf, 0x50));
+ eor3(v4, T16B, v25, v26, v4);
+
+ pmull (v27, T1Q, v5, v16, T1D);
+ pmull2(v28, T1Q, v5, v16, T2D);
+ ldrq(v5, Address(buf, 0x60));
+ eor3(v5, T16B, v27, v28, v5);
+
+ pmull (v29, T1Q, v6, v16, T1D);
+ pmull2(v30, T1Q, v6, v16, T2D);
+ ldrq(v6, Address(buf, 0x70));
+ eor3(v6, T16B, v29, v30, v6);
+
+ // Reuse registers v23, v24.
+ // Using them won't block the first instruction of the next iteration.
+ pmull (v23, T1Q, v7, v16, T1D);
+ pmull2(v24, T1Q, v7, v16, T2D);
+ ldrq(v7, Address(pre(buf, 0x80)));
+ eor3(v7, T16B, v23, v24, v7);
subs(len, len, 0x80);
br(Assembler::GE, CRC_by128_loop);
// fold into 512 bits
- ldrq(v0, Address(table, 0x10));
+ // Use v31 for constants because v16 can be still in use.
+ ldrq(v31, Address(table, 0x10));
- pmull (v10, T1Q, v1, v0, T1D);
- pmull2(v11, T1Q, v1, v0, T2D);
- eor3(v1, T16B, v10, v11, v5);
+ pmull (v17, T1Q, v0, v31, T1D);
+ pmull2(v18, T1Q, v0, v31, T2D);
+ eor3(v0, T16B, v17, v18, v4);
- pmull (v12, T1Q, v2, v0, T1D);
- pmull2(v13, T1Q, v2, v0, T2D);
- eor3(v2, T16B, v12, v13, v6);
+ pmull (v19, T1Q, v1, v31, T1D);
+ pmull2(v20, T1Q, v1, v31, T2D);
+ eor3(v1, T16B, v19, v20, v5);
- pmull (v14, T1Q, v3, v0, T1D);
- pmull2(v15, T1Q, v3, v0, T2D);
- eor3(v3, T16B, v14, v15, v7);
+ pmull (v21, T1Q, v2, v31, T1D);
+ pmull2(v22, T1Q, v2, v31, T2D);
+ eor3(v2, T16B, v21, v22, v6);
- pmull (v16, T1Q, v4, v0, T1D);
- pmull2(v17, T1Q, v4, v0, T2D);
- eor3(v4, T16B, v16, v17, v8);
+ pmull (v23, T1Q, v3, v31, T1D);
+ pmull2(v24, T1Q, v3, v31, T2D);
+ eor3(v3, T16B, v23, v24, v7);
// fold into 128 bits
- ldrq(v5, Address(table, 0x20));
- pmull (v10, T1Q, v1, v5, T1D);
- pmull2(v11, T1Q, v1, v5, T2D);
- eor3(v4, T16B, v4, v10, v11);
-
- ldrq(v6, Address(table, 0x30));
- pmull (v12, T1Q, v2, v6, T1D);
- pmull2(v13, T1Q, v2, v6, T2D);
- eor3(v4, T16B, v4, v12, v13);
-
- ldrq(v7, Address(table, 0x40));
- pmull (v14, T1Q, v3, v7, T1D);
- pmull2(v15, T1Q, v3, v7, T2D);
- eor3(v1, T16B, v4, v14, v15);
+ // Use v17 for constants because v31 can be still in use.
+ ldrq(v17, Address(table, 0x20));
+ pmull (v25, T1Q, v0, v17, T1D);
+ pmull2(v26, T1Q, v0, v17, T2D);
+ eor3(v3, T16B, v3, v25, v26);
+
+ // Use v18 for constants because v17 can be still in use.
+ ldrq(v18, Address(table, 0x30));
+ pmull (v27, T1Q, v1, v18, T1D);
+ pmull2(v28, T1Q, v1, v18, T2D);
+ eor3(v3, T16B, v3, v27, v28);
+
+ // Use v19 for constants because v18 can be still in use.
+ ldrq(v19, Address(table, 0x40));
+ pmull (v29, T1Q, v2, v19, T1D);
+ pmull2(v30, T1Q, v2, v19, T2D);
+ eor3(v0, T16B, v3, v29, v30);
add(len, len, 0x80);
add(buf, buf, 0x10);
- mov(tmp0, v1, D, 0);
- mov(tmp1, v1, D, 1);
+ mov(tmp0, v0, D, 0);
+ mov(tmp1, v0, D, 1);
}
SkipIfEqual::SkipIfEqual(
@@ -5339,28 +5398,25 @@ address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
// For Strings we're passed the address of the first characters in a1
// and a2 and the length in cnt1.
-// elem_size is the element size in bytes: either 1 or 2.
// There are two implementations. For arrays >= 8 bytes, all
// comparisons (including the final one, which may overlap) are
// performed 8 bytes at a time. For strings < 8 bytes, we compare a
// halfword, then a short, and then a byte.
void MacroAssembler::string_equals(Register a1, Register a2,
- Register result, Register cnt1, int elem_size)
+ Register result, Register cnt1)
{
Label SAME, DONE, SHORT, NEXT_WORD;
Register tmp1 = rscratch1;
Register tmp2 = rscratch2;
Register cnt2 = tmp2; // cnt2 only used in array length compare
- assert(elem_size == 1 || elem_size == 2, "must be 2 or 1 byte");
assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2);
#ifndef PRODUCT
{
- const char kind = (elem_size == 2) ? 'U' : 'L';
char comment[64];
- snprintf(comment, sizeof comment, "{string_equals%c", kind);
+ snprintf(comment, sizeof comment, "{string_equalsL");
BLOCK_COMMENT(comment);
}
#endif
@@ -5408,14 +5464,12 @@ void MacroAssembler::string_equals(Register a1, Register a2,
cbnzw(tmp1, DONE);
}
bind(TAIL01);
- if (elem_size == 1) { // Only needed when comparing 1-byte elements
- tbz(cnt1, 0, SAME); // 0-1 bytes left.
+ tbz(cnt1, 0, SAME); // 0-1 bytes left.
{
- ldrb(tmp1, a1);
- ldrb(tmp2, a2);
- eorw(tmp1, tmp1, tmp2);
- cbnzw(tmp1, DONE);
- }
+ ldrb(tmp1, a1);
+ ldrb(tmp2, a2);
+ eorw(tmp1, tmp1, tmp2);
+ cbnzw(tmp1, DONE);
}
// Arrays are equal.
bind(SAME);
@@ -6328,97 +6382,122 @@ void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
}
// Implements lightweight-locking.
-// Branches to slow upon failure to lock the object, with ZF cleared.
-// Falls through upon success with ZF set.
//
// - obj: the object to be locked
-// - hdr: the header, already loaded from obj, will be destroyed
-// - t1, t2: temporary registers, will be destroyed
-void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+// - t1, t2, t3: temporary registers, will be destroyed
+// - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding).
+void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, t1, t2, rscratch1);
-
- // Check if we would have space on lock-stack for the object.
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- cmpw(t1, (unsigned)LockStack::end_offset() - 1);
- br(Assembler::GT, slow);
-
- // Load (object->mark() | 1) into hdr
- orr(hdr, hdr, markWord::unlocked_value);
- // Clear lock-bits, into t2
- eor(t2, hdr, markWord::unlocked_value);
- // Try to swing header from unlocked to locked
- // Clobbers rscratch1 when UseLSE is false
- cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
- /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
+ assert_different_registers(obj, t1, t2, t3, rscratch1);
+
+ Label push;
+ const Register top = t1;
+ const Register mark = t2;
+ const Register t = t3;
+
+ // Preload the markWord. It is important that this is the first
+ // instruction emitted as it is part of C1's null check semantics.
+ ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check if the lock-stack is full.
+ ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ cmpw(top, (unsigned)LockStack::end_offset());
+ br(Assembler::GE, slow);
+
+ // Check for recursion.
+ subw(t, top, oopSize);
+ ldr(t, Address(rthread, t));
+ cmp(obj, t);
+ br(Assembler::EQ, push);
+
+ // Check header for monitor (0b10).
+ tst(mark, markWord::monitor_value);
+ br(Assembler::NE, slow);
+
+ // Try to lock. Transition lock bits 0b01 => 0b00
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(mark, mark, markWord::unlocked_value);
+ eor(t, mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword,
+ /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
br(Assembler::NE, slow);
- // After successful lock, push object on lock-stack
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- str(obj, Address(rthread, t1));
- addw(t1, t1, oopSize);
- strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ str(obj, Address(rthread, top));
+ addw(top, top, oopSize);
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
}
// Implements lightweight-unlocking.
-// Branches to slow upon failure, with ZF cleared.
-// Falls through upon success, with ZF set.
//
// - obj: the object to be unlocked
-// - hdr: the (pre-loaded) header of the object
-// - t1, t2: temporary registers
-void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+// - t1, t2, t3: temporary registers
+// - slow: branched to if unlocking fails, absolute offset may larger than 32KB (imm14 encoding).
+void MacroAssembler::lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, t1, t2, rscratch1);
+ // cmpxchg clobbers rscratch1.
+ assert_different_registers(obj, t1, t2, t3, rscratch1);
#ifdef ASSERT
{
- // The following checks rely on the fact that LockStack is only ever modified by
- // its owning thread, even if the lock got inflated concurrently; removal of LockStack
- // entries after inflation will happen delayed in that case.
-
// Check for lock-stack underflow.
Label stack_ok;
ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
cmpw(t1, (unsigned)LockStack::start_offset());
- br(Assembler::GT, stack_ok);
+ br(Assembler::GE, stack_ok);
STOP("Lock-stack underflow");
bind(stack_ok);
}
- {
- // Check if the top of the lock-stack matches the unlocked object.
- Label tos_ok;
- subw(t1, t1, oopSize);
- ldr(t1, Address(rthread, t1));
- cmpoop(t1, obj);
- br(Assembler::EQ, tos_ok);
- STOP("Top of lock-stack does not match the unlocked object");
- bind(tos_ok);
- }
- {
- // Check that hdr is fast-locked.
- Label hdr_ok;
- tst(hdr, markWord::lock_mask_in_place);
- br(Assembler::EQ, hdr_ok);
- STOP("Header is not fast-locked");
- bind(hdr_ok);
- }
#endif
- // Load the new header (unlocked) into t1
- orr(t1, hdr, markWord::unlocked_value);
+ Label unlocked, push_and_slow;
+ const Register top = t1;
+ const Register mark = t2;
+ const Register t = t3;
- // Try to swing header from locked to unlocked
- // Clobbers rscratch1 when UseLSE is false
- cmpxchg(obj, hdr, t1, Assembler::xword,
- /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
+ // Check if obj is top of lock-stack.
+ ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ subw(top, top, oopSize);
+ ldr(t, Address(rthread, top));
+ cmp(obj, t);
br(Assembler::NE, slow);
- // After successful unlock, pop object from lock-stack
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- subw(t1, t1, oopSize);
+ // Pop lock-stack.
+ DEBUG_ONLY(str(zr, Address(rthread, top));)
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Check if recursive.
+ subw(t, top, oopSize);
+ ldr(t, Address(rthread, t));
+ cmp(obj, t);
+ br(Assembler::EQ, unlocked);
+
+ // Not recursive. Check header for monitor (0b10).
+ ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ tbnz(mark, log2i_exact(markWord::monitor_value), push_and_slow);
+
#ifdef ASSERT
- str(zr, Address(rthread, t1));
+ // Check header not unlocked (0b01).
+ Label not_unlocked;
+ tbz(mark, log2i_exact(markWord::unlocked_value), not_unlocked);
+ stop("lightweight_unlock already unlocked");
+ bind(not_unlocked);
#endif
- strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(t, mark, markWord::unlocked_value);
+ cmpxchg(obj, mark, t, Assembler::xword,
+ /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
+ br(Assembler::EQ, unlocked);
+
+ bind(push_and_slow);
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(str(obj, Address(rthread, top));)
+ addw(top, top, oopSize);
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(slow);
+
+ bind(unlocked);
}
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index 84931c409cfe9..dad7ec4d4975e 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -720,6 +720,7 @@ class MacroAssembler: public Assembler {
// Alignment
void align(int modulus);
+ void align(int modulus, int target);
// nop
void post_call_nop();
@@ -1247,6 +1248,8 @@ class MacroAssembler: public Assembler {
// Emit the CompiledIC call idiom
address ic_call(address entry, jint method_index = 0);
+ static int ic_check_size();
+ int ic_check(int end_alignment);
public:
@@ -1399,8 +1402,7 @@ class MacroAssembler: public Assembler {
address arrays_equals(Register a1, Register a2, Register result, Register cnt1,
Register tmp1, Register tmp2, Register tmp3, int elem_size);
- void string_equals(Register a1, Register a2, Register result, Register cnt1,
- int elem_size);
+ void string_equals(Register a1, Register a2, Register result, Register cnt1);
void fill_words(Register base, Register cnt, Register value);
address zero_words(Register base, uint64_t cnt);
@@ -1425,11 +1427,6 @@ class MacroAssembler: public Assembler {
FloatRegister vtmp2, FloatRegister vtmp3,
FloatRegister vtmp4, FloatRegister vtmp5);
- void fast_log(FloatRegister vtmp0, FloatRegister vtmp1, FloatRegister vtmp2,
- FloatRegister vtmp3, FloatRegister vtmp4, FloatRegister vtmp5,
- FloatRegister tmpC1, FloatRegister tmpC2, FloatRegister tmpC3,
- FloatRegister tmpC4, Register tmp1, Register tmp2,
- Register tmp3, Register tmp4, Register tmp5);
void generate_dsin_dcos(bool isCos, address npio2_hw, address two_over_pi,
address pio2, address dsin_coef, address dcos_coef);
private:
@@ -1605,8 +1602,8 @@ class MacroAssembler: public Assembler {
// Code for java.lang.Thread::onSpinWait() intrinsic.
void spin_wait();
- void lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow);
- void lightweight_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow);
+ void lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow);
+ void lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow);
private:
// Check the current thread doesn't need a cross modify fence.
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64_log.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64_log.cpp
deleted file mode 100644
index 45772ff1afd63..0000000000000
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64_log.cpp
+++ /dev/null
@@ -1,366 +0,0 @@
-/* Copyright (c) 2018, Cavium. All rights reserved. (By BELLSOFT)
- * Copyright (c) 2016, 2021, Intel Corporation. All rights reserved.
- * Intel Math Library (LIBM) Source Code
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/assembler.hpp"
-#include "asm/assembler.inline.hpp"
-#include "macroAssembler_aarch64.hpp"
-
-// Algorithm idea is taken from x86 hotspot intrinsic and adapted for AARCH64.
-//
-// For mathematical background please refer to the following literature:
-//
-// Tang, Ping-Tak Peter.
-// Table-driven implementation of the logarithm function
-// in IEEE floating-point arithmetic.
-// ACM Transactions on Mathematical Software (TOMS) 16, no. 4, 1990: 378-400.
-
-/******************************************************************************/
-// ALGORITHM DESCRIPTION - LOG()
-// ---------------------
-//
-// x=2^k * mx, mx in [1,2)
-//
-// Get B~1/mx based on the output of frecpe instruction (B0)
-// B = int((B0*2^7+0.5))/2^7
-//
-// Reduced argument: r=B*mx-1.0 (computed accurately in high and low parts)
-//
-// Result: k*log(2) - log(B) + p(r) if |x-1| >= small value (2^-6) and
-// p(r) is a degree 7 polynomial
-// -log(B) read from data table (high, low parts)
-// Result is formed from high and low parts
-//
-// Special cases:
-// 1. log(NaN) = quiet NaN
-// 2. log(+INF) = that INF
-// 3. log(0) = -INF
-// 4. log(1) = +0
-// 5. log(x) = NaN if x < -0, including -INF
-//
-/******************************************************************************/
-
-// Table with p(r) polynomial coefficients
-// and table representation of logarithm values (hi and low parts)
-ATTRIBUTE_ALIGNED(64) juint _L_tbl[] =
-{
- // coefficients of p(r) polynomial:
- // _coeff[]
- 0x00000000UL, 0xbfd00000UL, // C1_0 = -0.25
- 0x92492492UL, 0x3fc24924UL, // C1_1 = 0.14285714285714285
- 0x55555555UL, 0x3fd55555UL, // C2_0 = 0.3333333333333333
- 0x3d6fb175UL, 0xbfc5555eUL, // C2_1 = -0.16666772842235003
- 0x00000000UL, 0xbfe00000UL, // C3_0 = -0.5
- 0x9999999aUL, 0x3fc99999UL, // C3_1 = 0.2
- // _log2[]
- 0xfefa3800UL, 0x3fa62e42UL, // C4_0 = 0.043321698784993146
- 0x93c76730UL, 0x3ceef357UL, // C4_1 = 3.436201886692732e-15
- // _L_tbl[] with logarithm values (hi and low parts)
- 0xfefa3800UL, 0x3fe62e42UL, 0x93c76730UL, 0x3d2ef357UL, 0xaa241800UL,
- 0x3fe5ee82UL, 0x0cda46beUL, 0x3d220238UL, 0x5c364800UL, 0x3fe5af40UL,
- 0xac10c9fbUL, 0x3d2dfa63UL, 0x26bb8c00UL, 0x3fe5707aUL, 0xff3303ddUL,
- 0x3d09980bUL, 0x26867800UL, 0x3fe5322eUL, 0x5d257531UL, 0x3d05ccc4UL,
- 0x835a5000UL, 0x3fe4f45aUL, 0x6d93b8fbUL, 0xbd2e6c51UL, 0x6f970c00UL,
- 0x3fe4b6fdUL, 0xed4c541cUL, 0x3cef7115UL, 0x27e8a400UL, 0x3fe47a15UL,
- 0xf94d60aaUL, 0xbd22cb6aUL, 0xf2f92400UL, 0x3fe43d9fUL, 0x481051f7UL,
- 0xbcfd984fUL, 0x2125cc00UL, 0x3fe4019cUL, 0x30f0c74cUL, 0xbd26ce79UL,
- 0x0c36c000UL, 0x3fe3c608UL, 0x7cfe13c2UL, 0xbd02b736UL, 0x17197800UL,
- 0x3fe38ae2UL, 0xbb5569a4UL, 0xbd218b7aUL, 0xad9d8c00UL, 0x3fe35028UL,
- 0x9527e6acUL, 0x3d10b83fUL, 0x44340800UL, 0x3fe315daUL, 0xc5a0ed9cUL,
- 0xbd274e93UL, 0x57b0e000UL, 0x3fe2dbf5UL, 0x07b9dc11UL, 0xbd17a6e5UL,
- 0x6d0ec000UL, 0x3fe2a278UL, 0xe797882dUL, 0x3d206d2bUL, 0x1134dc00UL,
- 0x3fe26962UL, 0x05226250UL, 0xbd0b61f1UL, 0xd8bebc00UL, 0x3fe230b0UL,
- 0x6e48667bUL, 0x3d12fc06UL, 0x5fc61800UL, 0x3fe1f863UL, 0xc9fe81d3UL,
- 0xbd2a7242UL, 0x49ae6000UL, 0x3fe1c078UL, 0xed70e667UL, 0x3cccacdeUL,
- 0x40f23c00UL, 0x3fe188eeUL, 0xf8ab4650UL, 0x3d14cc4eUL, 0xf6f29800UL,
- 0x3fe151c3UL, 0xa293ae49UL, 0xbd2edd97UL, 0x23c75c00UL, 0x3fe11af8UL,
- 0xbb9ddcb2UL, 0xbd258647UL, 0x8611cc00UL, 0x3fe0e489UL, 0x07801742UL,
- 0x3d1c2998UL, 0xe2d05400UL, 0x3fe0ae76UL, 0x887e7e27UL, 0x3d1f486bUL,
- 0x0533c400UL, 0x3fe078bfUL, 0x41edf5fdUL, 0x3d268122UL, 0xbe760400UL,
- 0x3fe04360UL, 0xe79539e0UL, 0xbd04c45fUL, 0xe5b20800UL, 0x3fe00e5aUL,
- 0xb1727b1cUL, 0xbd053ba3UL, 0xaf7a4800UL, 0x3fdfb358UL, 0x3c164935UL,
- 0x3d0085faUL, 0xee031800UL, 0x3fdf4aa7UL, 0x6f014a8bUL, 0x3d12cde5UL,
- 0x56b41000UL, 0x3fdee2a1UL, 0x5a470251UL, 0x3d2f27f4UL, 0xc3ddb000UL,
- 0x3fde7b42UL, 0x5372bd08UL, 0xbd246550UL, 0x1a272800UL, 0x3fde148aUL,
- 0x07322938UL, 0xbd1326b2UL, 0x484c9800UL, 0x3fddae75UL, 0x60dc616aUL,
- 0xbd1ea42dUL, 0x46def800UL, 0x3fdd4902UL, 0xe9a767a8UL, 0x3d235bafUL,
- 0x18064800UL, 0x3fdce42fUL, 0x3ec7a6b0UL, 0xbd0797c3UL, 0xc7455800UL,
- 0x3fdc7ff9UL, 0xc15249aeUL, 0xbd29b6ddUL, 0x693fa000UL, 0x3fdc1c60UL,
- 0x7fe8e180UL, 0x3d2cec80UL, 0x1b80e000UL, 0x3fdbb961UL, 0xf40a666dUL,
- 0x3d27d85bUL, 0x04462800UL, 0x3fdb56faUL, 0x2d841995UL, 0x3d109525UL,
- 0x5248d000UL, 0x3fdaf529UL, 0x52774458UL, 0xbd217cc5UL, 0x3c8ad800UL,
- 0x3fda93edUL, 0xbea77a5dUL, 0x3d1e36f2UL, 0x0224f800UL, 0x3fda3344UL,
- 0x7f9d79f5UL, 0x3d23c645UL, 0xea15f000UL, 0x3fd9d32bUL, 0x10d0c0b0UL,
- 0xbd26279eUL, 0x43135800UL, 0x3fd973a3UL, 0xa502d9f0UL, 0xbd152313UL,
- 0x635bf800UL, 0x3fd914a8UL, 0x2ee6307dUL, 0xbd1766b5UL, 0xa88b3000UL,
- 0x3fd8b639UL, 0xe5e70470UL, 0xbd205ae1UL, 0x776dc800UL, 0x3fd85855UL,
- 0x3333778aUL, 0x3d2fd56fUL, 0x3bd81800UL, 0x3fd7fafaUL, 0xc812566aUL,
- 0xbd272090UL, 0x687cf800UL, 0x3fd79e26UL, 0x2efd1778UL, 0x3d29ec7dUL,
- 0x76c67800UL, 0x3fd741d8UL, 0x49dc60b3UL, 0x3d2d8b09UL, 0xe6af1800UL,
- 0x3fd6e60eUL, 0x7c222d87UL, 0x3d172165UL, 0x3e9c6800UL, 0x3fd68ac8UL,
- 0x2756eba0UL, 0x3d20a0d3UL, 0x0b3ab000UL, 0x3fd63003UL, 0xe731ae00UL,
- 0xbd2db623UL, 0xdf596000UL, 0x3fd5d5bdUL, 0x08a465dcUL, 0xbd0a0b2aUL,
- 0x53c8d000UL, 0x3fd57bf7UL, 0xee5d40efUL, 0x3d1fadedUL, 0x0738a000UL,
- 0x3fd522aeUL, 0x8164c759UL, 0x3d2ebe70UL, 0x9e173000UL, 0x3fd4c9e0UL,
- 0x1b0ad8a4UL, 0xbd2e2089UL, 0xc271c800UL, 0x3fd4718dUL, 0x0967d675UL,
- 0xbd2f27ceUL, 0x23d5e800UL, 0x3fd419b4UL, 0xec90e09dUL, 0x3d08e436UL,
- 0x77333000UL, 0x3fd3c252UL, 0xb606bd5cUL, 0x3d183b54UL, 0x76be1000UL,
- 0x3fd36b67UL, 0xb0f177c8UL, 0x3d116ecdUL, 0xe1d36000UL, 0x3fd314f1UL,
- 0xd3213cb8UL, 0xbd28e27aUL, 0x7cdc9000UL, 0x3fd2bef0UL, 0x4a5004f4UL,
- 0x3d2a9cfaUL, 0x1134d800UL, 0x3fd26962UL, 0xdf5bb3b6UL, 0x3d2c93c1UL,
- 0x6d0eb800UL, 0x3fd21445UL, 0xba46baeaUL, 0x3d0a87deUL, 0x635a6800UL,
- 0x3fd1bf99UL, 0x5147bdb7UL, 0x3d2ca6edUL, 0xcbacf800UL, 0x3fd16b5cUL,
- 0xf7a51681UL, 0x3d2b9acdUL, 0x8227e800UL, 0x3fd1178eUL, 0x63a5f01cUL,
- 0xbd2c210eUL, 0x67616000UL, 0x3fd0c42dUL, 0x163ceae9UL, 0x3d27188bUL,
- 0x604d5800UL, 0x3fd07138UL, 0x16ed4e91UL, 0x3cf89cdbUL, 0x5626c800UL,
- 0x3fd01eaeUL, 0x1485e94aUL, 0xbd16f08cUL, 0x6cb3b000UL, 0x3fcf991cUL,
- 0xca0cdf30UL, 0x3d1bcbecUL, 0xe4dd0000UL, 0x3fcef5adUL, 0x65bb8e11UL,
- 0xbcca2115UL, 0xffe71000UL, 0x3fce530eUL, 0x6041f430UL, 0x3cc21227UL,
- 0xb0d49000UL, 0x3fcdb13dUL, 0xf715b035UL, 0xbd2aff2aUL, 0xf2656000UL,
- 0x3fcd1037UL, 0x75b6f6e4UL, 0xbd084a7eUL, 0xc6f01000UL, 0x3fcc6ffbUL,
- 0xc5962bd2UL, 0xbcf1ec72UL, 0x383be000UL, 0x3fcbd087UL, 0x595412b6UL,
- 0xbd2d4bc4UL, 0x575bd000UL, 0x3fcb31d8UL, 0x4eace1aaUL, 0xbd0c358dUL,
- 0x3c8ae000UL, 0x3fca93edUL, 0x50562169UL, 0xbd287243UL, 0x07089000UL,
- 0x3fc9f6c4UL, 0x6865817aUL, 0x3d29904dUL, 0xdcf70000UL, 0x3fc95a5aUL,
- 0x58a0ff6fUL, 0x3d07f228UL, 0xeb390000UL, 0x3fc8beafUL, 0xaae92cd1UL,
- 0xbd073d54UL, 0x6551a000UL, 0x3fc823c1UL, 0x9a631e83UL, 0x3d1e0ddbUL,
- 0x85445000UL, 0x3fc7898dUL, 0x70914305UL, 0xbd1c6610UL, 0x8b757000UL,
- 0x3fc6f012UL, 0xe59c21e1UL, 0xbd25118dUL, 0xbe8c1000UL, 0x3fc6574eUL,
- 0x2c3c2e78UL, 0x3d19cf8bUL, 0x6b544000UL, 0x3fc5bf40UL, 0xeb68981cUL,
- 0xbd127023UL, 0xe4a1b000UL, 0x3fc527e5UL, 0xe5697dc7UL, 0x3d2633e8UL,
- 0x8333b000UL, 0x3fc4913dUL, 0x54fdb678UL, 0x3d258379UL, 0xa5993000UL,
- 0x3fc3fb45UL, 0x7e6a354dUL, 0xbd2cd1d8UL, 0xb0159000UL, 0x3fc365fcUL,
- 0x234b7289UL, 0x3cc62fa8UL, 0x0c868000UL, 0x3fc2d161UL, 0xcb81b4a1UL,
- 0x3d039d6cUL, 0x2a49c000UL, 0x3fc23d71UL, 0x8fd3df5cUL, 0x3d100d23UL,
- 0x7e23f000UL, 0x3fc1aa2bUL, 0x44389934UL, 0x3d2ca78eUL, 0x8227e000UL,
- 0x3fc1178eUL, 0xce2d07f2UL, 0x3d21ef78UL, 0xb59e4000UL, 0x3fc08598UL,
- 0x7009902cUL, 0xbd27e5ddUL, 0x39dbe000UL, 0x3fbfe891UL, 0x4fa10afdUL,
- 0xbd2534d6UL, 0x830a2000UL, 0x3fbec739UL, 0xafe645e0UL, 0xbd2dc068UL,
- 0x63844000UL, 0x3fbda727UL, 0x1fa71733UL, 0x3d1a8940UL, 0x01bc4000UL,
- 0x3fbc8858UL, 0xc65aacd3UL, 0x3d2646d1UL, 0x8dad6000UL, 0x3fbb6ac8UL,
- 0x2bf768e5UL, 0xbd139080UL, 0x40b1c000UL, 0x3fba4e76UL, 0xb94407c8UL,
- 0xbd0e42b6UL, 0x5d594000UL, 0x3fb9335eUL, 0x3abd47daUL, 0x3d23115cUL,
- 0x2f40e000UL, 0x3fb8197eUL, 0xf96ffdf7UL, 0x3d0f80dcUL, 0x0aeac000UL,
- 0x3fb700d3UL, 0xa99ded32UL, 0x3cec1e8dUL, 0x4d97a000UL, 0x3fb5e95aUL,
- 0x3c5d1d1eUL, 0xbd2c6906UL, 0x5d208000UL, 0x3fb4d311UL, 0x82f4e1efUL,
- 0xbcf53a25UL, 0xa7d1e000UL, 0x3fb3bdf5UL, 0xa5db4ed7UL, 0x3d2cc85eUL,
- 0xa4472000UL, 0x3fb2aa04UL, 0xae9c697dUL, 0xbd20b6e8UL, 0xd1466000UL,
- 0x3fb1973bUL, 0x560d9e9bUL, 0xbd25325dUL, 0xb59e4000UL, 0x3fb08598UL,
- 0x7009902cUL, 0xbd17e5ddUL, 0xc006c000UL, 0x3faeea31UL, 0x4fc93b7bUL,
- 0xbd0e113eUL, 0xcdddc000UL, 0x3faccb73UL, 0x47d82807UL, 0xbd1a68f2UL,
- 0xd0fb0000UL, 0x3faaaef2UL, 0x353bb42eUL, 0x3d20fc1aUL, 0x149fc000UL,
- 0x3fa894aaUL, 0xd05a267dUL, 0xbd197995UL, 0xf2d4c000UL, 0x3fa67c94UL,
- 0xec19afa2UL, 0xbd029efbUL, 0xd42e0000UL, 0x3fa466aeUL, 0x75bdfd28UL,
- 0xbd2c1673UL, 0x2f8d0000UL, 0x3fa252f3UL, 0xe021b67bUL, 0x3d283e9aUL,
- 0x89e74000UL, 0x3fa0415dUL, 0x5cf1d753UL, 0x3d0111c0UL, 0xec148000UL,
- 0x3f9c63d2UL, 0x3f9eb2f3UL, 0x3d2578c6UL, 0x28c90000UL, 0x3f984925UL,
- 0x325a0c34UL, 0xbd2aa0baUL, 0x25980000UL, 0x3f9432a9UL, 0x928637feUL,
- 0x3d098139UL, 0x58938000UL, 0x3f902056UL, 0x06e2f7d2UL, 0xbd23dc5bUL,
- 0xa3890000UL, 0x3f882448UL, 0xda74f640UL, 0xbd275577UL, 0x75890000UL,
- 0x3f801015UL, 0x999d2be8UL, 0xbd10c76bUL, 0x59580000UL, 0x3f700805UL,
- 0xcb31c67bUL, 0x3d2166afUL, 0x00000000UL, 0x00000000UL, 0x00000000UL,
- 0x80000000UL
-};
-
-// BEGIN dlog PSEUDO CODE:
-// double dlog(double X) {
-// // p(r) polynomial coefficients initialized from _L_tbl table
-// double C1_0 = _L_tbl[0];
-// double C1_1 = _L_tbl[1];
-// double C2_0 = _L_tbl[2];
-// double C2_1 = _L_tbl[3];
-// double C3_0 = _L_tbl[4];
-// double C3_1 = _L_tbl[5];
-// double C4_0 = _L_tbl[6];
-// double C4_1 = _L_tbl[7];
-// // NOTE: operations with coefficients above are mostly vectorized in assembly
-// // Check corner cases first
-// if (X == 1.0d || AS_LONG_BITS(X) + 0x0010000000000000 <= 0x0010000000000000) {
-// // NOTE: AS_LONG_BITS(X) + 0x0010000000000000 <= 0x0010000000000000 means
-// // that X < 0 or X >= 0x7FF0000000000000 (0x7FF* is NaN or INF)
-// if (X < 0 || X is NaN) return NaN;
-// if (X == 1.0d) return 0.0d;
-// if (X == 0.0d) return -INFINITY;
-// if (X is INFINITY) return INFINITY;
-// }
-// // double representation is 2^exponent * mantissa
-// // split X into two multipliers: 2^exponent and 1.0 * mantissa
-// // pseudo function: zeroExponent(X) return value of X with exponent == 0
-// float vtmp5 = 1/(float)(zeroExponent(X)); // reciprocal estimate
-// // pseudo function: HI16(X) returns high 16 bits of double value
-// int hiWord = HI16(X);
-// double vtmp1 = (double) 0x77F0 << 48 | mantissa(X);
-// hiWord -= 16;
-// if (AS_LONG_BITS(hiWord) > 0x8000) {
-// // SMALL_VALUE branch
-// vtmp0 = vtmp1 = vtmp0 * AS_DOUBLE_BITS(0x47F0000000000000);
-// hiWord = HI16(vtmp1);
-// vtmp0 = AS_DOUBLE_BITS(AS_LONG_BITS(vtmp0) |= 0x3FF0000000000000);
-// vtmp5 = (double) (1/(float)vtmp0);
-// vtmp1 <<= 12;
-// vtmp1 >>= 12;
-// }
-// // MAIN branch
-// double vtmp3 = AS_LONG_BITS(vtmp1) & 0xffffe00000000000; // hi part
-// int intB0 = AS_INT_BITS(vtmp5) + 0x8000;
-// double vtmp0 = AS_DOUBLE_BITS(0xffffe00000000000 & (intB0<<29));
-// int index = (intB0 >> 16) && 0xFF;
-// double hiTableValue = _L_tbl[8+index]; // vtmp2[0]
-// double lowTableValue = _L_tbl[16+index]; // vtmp2[1]
-// vtmp5 = AS_DOUBLE_BITS(hiWord & 0x7FF0 - 0x3FE0); // 0x3FE = 1023 << 4
-// vtmp1 -= vtmp3; // low part
-// vtmp3 = vtmp3*vtmp0 - 1.0;
-// hiTableValue += C4_0 * vtmp5;
-// lowTableValue += C4_1 * vtmp5;
-// double r = vtmp1 * vtmp0 + vtmp3; // r = B*mx-1.0, computed in hi and low parts
-// vtmp0 = hiTableValue + r;
-// hiTableValue -= vtmp0;
-// double r2 = r*r;
-// double r3 = r2*r;
-// double p7 = C3_0*r2 + C2_0*r3 + C1_0*r2*r2 + C3_1*r3*r2 + C2_1*r3*r3
-// + C1_1*r3*r2*r2; // degree 7 polynomial
-// return p7 + (vtmp0 + ((r + hiTableValue) + lowTableValue));
-// }
-//
-// END dlog PSEUDO CODE
-
-
-// Generate log(X). X passed in register v0. Return log(X) into v0.
-// Generator parameters: 10 temporary FPU registers and temporary general
-// purpose registers
-void MacroAssembler::fast_log(FloatRegister vtmp0, FloatRegister vtmp1,
- FloatRegister vtmp2, FloatRegister vtmp3,
- FloatRegister vtmp4, FloatRegister vtmp5,
- FloatRegister C1, FloatRegister C2,
- FloatRegister C3, FloatRegister C4,
- Register tmp1, Register tmp2, Register tmp3,
- Register tmp4, Register tmp5) {
- Label DONE, CHECK_CORNER_CASES, SMALL_VALUE, MAIN,
- CHECKED_CORNER_CASES, RETURN_MINF_OR_NAN;
- const int64_t INF_OR_NAN_PREFIX = 0x7FF0;
- const int64_t MINF_OR_MNAN_PREFIX = 0xFFF0;
- const int64_t ONE_PREFIX = 0x3FF0;
- movz(tmp2, ONE_PREFIX, 48);
- movz(tmp4, 0x0010, 48);
- fmovd(rscratch1, v0); // rscratch1 = AS_LONG_BITS(X)
- lea(rscratch2, ExternalAddress((address)_L_tbl));
- movz(tmp5, 0x7F);
- add(tmp1, rscratch1, tmp4);
- cmp(tmp2, rscratch1);
- lsr(tmp3, rscratch1, 29);
- ccmp(tmp1, tmp4, 0b1101 /* LE */, NE);
- bfm(tmp3, tmp5, 41, 8);
- fmovs(vtmp5, tmp3);
- // Load coefficients from table. All coefficients are organized to be
- // in specific order, because load below will load it in vectors to be used
- // later in vector instructions. Load will be performed in parallel while
- // branches are taken. C1 will contain vector of {C1_0, C1_1}, C2 =
- // {C2_0, C2_1}, C3 = {C3_0, C3_1}, C4 = {C4_0, C4_1}
- ld1(C1, C2, C3, C4, T2D, post(rscratch2, 64));
- br(LE, CHECK_CORNER_CASES);
- bind(CHECKED_CORNER_CASES);
- // all corner cases are handled
- frecpe(vtmp5, vtmp5, S); // vtmp5 ~= 1/vtmp5
- lsr(tmp2, rscratch1, 48);
- movz(tmp4, 0x77f0, 48);
- fmovd(vtmp4, 1.0);
- movz(tmp1, INF_OR_NAN_PREFIX, 48);
- bfm(tmp4, rscratch1, 0, 51); // tmp4 = 0x77F0 << 48 | mantissa(X)
- // vtmp1 = AS_DOUBLE_BITS(0x77F0 << 48 | mantissa(X)) == mx
- fmovd(vtmp1, tmp4);
- subw(tmp2, tmp2, 16);
- subs(zr, tmp2, 0x8000);
- br(GE, SMALL_VALUE);
- bind(MAIN);
- fmovs(tmp3, vtmp5); // int intB0 = AS_INT_BITS(B);
- mov(tmp5, 0x3FE0);
- uint64_t mask = UCONST64(0xffffe00000000000);
- mov(rscratch1, mask);
- andr(tmp2, tmp2, tmp1, LSR, 48); // hiWord & 0x7FF0
- sub(tmp2, tmp2, tmp5); // tmp2 = hiWord & 0x7FF0 - 0x3FE0
- scvtfwd(vtmp5, tmp2); // vtmp5 = (double)tmp2;
- addw(tmp3, tmp3, 0x8000); // tmp3 = B
- andr(tmp4, tmp4, rscratch1); // tmp4 == hi_part(mx)
- andr(rscratch1, rscratch1, tmp3, LSL, 29); // rscratch1 = hi_part(B)
- ubfm(tmp3, tmp3, 16, 23); // int index = (intB0 >> 16) && 0xFF
- ldrq(vtmp2, Address(rscratch2, tmp3, Address::lsl(4))); // vtmp2 = _L_tbl[index]
- // AS_LONG_BITS(vtmp1) & 0xffffe00000000000 // hi_part(mx)
- fmovd(vtmp3, tmp4);
- fmovd(vtmp0, rscratch1); // vtmp0 = hi_part(B)
- fsubd(vtmp1, vtmp1, vtmp3); // vtmp1 -= vtmp3; // low_part(mx)
- fnmsub(vtmp3, vtmp3, vtmp0, vtmp4); // vtmp3 = vtmp3*vtmp0 - vtmp4
- fmlavs(vtmp2, T2D, C4, vtmp5, 0); // vtmp2 += {C4} * vtmp5
- // vtmp1 = r = vtmp1 * vtmp0 + vtmp3 == low_part(mx) * hi_part(B) + (hi_part(mx)*hi_part(B) - 1.0)
- fmaddd(vtmp1, vtmp1, vtmp0, vtmp3);
- ins(vtmp5, D, vtmp2, 0, 1); // vtmp5 = vtmp2[1];
- faddd(vtmp0, vtmp2, vtmp1); // vtmp0 = vtmp2 + vtmp1
- fmlavs(C3, T2D, C2, vtmp1, 0); // {C3} += {C2}*vtmp1
- fsubd(vtmp2, vtmp2, vtmp0); // vtmp2 -= vtmp0
- fmuld(vtmp3, vtmp1, vtmp1); // vtmp3 = vtmp1*vtmp1
- faddd(C4, vtmp1, vtmp2); // C4[0] = vtmp1 + vtmp2
- fmlavs(C3, T2D, C1, vtmp3, 0); // {C3} += {C1}*vtmp3
- faddd(C4, C4, vtmp5); // C4 += vtmp5
- fmuld(vtmp4, vtmp3, vtmp1); // vtmp4 = vtmp3*vtmp1
- faddd(vtmp0, vtmp0, C4); // vtmp0 += C4
- fmlavs(C3, T2D, vtmp4, C3, 1); // {C3} += {vtmp4}*C3[1]
- fmaddd(vtmp0, C3, vtmp3, vtmp0); // vtmp0 = C3 * vtmp3 + vtmp0
- ret(lr);
-
- block_comment("if (AS_LONG_BITS(hiWord) > 0x8000)"); {
- bind(SMALL_VALUE);
- movz(tmp2, 0x47F0, 48);
- fmovd(vtmp1, tmp2);
- fmuld(vtmp0, vtmp1, v0);
- fmovd(vtmp1, vtmp0);
- umov(tmp2, vtmp1, S, 3);
- orr(vtmp0, T16B, vtmp0, vtmp4);
- ushr(vtmp5, T2D, vtmp0, 27);
- ushr(vtmp5, T4S, vtmp5, 2);
- frecpe(vtmp5, vtmp5, S);
- shl(vtmp1, T2D, vtmp1, 12);
- ushr(vtmp1, T2D, vtmp1, 12);
- b(MAIN);
- }
-
- block_comment("Corner cases"); {
- bind(RETURN_MINF_OR_NAN);
- movz(tmp1, MINF_OR_MNAN_PREFIX, 48);
- orr(rscratch1, rscratch1, tmp1);
- fmovd(v0, rscratch1);
- ret(lr);
- bind(CHECK_CORNER_CASES);
- movz(tmp1, INF_OR_NAN_PREFIX, 48);
- cmp(rscratch1, zr);
- br(LE, RETURN_MINF_OR_NAN);
- cmp(rscratch1, tmp1);
- br(GE, DONE);
- cmp(rscratch1, tmp2);
- br(NE, CHECKED_CORNER_CASES);
- fmovd(v0, 0.0);
- }
- bind(DONE);
- ret(lr);
-}
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
index a12caa3daeefb..192a0d1ac1c9e 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
@@ -28,7 +28,6 @@
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
#include "gc/shared/collectedHeap.hpp"
-#include "memory/resourceArea.hpp"
#include "nativeInst_aarch64.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/handles.hpp"
@@ -189,8 +188,6 @@ void NativeCall::set_destination_mt_safe(address dest, bool assert_lock) {
CompiledICLocker::is_safe(addr_at(0)),
"concurrent code patching");
- ResourceMark rm;
- int code_size = NativeInstruction::instruction_size;
address addr_call = addr_at(0);
bool reachable = Assembler::reachable_from_branch_at(addr_call, dest);
assert(NativeCall::is_call_at(addr_call), "unexpected code at call site");
diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
index 8694734c751c6..97a10afde7ab2 100644
--- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -30,7 +30,6 @@
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -39,7 +38,6 @@
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_aarch64.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "oops/method.inline.hpp"
#include "prims/methodHandles.hpp"
@@ -740,9 +738,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
address c2i_unverified_entry = __ pc();
Label skip_fixup;
- Label ok;
-
- Register holder = rscratch2;
+ Register data = rscratch2;
Register receiver = j_rarg0;
Register tmp = r10; // A call-clobbered register not used for arg passing
@@ -757,17 +753,12 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
{
__ block_comment("c2i_unverified_entry {");
- __ load_klass(rscratch1, receiver);
- __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
- __ cmp(rscratch1, tmp);
- __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
- __ br(Assembler::EQ, ok);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- __ bind(ok);
// Method might have been compiled since the call site was patched to
// interpreted; if that is the case treat it as a miss so we can get
// the call site corrected.
+ __ ic_check(1 /* end_alignment */);
+ __ ldr(rmethod, Address(data, CompiledICData::speculated_method_offset()));
+
__ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
__ cbz(rscratch1, skip_fixup);
__ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
@@ -1118,7 +1109,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ b(exit);
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, tr_call);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1183,7 +1174,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
}
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, tr_call);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1391,6 +1382,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
in_ByteSize(-1),
oop_maps,
exception_offset);
+ if (nm == nullptr) return nm;
if (method->is_continuation_enter_intrinsic()) {
ContinuationEntry::set_enter_code(nm, interpreted_entry_offset);
} else if (method->is_continuation_yield_intrinsic()) {
@@ -1538,25 +1530,15 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// restoring them except rfp. rfp is the only callee save register
// as far as the interpreter and the compiler(s) are concerned.
-
- const Register ic_reg = rscratch2;
const Register receiver = j_rarg0;
- Label hit;
Label exception_pending;
- assert_different_registers(ic_reg, receiver, rscratch1);
+ assert_different_registers(receiver, rscratch1);
__ verify_oop(receiver);
- __ cmp_klass(receiver, ic_reg, rscratch1);
- __ br(Assembler::EQ, hit);
-
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ __ ic_check(8 /* end_alignment */);
// Verified entry point must be aligned
- __ align(8);
-
- __ bind(hit);
-
int vep_offset = ((intptr_t)__ pc()) - start;
// If we have to make this method not-entrant we'll overwrite its
@@ -1814,7 +1796,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ br(Assembler::NE, slow_path_lock);
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- __ ldr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ lightweight_lock(obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock);
}
__ bind(count);
@@ -1957,8 +1938,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "");
- __ ldr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- __ tbnz(old_hdr, exact_log2(markWord::monitor_value), slow_path_unlock);
__ lightweight_unlock(obj_reg, old_hdr, swap_reg, lock_tmp, slow_path_unlock);
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
}
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index 97ca90ac76454..46a7d79626749 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2022, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -5327,19 +5327,6 @@ class StubGenerator: public StubCodeGenerator {
return start;
}
- address generate_dlog() {
- __ align(CodeEntryAlignment);
- StubCodeMark mark(this, "StubRoutines", "dlog");
- address entry = __ pc();
- FloatRegister vtmp0 = v0, vtmp1 = v1, vtmp2 = v2, vtmp3 = v3, vtmp4 = v4,
- vtmp5 = v5, tmpC1 = v16, tmpC2 = v17, tmpC3 = v18, tmpC4 = v19;
- Register tmp1 = r0, tmp2 = r1, tmp3 = r2, tmp4 = r3, tmp5 = r4;
- __ fast_log(vtmp0, vtmp1, vtmp2, vtmp3, vtmp4, vtmp5, tmpC1, tmpC2, tmpC3,
- tmpC4, tmp1, tmp2, tmp3, tmp4, tmp5);
- return entry;
- }
-
-
// code for comparing 16 characters of strings with Latin1 and Utf16 encoding
void compare_string_16_x_LU(Register tmpL, Register tmpU, Label &DIFF1,
Label &DIFF2) {
@@ -5487,6 +5474,32 @@ class StubGenerator: public StubCodeGenerator {
return entry;
}
+ // r0 = input (float16)
+ // v0 = result (float)
+ // v1 = temporary float register
+ address generate_float16ToFloat() {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", "float16ToFloat");
+ address entry = __ pc();
+ BLOCK_COMMENT("Entry:");
+ __ flt16_to_flt(v0, r0, v1);
+ __ ret(lr);
+ return entry;
+ }
+
+ // v0 = input (float)
+ // r0 = result (float16)
+ // v1 = temporary float register
+ address generate_floatToFloat16() {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", "floatToFloat16");
+ address entry = __ pc();
+ BLOCK_COMMENT("Entry:");
+ __ flt_to_flt16(r0, v0, v1);
+ __ ret(lr);
+ return entry;
+ }
+
address generate_method_entry_barrier() {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
@@ -8333,11 +8346,6 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
}
- // Disabled until JDK-8210858 is fixed
- // if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
- // StubRoutines::_dlog = generate_dlog();
- // }
-
if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
}
@@ -8345,6 +8353,12 @@ class StubGenerator: public StubCodeGenerator {
if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
}
+
+ if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
+ vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
+ StubRoutines::_hf2f = generate_float16ToFloat();
+ StubRoutines::_f2hf = generate_floatToFloat16();
+ }
}
void generate_continuation_stubs() {
diff --git a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
index 14e9764457508..28ec07815be5c 100644
--- a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
@@ -245,9 +245,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ mov_metadata(rmethod, entry);
__ str(rmethod, Address(rthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+ __ push_cont_fastpath(rthread);
+
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ blr(rscratch1);
+ __ pop_cont_fastpath(rthread);
+
// return value shuffle
if (!needs_return_buffer) {
#ifdef ASSERT
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index f7fe2f7dec81a..18f310c746cd4 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -143,11 +143,19 @@ void VM_Version::initialize() {
}
}
- // Ampere CPUs: Ampere-1 and Ampere-1A
- if (_cpu == CPU_AMPERE && ((_model == CPU_MODEL_AMPERE_1) || (_model == CPU_MODEL_AMPERE_1A))) {
+ // Ampere CPUs
+ if (_cpu == CPU_AMPERE && ((_model == CPU_MODEL_AMPERE_1) ||
+ (_model == CPU_MODEL_AMPERE_1A) ||
+ (_model == CPU_MODEL_AMPERE_1B))) {
if (FLAG_IS_DEFAULT(UseSIMDForMemoryOps)) {
FLAG_SET_DEFAULT(UseSIMDForMemoryOps, true);
}
+ if (FLAG_IS_DEFAULT(OnSpinWaitInst)) {
+ FLAG_SET_DEFAULT(OnSpinWaitInst, "isb");
+ }
+ if (FLAG_IS_DEFAULT(OnSpinWaitInstCount)) {
+ FLAG_SET_DEFAULT(OnSpinWaitInstCount, 2);
+ }
}
// ThunderX
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
index 4b2e5cc5a4ddd..6883dc0d93e16 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -110,7 +110,8 @@ enum Ampere_CPU_Model {
CPU_MODEL_ALTRA = 0xd0c, /* CPU implementer is CPU_ARM, Neoverse N1 */
CPU_MODEL_ALTRAMAX = 0xd0c, /* CPU implementer is CPU_ARM, Neoverse N1 */
CPU_MODEL_AMPERE_1 = 0xac3, /* CPU implementer is CPU_AMPERE */
- CPU_MODEL_AMPERE_1A = 0xac4 /* CPU implementer is CPU_AMPERE */
+ CPU_MODEL_AMPERE_1A = 0xac4, /* CPU implementer is CPU_AMPERE */
+ CPU_MODEL_AMPERE_1B = 0xac5 /* AMPERE_1B core Implements ARMv8.7 with CSSC, MTE, SM3/SM4 extensions */
};
#define CPU_FEATURE_FLAGS(decl) \
@@ -168,6 +169,7 @@ enum Ampere_CPU_Model {
// Aarch64 supports fast class initialization checks
static bool supports_fast_class_init_checks() { return true; }
constexpr static bool supports_stack_watermark_barrier() { return true; }
+ constexpr static bool supports_recursive_lightweight_locking() { return true; }
static void get_compatible_board(char *buf, int buflen);
diff --git a/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
index 6e345c96ce20c..2bb53d16a3c97 100644
--- a/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
@@ -26,10 +26,10 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_aarch64.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -168,22 +168,22 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// Entry arguments:
- // rscratch2: CompiledICHolder
+ // rscratch2: CompiledICData
// j_rarg0: Receiver
// This stub is called from compiled code which has no callee-saved registers,
// so all registers except arguments are free at this point.
const Register recv_klass_reg = r10;
- const Register holder_klass_reg = r16; // declaring interface klass (DECC)
+ const Register holder_klass_reg = r16; // declaring interface klass (DEFC)
const Register resolved_klass_reg = r17; // resolved interface klass (REFC)
const Register temp_reg = r11;
const Register temp_reg2 = r15;
- const Register icholder_reg = rscratch2;
+ const Register icdata_reg = rscratch2;
Label L_no_such_interface;
- __ ldr(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
- __ ldr(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
+ __ ldr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+ __ ldr(holder_klass_reg, Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
start_pc = __ pc();
@@ -197,7 +197,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
temp_reg, temp_reg2, itable_index, L_no_such_interface);
// Reduce "estimate" such that "padding" does not drop below 8.
- const ptrdiff_t estimate = 124;
+ const ptrdiff_t estimate = 144;
const ptrdiff_t codesize = __ pc() - start_pc;
slop_delta = (int)(estimate - codesize);
slop_bytes += slop_delta;
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index e31ad91613a1a..1a833b08c4cf4 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -195,7 +195,7 @@ void emit_call_reloc(CodeBuffer &cbuf, const MachCallNode *n, MachOper *m, Reloc
assert(maybe_far_call(n) == !__ reachable_from_cache(target), "sanity");
assert(cache_reachable() == __ cache_fully_reachable(), "sanity");
- assert(target != NULL, "need real address");
+ assert(target != nullptr, "need real address");
int ret_addr_offset = -1;
if (rspec.type() == relocInfo::runtime_call_type) {
@@ -290,7 +290,7 @@ void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
st->print ("SUB R_SP, R_SP, " SIZE_FORMAT,framesize);
}
- if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
+ if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
st->print("ldr t0, [guard]\n\t");
st->print("ldr t1, [Rthread, #thread_disarmed_guard_value_offset]\n\t");
st->print("cmp t0, t1\n\t");
@@ -332,7 +332,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
__ sub_slow(SP, SP, framesize);
}
- if (C->stub_function() == NULL) {
+ if (C->stub_function() == nullptr) {
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->nmethod_entry_barrier(&_masm);
}
@@ -454,7 +454,7 @@ uint MachSpillCopyNode::implementation( CodeBuffer *cbuf,
return size; // Self copy, no move
#ifdef TODO
- if (bottom_type()->isa_vect() != NULL) {
+ if (bottom_type()->isa_vect() != nullptr) {
}
#endif
@@ -804,16 +804,16 @@ uint MachSpillCopyNode::implementation( CodeBuffer *cbuf,
#ifndef PRODUCT
void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
- implementation( NULL, ra_, false, st );
+ implementation(nullptr, ra_, false, st );
}
#endif
void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- implementation( &cbuf, ra_, false, NULL );
+ implementation( &cbuf, ra_, false, nullptr );
}
uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
- return implementation( NULL, ra_, true, NULL );
+ return implementation( nullptr, ra_, true, nullptr );
}
//=============================================================================
@@ -869,12 +869,7 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
#define R_RTEMP "R_R12"
void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
st->print_cr("\nUEP:");
- if (UseCompressedClassPointers) {
- st->print_cr("\tLDR_w " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
- st->print_cr("\tdecode_klass " R_RTEMP);
- } else {
- st->print_cr("\tLDR " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
- }
+ st->print_cr("\tLDR " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
st->print_cr("\tCMP " R_RTEMP ",R_R8" );
st->print ("\tB.NE SharedRuntime::handle_ic_miss_stub");
}
@@ -882,13 +877,7 @@ void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
C2_MacroAssembler _masm(&cbuf);
- Register iCache = reg_to_register_object(Matcher::inline_cache_reg_encode());
- assert(iCache == Ricklass, "should be");
- Register receiver = R0;
-
- __ load_klass(Rtemp, receiver);
- __ cmp(Rtemp, iCache);
- __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
+ __ ic_check(InteriorEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
@@ -903,7 +892,7 @@ int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_exception_handler());
- if (base == NULL) {
+ if (base == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -926,7 +915,7 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_deopt_handler());
- if (base == NULL) {
+ if (base == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -1002,7 +991,7 @@ bool Matcher::match_rule_supported(int opcode) {
return true; // Per default match rules are supported.
}
-bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
return match_rule_supported_vector(opcode, vlen, bt);
}
@@ -1026,11 +1015,11 @@ bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
}
const RegMask* Matcher::predicate_reg_mask(void) {
- return NULL;
+ return nullptr;
}
const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return NULL;
+ return nullptr;
}
// Vector calling convention not yet implemented.
@@ -1074,7 +1063,7 @@ int Matcher::min_vector_size(const BasicType bt) {
return 8/type2aelembytes(bt);
}
-int Matcher::superword_max_vector_size(const BasicType bt) {
+int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
return Matcher::max_vector_size(bt);
}
@@ -1094,7 +1083,7 @@ bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
ShouldNotReachHere(); // generic vector operands not supported
- return NULL;
+ return nullptr;
}
bool Matcher::is_reg2reg_move(MachNode* m) {
@@ -1241,8 +1230,8 @@ encode %{
emit_call_reloc(cbuf, as_MachCall(), $meth, rspec);
// Emit stubs for static call.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
- if (stub == NULL) {
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf);
+ if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -1987,7 +1976,7 @@ operand immNKlass()
interface(CONST_INTER);
%}
-// NULL Pointer Immediate
+// Null Pointer Immediate
operand immN0()
%{
predicate(n->get_narrowcon() == 0);
diff --git a/src/hotspot/cpu/arm/arm_32.ad b/src/hotspot/cpu/arm/arm_32.ad
index affe5a816fcaf..dd7d6f491dad4 100644
--- a/src/hotspot/cpu/arm/arm_32.ad
+++ b/src/hotspot/cpu/arm/arm_32.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -431,7 +431,7 @@ OptoRegPair c2::return_value(int ideal_reg) {
// will point.
int MachCallStaticJavaNode::ret_addr_offset() {
- bool far = (_method == NULL) ? maybe_far_call(this) : !cache_reachable();
+ bool far = (_method == nullptr) ? maybe_far_call(this) : !cache_reachable();
return ((far ? 3 : 1) + (_method_handle_invoke ? 1 : 0)) *
NativeInstruction::instruction_size;
}
diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
index 999309c02258d..688790f07e548 100644
--- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
@@ -161,10 +161,7 @@ void LIR_Assembler::osr_entry() {
int LIR_Assembler::check_icache() {
- Register receiver = LIR_Assembler::receiverOpr()->as_register();
- int offset = __ offset();
- __ inline_cache_check(receiver, Ricklass);
- return offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -971,7 +968,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->tmp1()->as_register(),
op->tmp2()->as_register(),
op->tmp3()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -1950,7 +1947,7 @@ void LIR_Assembler::emit_static_call_stub() {
__ relocate(static_stub_Relocation::spec(call_pc));
// If not a single instruction, NativeMovConstReg::next_instruction_address()
// must jump over the whole following ldr_literal.
- // (See CompiledStaticCall::set_to_interpreted())
+ // (See CompiledDirectCall::set_to_interpreted())
#ifdef ASSERT
address ldr_site = __ pc();
#endif
diff --git a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
index c09e54e0e57ad..d9d042bb2e4e7 100644
--- a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
@@ -43,16 +43,6 @@
// arm [macro]assembler) and used with care in the other C1 specific
// files.
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- Label verified;
- load_klass(Rtemp, receiver);
- cmp(Rtemp, iCache);
- b(verified, eq); // jump over alignment no-ops
- jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type);
- align(CodeEntryAlignment);
- bind(verified);
-}
-
void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
assert((frame_size_in_bytes % StackAlignmentInBytes) == 0, "frame size should be aligned");
diff --git a/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp b/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
index 62faa6170833b..9862a074a687f 100644
--- a/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
@@ -37,7 +37,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_arm.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_arm.hpp"
diff --git a/src/hotspot/cpu/arm/compiledIC_arm.cpp b/src/hotspot/cpu/arm/compiledIC_arm.cpp
index 2d4187b7d6c6a..71389d2353d66 100644
--- a/src/hotspot/cpu/arm/compiledIC_arm.cpp
+++ b/src/hotspot/cpu/arm/compiledIC_arm.cpp
@@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/nmethod.hpp"
#include "logging/log.hpp"
@@ -37,7 +36,7 @@
#if COMPILER2_OR_JVMCI
#define __ _masm.
// emit call stub, compiled java to interpreter
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
// Stub is fixed up when the corresponding call is converted from calling
// compiled code to calling interpreted code.
// set (empty), R9
@@ -59,7 +58,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
InlinedMetadata object_literal(nullptr);
// single instruction, see NativeMovConstReg::next_instruction_address() in
- // CompiledStaticCall::set_to_interpreted()
+ // CompiledDirectCall::set_to_interpreted()
__ ldr_literal(Rmethod, object_literal);
__ set_inst_mark(); // Who uses this?
@@ -87,32 +86,25 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
#undef __
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 10; // 4 in emit_to_interp_stub + 1 in Java_Static_Call
}
#endif // COMPILER2_OR_JVMCI
-int CompiledStaticCall::to_trampoline_stub_size() {
+int CompiledDirectCall::to_trampoline_stub_size() {
// ARM doesn't use trampolines.
return 0;
}
// size of C2 call stub, compiled java to interpreter
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return 8 * NativeInstruction::instruction_size;
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder = nativeMovConstReg_at(stub);
NativeJump* jump = nativeJump_at(method_holder->next_instruction_address());
@@ -128,7 +120,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -144,7 +136,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
index 3aa71dca8cbf0..6d724c750aa34 100644
--- a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
@@ -29,8 +29,8 @@
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/sharedRuntime.hpp"
diff --git a/src/hotspot/cpu/arm/icBuffer_arm.cpp b/src/hotspot/cpu/arm/icBuffer_arm.cpp
deleted file mode 100644
index e3a1c148ec6a0..0000000000000
--- a/src/hotspot/cpu/arm/icBuffer_arm.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/assembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_arm.hpp"
-#include "oops/oop.inline.hpp"
-
-#define __ masm->
-
-int InlineCacheBuffer::ic_stub_code_size() {
- return (4 * Assembler::InstructionSize);
-}
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler* masm = new MacroAssembler(&code);
-
- InlinedAddress oop_literal((address) cached_value);
- __ ldr_literal(Ricklass, oop_literal);
- // FIXME: OK to remove reloc here?
- __ patchable_jump(entry_point, relocInfo::runtime_call_type, Rtemp);
- __ bind_literal(oop_literal);
- __ flush();
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- address jump_address;
- jump_address = code_begin + NativeInstruction::instruction_size;
- NativeJump* jump = nativeJump_at(jump_address);
- return jump->jump_destination();
-}
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin);
- return (void*)move->data();
-}
-
-#undef __
diff --git a/src/hotspot/cpu/arm/macroAssembler_arm.cpp b/src/hotspot/cpu/arm/macroAssembler_arm.cpp
index b827e69d02233..99d619bddb55a 100644
--- a/src/hotspot/cpu/arm/macroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/macroAssembler_arm.cpp
@@ -28,6 +28,7 @@
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.hpp"
#include "ci/ciEnv.hpp"
+#include "code/compiledIC.hpp"
#include "code/nativeInst.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/barrierSet.hpp"
@@ -297,11 +298,13 @@ Address MacroAssembler::receiver_argument_address(Register params_base, Register
return Address(tmp, -Interpreter::stackElementSize);
}
+void MacroAssembler::align(int modulus, int target) {
+ int delta = target - offset();
+ while ((offset() + delta) % modulus != 0) nop();
+}
void MacroAssembler::align(int modulus) {
- while (offset() % modulus != 0) {
- nop();
- }
+ align(modulus, offset());
}
int MacroAssembler::set_last_Java_frame(Register last_java_sp,
@@ -1860,3 +1863,31 @@ void MacroAssembler::lightweight_unlock(Register obj, Register t1, Register t2,
// Fallthrough: success
}
+
+int MacroAssembler::ic_check_size() {
+ return NativeInstruction::instruction_size * 7;
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ Register receiver = j_rarg0;
+ Register tmp1 = R4;
+ Register tmp2 = R5;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, offset() + ic_check_size());
+
+ int uep_offset = offset();
+
+ ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ ldr(tmp2, Address(Ricklass, CompiledICData::speculated_klass_offset()));
+ cmp(tmp1, tmp2);
+
+ Label dont;
+ b(dont, eq);
+ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type);
+ bind(dont);
+ return uep_offset;
+}
diff --git a/src/hotspot/cpu/arm/macroAssembler_arm.hpp b/src/hotspot/cpu/arm/macroAssembler_arm.hpp
index d9e49ab986c3a..691c8fa70ee8b 100644
--- a/src/hotspot/cpu/arm/macroAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/macroAssembler_arm.hpp
@@ -221,6 +221,7 @@ class MacroAssembler: public Assembler {
inline bool ignore_non_patchable_relocations() { return true; }
void align(int modulus);
+ void align(int modulus, int target);
// Support for VM calls
//
@@ -1077,6 +1078,9 @@ class MacroAssembler: public Assembler {
void safepoint_poll(Register tmp1, Label& slow_path);
void get_polling_page(Register dest);
void read_polling_page(Register dest, relocInfo::relocType rtype);
+
+ static int ic_check_size();
+ int ic_check(int end_alignment);
};
diff --git a/src/hotspot/cpu/arm/nativeInst_arm_32.cpp b/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
index 23ee01d335264..6a4062f29b3ba 100644
--- a/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
+++ b/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
@@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_arm.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
index 7006d7709813a..15b57188730df 100644
--- a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
+++ b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
@@ -385,7 +385,7 @@ class NativeMovConstReg: public NativeInstruction {
}
void set_pc_relative_offset(address addr, address pc);
address next_instruction_address() const {
- // NOTE: CompiledStaticCall::set_to_interpreted() calls this but
+ // NOTE: CompiledDirectCall::set_to_interpreted() calls this but
// are restricted to single-instruction ldr. No need to jump over
// several instructions.
assert(is_ldr_literal(), "Should only use single-instructions load");
diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
index 716c7b7575e9c..3792fab082ba6 100644
--- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
+++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
@@ -24,15 +24,14 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "interpreter/interpreter.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/jniHandles.hpp"
@@ -626,12 +625,9 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
Label skip_fixup;
const Register receiver = R0;
const Register holder_klass = Rtemp; // XXX should be OK for C2 but not 100% sure
- const Register receiver_klass = R4;
- __ load_klass(receiver_klass, receiver);
- __ ldr(holder_klass, Address(Ricklass, CompiledICHolder::holder_klass_offset()));
- __ ldr(Rmethod, Address(Ricklass, CompiledICHolder::holder_metadata_offset()));
- __ cmp(receiver_klass, holder_klass);
+ __ ic_check(1 /* end_alignment */);
+ __ ldr(Rmethod, Address(Ricklass, CompiledICData::speculated_method_offset()));
__ ldr(Rtemp, Address(Rmethod, Method::code_offset()), eq);
__ cmp(Rtemp, 0, eq);
@@ -819,21 +815,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// Unverified entry point
address start = __ pc();
- // Inline cache check, same as in C1_MacroAssembler::inline_cache_check()
const Register receiver = R0; // see receiverOpr()
- __ load_klass(Rtemp, receiver);
- __ cmp(Rtemp, Ricklass);
- Label verified;
-
- __ b(verified, eq); // jump over alignment no-ops too
- __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
- __ align(CodeEntryAlignment);
+ __ verify_oop(receiver);
+ // Inline cache check
+ __ ic_check(CodeEntryAlignment /* end_alignment */);
// Verified entry point
- __ bind(verified);
int vep_offset = __ pc() - start;
-
if ((InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) || (method->intrinsic_id() == vmIntrinsics::_identityHashCode)) {
// Object.hashCode, System.identityHashCode can pull the hashCode from the header word
// instead of doing a full VM transition once it's been computed.
diff --git a/src/hotspot/cpu/arm/vtableStubs_arm.cpp b/src/hotspot/cpu/arm/vtableStubs_arm.cpp
index 539e288f63fb2..1229b5073f506 100644
--- a/src/hotspot/cpu/arm/vtableStubs_arm.cpp
+++ b/src/hotspot/cpu/arm/vtableStubs_arm.cpp
@@ -25,10 +25,10 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_arm.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "oops/klass.inline.hpp"
@@ -160,7 +160,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
__ load_klass(Rclass, R0);
// Receiver subtype check against REFC.
- __ ldr(Rintf, Address(Ricklass, CompiledICHolder::holder_klass_offset()));
+ __ ldr(Rintf, Address(Ricklass, CompiledICData::itable_refc_klass_offset()));
__ lookup_interface_method(// inputs: rec. class, interface, itable index
Rclass, Rintf, noreg,
// outputs: temp reg1, temp reg2
@@ -171,7 +171,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
start_pc = __ pc();
// Get Method* and entry point for compiler
- __ ldr(Rintf, Address(Ricklass, CompiledICHolder::holder_metadata_offset()));
+ __ ldr(Rintf, Address(Ricklass, CompiledICData::itable_defc_klass_offset()));
__ lookup_interface_method(// inputs: rec. class, interface, itable index
Rclass, Rintf, itable_index,
// outputs: temp reg1, temp reg2, temp reg3
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 47b681ce26be4..d78dec964cbb0 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -451,7 +451,7 @@ inline void Assembler::bcctrl(int boint, int biint, int bhint, relocInfo::relocT
// helper function for b
inline bool Assembler::is_within_range_of_b(address a, address pc) {
- // Guard against illegal branch targets, e.g. -1 (see CompiledStaticCall and ad-file).
+ // Guard against illegal branch targets, e.g. -1 (see CompiledDirectCall and ad-file).
if ((((uint64_t)a) & 0x3) != 0) return false;
const int range = 1 << (29-6); // li field is from bit 6 to bit 29.
@@ -465,7 +465,7 @@ inline bool Assembler::is_within_range_of_b(address a, address pc) {
// helper functions for bcxx.
inline bool Assembler::is_within_range_of_bcxx(address a, address pc) {
- // Guard against illegal branch targets, e.g. -1 (see CompiledStaticCall and ad-file).
+ // Guard against illegal branch targets, e.g. -1 (see CompiledDirectCall and ad-file).
if ((((uint64_t)a) & 0x3) != 0) return false;
const int range = 1 << (29-16); // bd field is from bit 16 to bit 29.
diff --git a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
index 1a00c9ad268c7..dc70c73d4b330 100644
--- a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
@@ -456,6 +456,9 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
__ extsw(R7_ARG5, length()->as_register());
ce->emit_static_call_stub();
+ if (ce->compilation()->bailed_out()) {
+ return; // CodeCache is full
+ }
bool success = ce->emit_trampoline_stub_for_call(SharedRuntime::get_resolve_static_call_stub());
if (!success) { return; }
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
index 9913a589683a4..3ae35949b2148 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -77,9 +77,7 @@ int LIR_Assembler::initial_frame_size_in_bytes() const {
// we fetch the class of the receiver and compare it with the cached class.
// If they do not match we jump to slow case.
int LIR_Assembler::check_icache() {
- int offset = __ offset();
- __ inline_cache_check(R3_ARG1, R19_inline_cache_reg);
- return offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -2300,7 +2298,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->tmp1()->as_register(),
op->tmp2()->as_register(),
op->tmp3()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -2862,6 +2860,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest,
if (info != nullptr) {
add_call_info_here(info);
}
+ assert(__ last_calls_return_pc() == __ pc(), "pcn not at return pc");
__ post_call_nop();
}
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
index 577dcae25f4bc..219aeaf316d87 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2018 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -40,29 +40,6 @@
#include "utilities/macros.hpp"
#include "utilities/powerOfTwo.hpp"
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- const Register temp_reg = R12_scratch2;
- Label Lmiss;
-
- verify_oop(receiver, FILE_AND_LINE);
- load_klass_check_null(temp_reg, receiver, &Lmiss);
-
- if (TrapBasedICMissChecks && TrapBasedNullChecks) {
- trap_ic_miss_check(temp_reg, iCache);
- } else {
- Label Lok;
- cmpd(CCR0, temp_reg, iCache);
- beq(CCR0, Lok);
- bind(Lmiss);
- //load_const_optimized(temp_reg, SharedRuntime::get_ic_miss_stub(), R0);
- calculate_address_from_global_toc(temp_reg, SharedRuntime::get_ic_miss_stub(), true, true, false);
- mtctr(temp_reg);
- bctr();
- align(32, 12);
- bind(Lok);
- }
-}
-
void C1_MacroAssembler::explicit_null_check(Register base) {
Unimplemented();
@@ -178,9 +155,6 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb
verify_oop(Roop, FILE_AND_LINE);
if (LockingMode == LM_LIGHTWEIGHT) {
- ld(Rmark, oopDesc::mark_offset_in_bytes(), Roop);
- andi_(R0, Rmark, markWord::monitor_value);
- bne(CCR0, slow_int);
lightweight_unlock(Roop, Rmark, slow_int);
} else if (LockingMode == LM_LEGACY) {
// Check if it is still a light weight lock, this is is true if we see
@@ -333,7 +307,7 @@ void C1_MacroAssembler::allocate_array(
Register t1, // temp register
Register t2, // temp register
Register t3, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // continuation point if fast allocation fails
@@ -365,7 +339,7 @@ void C1_MacroAssembler::allocate_array(
sldi(t1, len, log2_elt_size);
arr_len_in_bytes = t1;
}
- addi(arr_size, arr_len_in_bytes, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
+ addi(arr_size, arr_len_in_bytes, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
clrrdi(arr_size, arr_size, LogMinObjAlignmentInBytes); // Align array size.
// Allocate space & initialize header.
@@ -375,8 +349,18 @@ void C1_MacroAssembler::allocate_array(
// Initialize body.
const Register base = t2;
const Register index = t3;
- addi(base, obj, hdr_size * wordSize); // compute address of first element
- addi(index, arr_size, -(hdr_size * wordSize)); // compute index = number of bytes to clear
+ addi(base, obj, base_offset_in_bytes); // compute address of first element
+ addi(index, arr_size, -(base_offset_in_bytes)); // compute index = number of bytes to clear
+
+ // Zero first 4 bytes, if start offset is not word aligned.
+ if (!is_aligned(base_offset_in_bytes, BytesPerWord)) {
+ assert(is_aligned(base_offset_in_bytes, BytesPerInt), "must be 4-byte aligned");
+ li(t1, 0);
+ stw(t1, 0, base);
+ addi(base, base, BytesPerInt);
+ // Note: initialize_body will align index down, no need to correct it here.
+ }
+
initialize_body(base, index);
if (CURRENT_ENV->dtrace_alloc_probes()) {
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp
index 5fa19d5fd5dad..ab31431e67d9c 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2015 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -80,7 +80,7 @@
Register t1, // temp register
Register t2, // temp register
Register t3, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // continuation point if fast allocation fails
diff --git a/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp b/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp
index 2ba6a6bca4e03..63914c5d1cb93 100644
--- a/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp
@@ -34,7 +34,6 @@
#include "gc/shared/cardTableBarrierSet.hpp"
#include "interpreter/interpreter.hpp"
#include "nativeInst_ppc.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_ppc.hpp"
diff --git a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
index d504c71e2b8bc..cc69c0abe361f 100644
--- a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
@@ -36,6 +36,17 @@
#endif
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
+
+void C2_MacroAssembler::fast_lock_lightweight(ConditionRegister flag, Register obj, Register box,
+ Register tmp1, Register tmp2, Register tmp3) {
+ compiler_fast_lock_lightweight_object(flag, obj, tmp1, tmp2, tmp3);
+}
+
+void C2_MacroAssembler::fast_unlock_lightweight(ConditionRegister flag, Register obj, Register box,
+ Register tmp1, Register tmp2, Register tmp3) {
+ compiler_fast_unlock_lightweight_object(flag, obj, tmp1, tmp2, tmp3);
+}
+
// Intrinsics for CompactStrings
// Compress char[] to byte[] by compressing 16 bytes at once.
diff --git a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp
index ef4840b08a256..5096810ef9110 100644
--- a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp
@@ -28,6 +28,12 @@
// C2_MacroAssembler contains high-level macros for C2
public:
+ // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in .ad file.
+ void fast_lock_lightweight(ConditionRegister flag, Register obj, Register box,
+ Register tmp1, Register tmp2, Register tmp3);
+ void fast_unlock_lightweight(ConditionRegister flag, Register obj, Register box,
+ Register tmp1, Register tmp2, Register tmp3);
+
// Intrinsics for CompactStrings
// Compress char[] to byte[] by compressing 16 bytes at once.
void string_compress_16(Register src, Register dst, Register cnt,
diff --git a/src/hotspot/cpu/ppc/compiledIC_ppc.cpp b/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
index 54f9cfa936797..355ac4815d551 100644
--- a/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
+++ b/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
@@ -26,7 +26,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/mutexLocker.hpp"
@@ -37,7 +36,7 @@
// ----------------------------------------------------------------------------
-// A PPC CompiledDirectStaticCall looks like this:
+// A PPC CompiledDirectCall looks like this:
//
// >>>> consts
//
@@ -79,7 +78,7 @@
const int IC_pos_in_java_to_interp_stub = 8;
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/* = nullptr*/) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/* = nullptr*/) {
#ifdef COMPILER2
if (mark == nullptr) {
// Get the mark within main instrs section which is set to the address of the call.
@@ -91,7 +90,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
MacroAssembler _masm(&cbuf);
// Start the stub.
- address stub = __ start_a_stub(CompiledStaticCall::to_interp_stub_size());
+ address stub = __ start_a_stub(CompiledDirectCall::to_interp_stub_size());
if (stub == nullptr) {
return nullptr; // CodeCache is full
}
@@ -135,7 +134,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
// FIXME: Assert that the stub can be identified and patched.
// Java_to_interp_stub_size should be good.
- assert((__ offset() - stub_start_offset) <= CompiledStaticCall::to_interp_stub_size(),
+ assert((__ offset() - stub_start_offset) <= CompiledDirectCall::to_interp_stub_size(),
"should be good size");
assert(!is_NativeCallTrampolineStub_at(__ addr_at(stub_start_offset)),
"must not confuse java_to_interp with trampoline stubs");
@@ -153,27 +152,20 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
// Size of java_to_interp stub, this doesn't need to be accurate but it must
// be larger or equal to the real size of the stub.
// Used for optimization in Compile::Shorten_branches.
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return 12 * BytesPerInstWord;
}
// Relocation entries for call stub, compiled java to interpreter.
// Used for optimization in Compile::Shorten_branches.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 5;
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder = nativeMovConstReg_at(stub + IC_pos_in_java_to_interp_stub);
NativeJump* jump = nativeJump_at(method_holder->next_instruction_address());
@@ -188,7 +180,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -204,7 +196,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp
index 23229d0a4c316..c1a98b734dab7 100644
--- a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp
@@ -350,7 +350,7 @@ inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) {
if (is_entry_frame) {
callers_sp = _cont.entryFP();
} else {
- CodeBlob* cb = CodeCache::find_blob(pc);
+ CodeBlob* cb = CodeCache::find_blob_fast(pc);
callers_sp = sp + cb->frame_size();
}
// set the back link
diff --git a/src/hotspot/cpu/ppc/frame_ppc.cpp b/src/hotspot/cpu/ppc/frame_ppc.cpp
index 637883986195b..b63789f320d7e 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.cpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.cpp
@@ -136,7 +136,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
// It should be safe to construct the sender though it might not be valid.
- frame sender(sender_sp, sender_pc);
+ frame sender(sender_sp, sender_pc, nullptr /* unextended_sp */, nullptr /* fp */, sender_blob);
// Do we have a valid fp?
address sender_fp = (address) sender.fp();
@@ -196,12 +196,12 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const {
assert(map->include_argument_oops(), "should be set by clear");
if (jfa->last_Java_pc() != nullptr) {
- frame fr(jfa->last_Java_sp(), jfa->last_Java_pc());
+ frame fr(jfa->last_Java_sp(), jfa->last_Java_pc(), kind::code_blob);
return fr;
}
// Last_java_pc is not set, if we come here from compiled code. The
// constructor retrieves the PC from the stack.
- frame fr(jfa->last_Java_sp());
+ frame fr(jfa->last_Java_sp(), nullptr, kind::code_blob);
return fr;
}
@@ -229,7 +229,7 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const {
assert(jfa->last_Java_sp() > sp(), "must be above this frame on stack");
map->clear();
assert(map->include_argument_oops(), "should be set by clear");
- frame fr(jfa->last_Java_sp(), jfa->last_Java_pc());
+ frame fr(jfa->last_Java_sp(), jfa->last_Java_pc(), kind::code_blob);
return fr;
}
@@ -451,7 +451,7 @@ intptr_t *frame::initial_deoptimization_info() {
#ifndef PRODUCT
// This is a generic constructor which is only used by pns() in debug.cpp.
// fp is dropped and gets determined by backlink.
-frame::frame(void* sp, void* fp, void* pc) : frame((intptr_t*)sp, (address)pc) {}
+frame::frame(void* sp, void* fp, void* pc) : frame((intptr_t*)sp, (address)pc, kind::unknown) {}
#endif
BasicObjectLock* frame::interpreter_frame_monitor_end() const {
diff --git a/src/hotspot/cpu/ppc/frame_ppc.hpp b/src/hotspot/cpu/ppc/frame_ppc.hpp
index d2c9927f119ee..0e0c1a388bfc6 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.hpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.hpp
@@ -393,16 +393,26 @@
inline common_abi* own_abi() const { return (common_abi*) _sp; }
inline common_abi* callers_abi() const { return (common_abi*) _fp; }
+ enum class kind {
+ unknown, // The frame's pc is not necessarily in the CodeCache.
+ // CodeCache::find_blob_fast(void* pc) can yield wrong results in this case and must not be used.
+ code_blob, // The frame's pc is known to be in the CodeCache but it is likely not in an nmethod.
+ // CodeCache::find_blob_fast() will be correct but not faster in this case.
+ nmethod // This is likely the frame of a nmethod.
+ // The code cache lookup is optimized based on NativePostCallNops.
+ };
+
private:
// Initialize frame members (_pc and _sp must be given)
- inline void setup();
+ inline void setup(kind knd);
public:
// Constructors
inline frame(intptr_t* sp, intptr_t* fp, address pc);
- inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp = nullptr, intptr_t* fp = nullptr, CodeBlob* cb = nullptr);
+ inline frame(intptr_t* sp, address pc, kind knd = kind::nmethod);
+ inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp, intptr_t* fp = nullptr, CodeBlob* cb = nullptr);
inline frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map);
inline frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map, bool on_heap);
diff --git a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
index 220b9c3241e01..7b1f37a342ffd 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
@@ -35,14 +35,14 @@
// Inline functions for ppc64 frames:
// Initialize frame members (_sp must be given)
-inline void frame::setup() {
+inline void frame::setup(kind knd) {
if (_pc == nullptr) {
_pc = (address)own_abi()->lr;
assert(_pc != nullptr, "must have PC");
}
if (_cb == nullptr) {
- _cb = CodeCache::find_blob(_pc);
+ _cb = (knd == kind::nmethod) ? CodeCache::find_blob_fast(_pc) : CodeCache::find_blob(_pc);
}
if (_unextended_sp == nullptr) {
@@ -89,21 +89,27 @@ inline void frame::setup() {
inline frame::frame() : _sp(nullptr), _pc(nullptr), _cb(nullptr), _oop_map(nullptr), _deopt_state(unknown),
_on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(nullptr), _fp(nullptr) {}
-inline frame::frame(intptr_t* sp) : frame(sp, nullptr) {}
+inline frame::frame(intptr_t* sp) : frame(sp, nullptr, kind::nmethod) {}
inline frame::frame(intptr_t* sp, intptr_t* fp, address pc) : frame(sp, pc, nullptr, fp, nullptr) {}
+inline frame::frame(intptr_t* sp, address pc, kind knd)
+ : _sp(sp), _pc(pc), _cb(nullptr), _oop_map(nullptr),
+ _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(sp), _fp(nullptr) {
+ setup(knd);
+}
+
inline frame::frame(intptr_t* sp, address pc, intptr_t* unextended_sp, intptr_t* fp, CodeBlob* cb)
: _sp(sp), _pc(pc), _cb(cb), _oop_map(nullptr),
_on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(unextended_sp), _fp(fp) {
- setup();
+ setup(kind::nmethod);
}
inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map)
: _sp(sp), _pc(pc), _cb(cb), _oop_map(oop_map),
_on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(unextended_sp), _fp(fp) {
assert(_cb != nullptr, "pc: " INTPTR_FORMAT, p2i(pc));
- setup();
+ setup(kind::nmethod);
}
inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb,
@@ -113,7 +119,7 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address
// In thaw, non-heap frames use this constructor to pass oop_map. I don't know why.
assert(_on_heap || _cb != nullptr, "these frames are always heap frames");
if (cb != nullptr) {
- setup();
+ setup(kind::nmethod);
}
#ifdef ASSERT
// The following assertion has been disabled because it would sometime trap for Continuation.run,
@@ -300,7 +306,7 @@ inline frame frame::sender_raw(RegisterMap* map) const {
// Must be native-compiled frame, i.e. the marshaling code for native
// methods that exists in the core system.
- return frame(sender_sp(), sender_pc());
+ return frame(sender_sp(), sender_pc(), kind::code_blob);
}
inline frame frame::sender(RegisterMap* map) const {
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
index cd2fd355bbb97..ab520162d350e 100644
--- a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
@@ -30,9 +30,9 @@
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/sharedRuntime.hpp"
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
index ec91e86cd7c9a..44bb03aa6d87d 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
@@ -23,11 +23,10 @@
*
*/
-#include "gc/shared/gcArguments.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "macroAssembler_ppc.hpp"
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "gc/shared/gcArguments.hpp"
+#include "gc/shared/gc_globals.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
@@ -38,6 +37,7 @@
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
#include "interpreter/interpreter.hpp"
+#include "macroAssembler_ppc.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/globalDefinitions.hpp"
diff --git a/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp
index b83994ee8de94..7db34177d3acf 100644
--- a/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp
@@ -22,8 +22,8 @@
* questions.
*/
-#include "asm/register.hpp"
#include "precompiled.hpp"
+#include "asm/register.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/codeBlob.hpp"
#include "code/vmreg.inline.hpp"
diff --git a/src/hotspot/cpu/ppc/globals_ppc.hpp b/src/hotspot/cpu/ppc/globals_ppc.hpp
index f46ca5db3b7df..a2a94c178fb8b 100644
--- a/src/hotspot/cpu/ppc/globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/globals_ppc.hpp
@@ -60,7 +60,7 @@ define_pd_global(bool, VMContinuations, true);
// Use large code-entry alignment.
define_pd_global(uintx, CodeCacheSegmentSize, 128);
-define_pd_global(intx, CodeEntryAlignment, 128);
+define_pd_global(intx, CodeEntryAlignment, 64);
define_pd_global(intx, OptoLoopAlignment, 16);
define_pd_global(intx, InlineSmallCode, 1500);
diff --git a/src/hotspot/cpu/ppc/icBuffer_ppc.cpp b/src/hotspot/cpu/ppc/icBuffer_ppc.cpp
deleted file mode 100644
index 4157a5b0fd788..0000000000000
--- a/src/hotspot/cpu/ppc/icBuffer_ppc.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/assembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_ppc.hpp"
-#include "oops/oop.inline.hpp"
-
-#define __ masm.
-
-int InlineCacheBuffer::ic_stub_code_size() {
- return MacroAssembler::load_const_size + MacroAssembler::b64_patchable_size;
-}
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler masm(&code);
- // Note: even though the code contains an embedded metadata, we do not need reloc info
- // because
- // (1) the metadata is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear.
-
- // Load the oop ...
- __ load_const(R19_method, (address) cached_value, R0);
- // ... and jump to entry point.
- __ b64_patchable((address) entry_point, relocInfo::none);
-
- __ flush();
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- NativeJump* jump = nativeJump_at(move->next_instruction_address());
- return jump->jump_destination();
-}
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- void* o = (void*)move->data();
- return o;
-}
-
diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
index 54a31a16c8aa1..94ef1b3c9d2be 100644
--- a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
@@ -970,9 +970,6 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
// markWord displaced_header = obj->mark().set_unlocked();
- // Load markWord from object into header.
- ld(header, oopDesc::mark_offset_in_bytes(), object);
-
if (DiagnoseSyncOnValueBasedClasses != 0) {
load_klass(tmp, object);
lwz(tmp, in_bytes(Klass::access_flags_offset()), tmp);
@@ -981,9 +978,11 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
}
if (LockingMode == LM_LIGHTWEIGHT) {
- lightweight_lock(object, /* mark word */ header, tmp, slow_case);
+ lightweight_lock(object, header, tmp, slow_case);
b(count_locking);
} else if (LockingMode == LM_LEGACY) {
+ // Load markWord from object into header.
+ ld(header, oopDesc::mark_offset_in_bytes(), object);
// Set displaced_header to be (markWord of object | UNLOCK_VALUE).
ori(header, header, markWord::unlocked_value);
@@ -1115,22 +1114,6 @@ void InterpreterMacroAssembler::unlock_object(Register monitor) {
ld(object, in_bytes(BasicObjectLock::obj_offset()), monitor);
if (LockingMode == LM_LIGHTWEIGHT) {
- // Check for non-symmetric locking. This is allowed by the spec and the interpreter
- // must handle it.
- Register tmp = current_header;
- // First check for lock-stack underflow.
- lwz(tmp, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
- cmplwi(CCR0, tmp, (unsigned)LockStack::start_offset());
- ble(CCR0, slow_case);
- // Then check if the top of the lock-stack matches the unlocked object.
- addi(tmp, tmp, -oopSize);
- ldx(tmp, tmp, R16_thread);
- cmpd(CCR0, tmp, object);
- bne(CCR0, slow_case);
-
- ld(header, oopDesc::mark_offset_in_bytes(), object);
- andi_(R0, header, markWord::monitor_value);
- bne(CCR0, slow_case);
lightweight_unlock(object, header, slow_case);
} else {
addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index 8942199610ebc..b7b5936a58da0 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
#include "gc/shared/barrierSet.hpp"
@@ -37,6 +38,7 @@
#include "oops/klass.inline.hpp"
#include "oops/methodData.hpp"
#include "prims/methodHandles.hpp"
+#include "register_ppc.hpp"
#include "runtime/icache.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/objectMonitor.hpp"
@@ -1187,8 +1189,87 @@ void MacroAssembler::post_call_nop() {
if (!Continuations::enabled()) {
return;
}
+ // We use CMPI/CMPLI instructions to encode post call nops.
+ // Refer to NativePostCallNop for details.
+ relocate(post_call_nop_Relocation::spec());
InlineSkippedInstructionsCounter skipCounter(this);
- nop();
+ Assembler::emit_int32(Assembler::CMPLI_OPCODE | Assembler::opp_u_field(1, 9, 9));
+ assert(is_post_call_nop(*(int*)(pc() - 4)), "post call not not found");
+}
+
+int MacroAssembler::ic_check_size() {
+ bool implicit_null_checks_available = ImplicitNullChecks && os::zero_page_read_protected(),
+ use_fast_receiver_null_check = implicit_null_checks_available || TrapBasedNullChecks,
+ use_trap_based_null_check = !implicit_null_checks_available && TrapBasedNullChecks;
+
+ int num_ins;
+ if (use_fast_receiver_null_check && TrapBasedICMissChecks) {
+ num_ins = 3;
+ if (use_trap_based_null_check) num_ins += 1;
+ } else {
+ num_ins = 7;
+ if (!implicit_null_checks_available) num_ins += 2;
+ }
+ return num_ins * BytesPerInstWord;
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ bool implicit_null_checks_available = ImplicitNullChecks && os::zero_page_read_protected(),
+ use_fast_receiver_null_check = implicit_null_checks_available || TrapBasedNullChecks,
+ use_trap_based_null_check = !implicit_null_checks_available && TrapBasedNullChecks;
+
+ Register receiver = R3_ARG1;
+ Register data = R19_inline_cache_reg;
+ Register tmp1 = R11_scratch1;
+ Register tmp2 = R12_scratch2;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, end_alignment, end_alignment - ic_check_size());
+
+ int uep_offset = offset();
+
+ if (use_fast_receiver_null_check && TrapBasedICMissChecks) {
+ // Fast version which uses SIGTRAP
+
+ if (use_trap_based_null_check) {
+ trap_null_check(receiver);
+ }
+ if (UseCompressedClassPointers) {
+ lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
+ } else {
+ ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
+ }
+ ld(tmp2, in_bytes(CompiledICData::speculated_klass_offset()), data);
+ trap_ic_miss_check(tmp1, tmp2);
+
+ } else {
+ // Slower version which doesn't use SIGTRAP
+
+ // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
+ calculate_address_from_global_toc(tmp1, SharedRuntime::get_ic_miss_stub(),
+ true, true, false); // 2 instructions
+ mtctr(tmp1);
+
+ if (!implicit_null_checks_available) {
+ cmpdi(CCR0, receiver, 0);
+ beqctr(CCR0);
+ }
+ if (UseCompressedClassPointers) {
+ lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
+ } else {
+ ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
+ }
+ ld(tmp2, in_bytes(CompiledICData::speculated_klass_offset()), data);
+ cmpd(CCR0, tmp1, tmp2);
+ bnectr(CCR0);
+ }
+
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
+
+ return uep_offset;
}
void MacroAssembler::call_VM_base(Register oop_result,
@@ -2179,8 +2260,8 @@ address MacroAssembler::emit_trampoline_stub(int destination_toc_offset,
// "The box" is the space on the stack where we copy the object mark.
void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box,
Register temp, Register displaced_header, Register current_header) {
+ assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_lock_lightweight");
assert_different_registers(oop, box, temp, displaced_header, current_header);
- assert(LockingMode != LM_LIGHTWEIGHT || flag == CCR0, "bad condition register");
Label object_has_monitor;
Label cas_failed;
Label success, failure;
@@ -2204,7 +2285,8 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
// Set NE to indicate 'failure' -> take slow-path.
crandc(flag, Assembler::equal, flag, Assembler::equal);
b(failure);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Set displaced_header to be (markWord of object | UNLOCK_VALUE).
ori(displaced_header, displaced_header, markWord::unlocked_value);
@@ -2248,10 +2330,6 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
}
beq(CCR0, success);
b(failure);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- lightweight_lock(oop, displaced_header, temp, failure);
- b(success);
}
// Handle existing monitor.
@@ -2269,10 +2347,8 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
MacroAssembler::cmpxchgx_hint_acquire_lock());
- if (LockingMode != LM_LIGHTWEIGHT) {
- // Store a non-null value into the box.
- std(box, BasicLock::displaced_header_offset_in_bytes(), box);
- }
+ // Store a non-null value into the box.
+ std(box, BasicLock::displaced_header_offset_in_bytes(), box);
beq(flag, success);
// Check for recursive locking.
@@ -2294,8 +2370,8 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
Register temp, Register displaced_header, Register current_header) {
+ assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
assert_different_registers(oop, box, temp, displaced_header, current_header);
- assert(LockingMode != LM_LIGHTWEIGHT || flag == CCR0, "bad condition register");
Label success, failure, object_has_monitor, notRecursive;
if (LockingMode == LM_LEGACY) {
@@ -2317,7 +2393,8 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe
// Set NE to indicate 'failure' -> take slow-path.
crandc(flag, Assembler::equal, flag, Assembler::equal);
b(failure);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Check if it is still a light weight lock, this is is true if we see
// the stack address of the basicLock in the markWord of the object.
// Cmpxchg sets flag to cmpd(current_header, box).
@@ -2332,10 +2409,6 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe
&failure);
assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
b(success);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- lightweight_unlock(oop, current_header, failure);
- b(success);
}
// Handle existing monitor.
@@ -2375,6 +2448,276 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe
bind(failure);
}
+void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister flag, Register obj, Register tmp1,
+ Register tmp2, Register tmp3) {
+ assert_different_registers(obj, tmp1, tmp2, tmp3);
+ assert(flag == CCR0, "bad condition register");
+
+ // Handle inflated monitor.
+ Label inflated;
+ // Finish fast lock successfully. MUST reach to with flag == NE
+ Label locked;
+ // Finish fast lock unsuccessfully. MUST branch to with flag == EQ
+ Label slow_path;
+
+ if (DiagnoseSyncOnValueBasedClasses != 0) {
+ load_klass(tmp1, obj);
+ lwz(tmp1, in_bytes(Klass::access_flags_offset()), tmp1);
+ testbitdi(flag, R0, tmp1, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
+ bne(flag, slow_path);
+ }
+
+ const Register mark = tmp1;
+ const Register t = tmp3; // Usage of R0 allowed!
+
+ { // Lightweight locking
+
+ // Push lock to the lock stack and finish successfully. MUST reach to with flag == EQ
+ Label push;
+
+ const Register top = tmp2;
+
+ // Check if lock-stack is full.
+ lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ cmplwi(flag, top, LockStack::end_offset() - 1);
+ bgt(flag, slow_path);
+
+ // The underflow check is elided. The recursive check will always fail
+ // when the lock stack is empty because of the _bad_oop_sentinel field.
+
+ // Check if recursive.
+ subi(t, top, oopSize);
+ ldx(t, R16_thread, t);
+ cmpd(flag, obj, t);
+ beq(flag, push);
+
+ // Check for monitor (0b10) or locked (0b00).
+ ld(mark, oopDesc::mark_offset_in_bytes(), obj);
+ andi_(t, mark, markWord::lock_mask_in_place);
+ cmpldi(flag, t, markWord::unlocked_value);
+ bgt(flag, inflated);
+ bne(flag, slow_path);
+
+ // Not inflated.
+
+ // Try to lock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
+ atomically_flip_locked_state(/* is_unlock */ false, obj, mark, slow_path, MacroAssembler::MemBarAcq);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ stdx(obj, R16_thread, top);
+ addi(top, top, oopSize);
+ stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ b(locked);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated);
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register tagged_monitor = mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+ const Register owner_addr = tmp2;
+
+ // Compute owner address.
+ addi(owner_addr, tagged_monitor, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
+
+ // CAS owner (null => current thread).
+ cmpxchgd(/*flag=*/flag,
+ /*current_value=*/t,
+ /*compare_value=*/(intptr_t)0,
+ /*exchange_value=*/R16_thread,
+ /*where=*/owner_addr,
+ MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
+ MacroAssembler::cmpxchgx_hint_acquire_lock());
+ beq(flag, locked);
+
+ // Check if recursive.
+ cmpd(flag, t, R16_thread);
+ bne(flag, slow_path);
+
+ // Recursive.
+ ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
+ addi(tmp1, tmp1, 1);
+ std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
+ }
+
+ bind(locked);
+ inc_held_monitor_count(tmp1);
+
+#ifdef ASSERT
+ // Check that locked label is reached with flag == EQ.
+ Label flag_correct;
+ beq(flag, flag_correct);
+ stop("Fast Lock Flag != EQ");
+#endif
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with flag == NE.
+ bne(flag, flag_correct);
+ stop("Fast Lock Flag != NE");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of flag (NE vs EQ) to determine the continuation.
+}
+
+void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister flag, Register obj, Register tmp1,
+ Register tmp2, Register tmp3) {
+ assert_different_registers(obj, tmp1, tmp2, tmp3);
+ assert(flag == CCR0, "bad condition register");
+
+ // Handle inflated monitor.
+ Label inflated, inflated_load_monitor;
+ // Finish fast unlock successfully. MUST reach to with flag == EQ.
+ Label unlocked;
+ // Finish fast unlock unsuccessfully. MUST branch to with flag == NE.
+ Label slow_path;
+
+ const Register mark = tmp1;
+ const Register top = tmp2;
+ const Register t = tmp3;
+
+ { // Lightweight unlock
+ Label push_and_slow;
+
+ // Check if obj is top of lock-stack.
+ lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ subi(top, top, oopSize);
+ ldx(t, R16_thread, top);
+ cmpd(flag, obj, t);
+ // Top of lock stack was not obj. Must be monitor.
+ bne(flag, inflated_load_monitor);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(li(t, 0);)
+ DEBUG_ONLY(stdx(t, R16_thread, top);)
+ stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+
+ // The underflow check is elided. The recursive check will always fail
+ // when the lock stack is empty because of the _bad_oop_sentinel field.
+
+ // Check if recursive.
+ subi(t, top, oopSize);
+ ldx(t, R16_thread, t);
+ cmpd(flag, obj, t);
+ beq(flag, unlocked);
+
+ // Not recursive.
+
+ // Check for monitor (0b10).
+ ld(mark, oopDesc::mark_offset_in_bytes(), obj);
+ andi_(t, mark, markWord::monitor_value);
+ bne(CCR0, inflated);
+
+#ifdef ASSERT
+ // Check header not unlocked (0b01).
+ Label not_unlocked;
+ andi_(t, mark, markWord::unlocked_value);
+ beq(CCR0, not_unlocked);
+ stop("lightweight_unlock already unlocked");
+ bind(not_unlocked);
+#endif
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ atomically_flip_locked_state(/* is_unlock */ true, obj, mark, push_and_slow, MacroAssembler::MemBarRel);
+ b(unlocked);
+
+ bind(push_and_slow);
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(stdx(obj, R16_thread, top);)
+ addi(top, top, oopSize);
+ stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ b(slow_path);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated_load_monitor);
+ ld(mark, oopDesc::mark_offset_in_bytes(), obj);
+#ifdef ASSERT
+ andi_(t, mark, markWord::monitor_value);
+ bne(CCR0, inflated);
+ stop("Fast Unlock not monitor");
+#endif
+
+ bind(inflated);
+
+#ifdef ASSERT
+ Label check_done;
+ subi(top, top, oopSize);
+ cmplwi(CCR0, top, in_bytes(JavaThread::lock_stack_base_offset()));
+ blt(CCR0, check_done);
+ ldx(t, R16_thread, top);
+ cmpd(flag, obj, t);
+ bne(flag, inflated);
+ stop("Fast Unlock lock on stack");
+ bind(check_done);
+#endif
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register monitor = mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+
+ // Untag the monitor.
+ subi(monitor, mark, monitor_tag);
+
+ const Register recursions = tmp2;
+ Label not_recursive;
+
+ // Check if recursive.
+ ld(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
+ addic_(recursions, recursions, -1);
+ blt(CCR0, not_recursive);
+
+ // Recursive unlock.
+ std(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
+ crorc(CCR0, Assembler::equal, CCR0, Assembler::equal);
+ b(unlocked);
+
+ bind(not_recursive);
+
+ Label release_;
+ const Register t2 = tmp2;
+
+ // Check if the entry lists are empty.
+ ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
+ ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
+ orr(t, t, t2);
+ cmpdi(flag, t, 0);
+ beq(flag, release_);
+
+ // The owner may be anonymous and we removed the last obj entry in
+ // the lock-stack. This loses the information about the owner.
+ // Write the thread to the owner field so the runtime knows the owner.
+ std(R16_thread, in_bytes(ObjectMonitor::owner_offset()), monitor);
+ b(slow_path);
+
+ bind(release_);
+ // Set owner to null.
+ release();
+ // t contains 0
+ std(t, in_bytes(ObjectMonitor::owner_offset()), monitor);
+ }
+
+ bind(unlocked);
+ dec_held_monitor_count(t);
+
+#ifdef ASSERT
+ // Check that unlocked label is reached with flag == EQ.
+ Label flag_correct;
+ beq(flag, flag_correct);
+ stop("Fast Lock Flag != EQ");
+#endif
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with flag == NE.
+ bne(flag, flag_correct);
+ stop("Fast Lock Flag != NE");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of flag (NE vs EQ) to determine the continuation.
+}
+
void MacroAssembler::safepoint_poll(Label& slow_path, Register temp, bool at_return, bool in_nmethod) {
ld(temp, in_bytes(JavaThread::polling_word_offset()), R16_thread);
@@ -3994,58 +4337,57 @@ void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj,
}
// Implements lightweight-locking.
-// Branches to slow upon failure to lock the object, with CCR0 NE.
-// Falls through upon success with CCR0 EQ.
//
// - obj: the object to be locked
-// - hdr: the header, already loaded from obj, will be destroyed
-// - t1: temporary register
-void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, Label& slow) {
+// - t1, t2: temporary register
+void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, t1);
+ assert_different_registers(obj, t1, t2);
+
+ Label push;
+ const Register top = t1;
+ const Register mark = t2;
+ const Register t = R0;
+
+ // Check if the lock-stack is full.
+ lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ cmplwi(CCR0, top, LockStack::end_offset());
+ bge(CCR0, slow);
+
+ // The underflow check is elided. The recursive check will always fail
+ // when the lock stack is empty because of the _bad_oop_sentinel field.
- // Check if we would have space on lock-stack for the object.
- lwz(t1, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
- cmplwi(CCR0, t1, LockStack::end_offset() - 1);
- bgt(CCR0, slow);
+ // Check for recursion.
+ subi(t, top, oopSize);
+ ldx(t, R16_thread, t);
+ cmpd(CCR0, obj, t);
+ beq(CCR0, push);
- // Quick check: Do not reserve cache line for atomic update if not unlocked.
- // (Similar to contention_hint in cmpxchg solutions.)
- xori(R0, hdr, markWord::unlocked_value); // flip unlocked bit
- andi_(R0, R0, markWord::lock_mask_in_place);
- bne(CCR0, slow); // failed if new header doesn't contain locked_value (which is 0)
+ // Check header for monitor (0b10) or locked (0b00).
+ ld(mark, oopDesc::mark_offset_in_bytes(), obj);
+ xori(t, mark, markWord::unlocked_value);
+ andi_(t, t, markWord::lock_mask_in_place);
+ bne(CCR0, slow);
- // Note: We're not publishing anything (like the displaced header in LM_LEGACY)
- // to other threads at this point. Hence, no release barrier, here.
- // (The obj has been written to the BasicObjectLock at obj_offset() within the own thread stack.)
- atomically_flip_locked_state(/* is_unlock */ false, obj, hdr, slow, MacroAssembler::MemBarAcq);
+ // Try to lock. Transition lock bits 0b00 => 0b01
+ atomically_flip_locked_state(/* is_unlock */ false, obj, mark, slow, MacroAssembler::MemBarAcq);
+ bind(push);
// After successful lock, push object on lock-stack
- stdx(obj, t1, R16_thread);
- addi(t1, t1, oopSize);
- stw(t1, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ stdx(obj, R16_thread, top);
+ addi(top, top, oopSize);
+ stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
}
// Implements lightweight-unlocking.
-// Branches to slow upon failure, with CCR0 NE.
-// Falls through upon success, with CCR0 EQ.
//
// - obj: the object to be unlocked
-// - hdr: the (pre-loaded) header of the object, will be destroyed
-void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Label& slow) {
+// - t1: temporary register
+void MacroAssembler::lightweight_unlock(Register obj, Register t1, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr);
+ assert_different_registers(obj, t1);
#ifdef ASSERT
- {
- // Check that hdr is fast-locked.
- Label hdr_ok;
- andi_(R0, hdr, markWord::lock_mask_in_place);
- beq(CCR0, hdr_ok);
- stop("Header is not fast-locked");
- bind(hdr_ok);
- }
- Register t1 = hdr; // Reuse in debug build.
{
// The following checks rely on the fact that LockStack is only ever modified by
// its owning thread, even if the lock got inflated concurrently; removal of LockStack
@@ -4055,32 +4397,67 @@ void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Label& slow)
Label stack_ok;
lwz(t1, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
cmplwi(CCR0, t1, LockStack::start_offset());
- bgt(CCR0, stack_ok);
+ bge(CCR0, stack_ok);
stop("Lock-stack underflow");
bind(stack_ok);
}
- {
- // Check if the top of the lock-stack matches the unlocked object.
- Label tos_ok;
- addi(t1, t1, -oopSize);
- ldx(t1, t1, R16_thread);
- cmpd(CCR0, t1, obj);
- beq(CCR0, tos_ok);
- stop("Top of lock-stack does not match the unlocked object");
- bind(tos_ok);
- }
#endif
- // Release the lock.
- atomically_flip_locked_state(/* is_unlock */ true, obj, hdr, slow, MacroAssembler::MemBarRel);
+ Label unlocked, push_and_slow;
+ const Register top = t1;
+ const Register mark = R0;
+ Register t = R0;
+
+ // Check if obj is top of lock-stack.
+ lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ subi(top, top, oopSize);
+ ldx(t, R16_thread, top);
+ cmpd(CCR0, obj, t);
+ bne(CCR0, slow);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(li(t, 0);)
+ DEBUG_ONLY(stdx(t, R16_thread, top);)
+ stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+
+ // The underflow check is elided. The recursive check will always fail
+ // when the lock stack is empty because of the _bad_oop_sentinel field.
+
+ // Check if recursive.
+ subi(t, top, oopSize);
+ ldx(t, R16_thread, t);
+ cmpd(CCR0, obj, t);
+ beq(CCR0, unlocked);
+
+ // Use top as tmp
+ t = top;
+
+ // Not recursive. Check header for monitor (0b10).
+ ld(mark, oopDesc::mark_offset_in_bytes(), obj);
+ andi_(t, mark, markWord::monitor_value);
+ bne(CCR0, push_and_slow);
- // After successful unlock, pop object from lock-stack
- Register t2 = hdr;
- lwz(t2, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
- addi(t2, t2, -oopSize);
#ifdef ASSERT
- li(R0, 0);
- stdx(R0, t2, R16_thread);
+ // Check header not unlocked (0b01).
+ Label not_unlocked;
+ andi_(t, mark, markWord::unlocked_value);
+ beq(CCR0, not_unlocked);
+ stop("lightweight_unlock already unlocked");
+ bind(not_unlocked);
#endif
- stw(t2, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
+ b(unlocked);
+
+ bind(push_and_slow);
+
+ // Restore lock-stack and handle the unlock in runtime.
+ lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ DEBUG_ONLY(stdx(obj, R16_thread, top);)
+ addi(top, top, oopSize);
+ stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
+ b(slow);
+
+ bind(unlocked);
}
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index 52d8e7fa96dec..92db8a86b42ff 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -367,6 +367,9 @@ class MacroAssembler: public Assembler {
Register toc);
#endif
+ static int ic_check_size();
+ int ic_check(int end_alignment);
+
protected:
// It is imperative that all calls into the VM are handled via the
@@ -417,6 +420,12 @@ class MacroAssembler: public Assembler {
inline void call_stub_and_return_to(Register function_entry, Register return_pc);
void post_call_nop();
+ static bool is_post_call_nop(int instr_bits) {
+ const uint32_t nineth_bit = opp_u_field(1, 9, 9);
+ const uint32_t opcode_mask = 0b111110 << OPCODE_SHIFT;
+ const uint32_t pcn_mask = opcode_mask | nineth_bit;
+ return (instr_bits & pcn_mask) == (Assembler::CMPLI_OPCODE | nineth_bit);
+ }
//
// Java utilities
@@ -606,8 +615,8 @@ class MacroAssembler: public Assembler {
void inc_held_monitor_count(Register tmp);
void dec_held_monitor_count(Register tmp);
void atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics);
- void lightweight_lock(Register obj, Register hdr, Register t1, Label& slow);
- void lightweight_unlock(Register obj, Register hdr, Label& slow);
+ void lightweight_lock(Register obj, Register t1, Register t2, Label& slow);
+ void lightweight_unlock(Register obj, Register t1, Label& slow);
// allocation (for C1)
void tlab_allocate(
@@ -628,6 +637,12 @@ class MacroAssembler: public Assembler {
void compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
Register tmp1, Register tmp2, Register tmp3);
+ void compiler_fast_lock_lightweight_object(ConditionRegister flag, Register oop, Register tmp1,
+ Register tmp2, Register tmp3);
+
+ void compiler_fast_unlock_lightweight_object(ConditionRegister flag, Register oop, Register tmp1,
+ Register tmp2, Register tmp3);
+
// Check if safepoint requested and if so branch
void safepoint_poll(Label& slow_path, Register temp, bool at_return, bool in_nmethod);
diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
index 06754ccfdd7d2..e1a41a17d936f 100644
--- a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
+++ b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
@@ -429,6 +429,31 @@ void NativePostCallNop::make_deopt() {
NativeDeoptInstruction::insert(addr_at(0));
}
+bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
+ int32_t i2, i1;
+ assert(is_aligned(cb_offset, 4), "cb offset alignment does not match instruction alignment");
+ assert(!decode(i1, i2), "already patched");
+
+ cb_offset = cb_offset >> 2;
+ if (((oopmap_slot & ppc_oopmap_slot_mask) != oopmap_slot) || ((cb_offset & ppc_cb_offset_mask) != cb_offset)) {
+ return false; // cannot encode
+ }
+ const uint32_t data = oopmap_slot << ppc_cb_offset_bits | cb_offset;
+ const uint32_t lo_data = data & ppc_data_lo_mask;
+ const uint32_t hi_data = data >> ppc_data_lo_bits;
+ const uint32_t nineth_bit = 1 << (31 - 9);
+ uint32_t instr = Assembler::CMPLI_OPCODE | hi_data << ppc_data_hi_shift | nineth_bit | lo_data;
+ *(uint32_t*)addr_at(0) = instr;
+
+ int32_t oopmap_slot_dec, cb_offset_dec;
+ assert(is_post_call_nop(), "pcn not recognized");
+ assert(decode(oopmap_slot_dec, cb_offset_dec), "encoding failed");
+ assert(oopmap_slot == oopmap_slot_dec, "oopmap slot encoding is wrong");
+ assert((cb_offset << 2) == cb_offset_dec, "cb offset encoding is wrong");
+
+ return true; // encoding succeeded
+}
+
void NativeDeoptInstruction::verify() {
}
diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
index ec6f5a90a7211..113cedfee7cab 100644
--- a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
+++ b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
@@ -51,7 +51,7 @@ class NativeInstruction {
friend class Relocation;
public:
- bool is_nop() const { return Assembler::is_nop(long_at(0)); }
+ bool is_post_call_nop() const { return MacroAssembler::is_post_call_nop(long_at(0)); }
bool is_jump() const { return Assembler::is_b(long_at(0)); } // See NativeGeneralJump.
@@ -506,10 +506,50 @@ class NativeMovRegMem: public NativeInstruction {
};
class NativePostCallNop: public NativeInstruction {
+
+ // We use CMPI/CMPLI to represent Post Call Nops (PCN)
+
+ // Bit |0 5|6 |9 |10|11 |16 31|
+ // +--------------------------------------------------------------+
+ // Field |OPCODE |BF |/ |L |RA |SI |
+ // +--------------------------------------------------------------+
+ // |0 0 1 0 1|DATA HI| 1| DATA LO |
+ // | |4 bits | | 22 bits |
+ //
+ // Bit 9 is always 1 for PCNs to distinguish them from regular CMPI/CMPLI
+ //
+ // Using both, CMPLI (opcode 10 = 0b001010) and CMPI (opcode 11 = 0b001011) for
+ // PCNs allows using bit 5 from the opcode to encode DATA HI.
+
+ enum {
+ ppc_data_lo_bits = 31 - 9,
+ ppc_data_lo_mask = right_n_bits(ppc_data_lo_bits),
+ ppc_data_hi_bits = 9 - 5,
+ ppc_data_hi_shift = ppc_data_lo_bits + 1,
+ ppc_data_hi_mask = right_n_bits(ppc_data_hi_bits) << ppc_data_hi_shift,
+ ppc_data_bits = ppc_data_lo_bits + ppc_data_hi_bits,
+
+ ppc_oopmap_slot_bits = 9,
+ ppc_oopmap_slot_mask = right_n_bits(ppc_oopmap_slot_bits),
+ ppc_cb_offset_bits = ppc_data_bits - ppc_oopmap_slot_bits,
+ ppc_cb_offset_mask = right_n_bits(ppc_cb_offset_bits),
+};
+
public:
- bool check() const { return is_nop(); }
- bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; }
- bool patch(int32_t oopmap_slot, int32_t cb_offset) { return false; }
+ bool check() const { return is_post_call_nop(); }
+ bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const {
+ uint32_t instr_bits = long_at(0);
+ uint32_t data_lo = instr_bits & ppc_data_lo_mask;
+ uint32_t data_hi = (instr_bits & ppc_data_hi_mask) >> 1;
+ uint32_t data = data_hi | data_lo;
+ if (data == 0) {
+ return false; // no data found
+ }
+ cb_offset = (data & ppc_cb_offset_mask) << 2;
+ oopmap_slot = data >> ppc_cb_offset_bits;
+ return true; // decoding succeeded
+ }
+ bool patch(int32_t oopmap_slot, int32_t cb_offset);
void make_deopt();
};
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index be37ff1785b64..1058ae35b76b7 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2012, 2023 SAP SE. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -1026,7 +1026,7 @@ bool followed_by_acquire(const Node *load) {
assert(load->is_Load(), "So far implemented only for loads.");
// Find MemBarAcquire.
- const Node *mba = NULL;
+ const Node *mba = nullptr;
for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) {
const Node *out = load->fast_out(i);
if (out->Opcode() == Op_MemBarAcquire) {
@@ -1043,7 +1043,7 @@ bool followed_by_acquire(const Node *load) {
// edge to assure no other operations are in between the two nodes.
//
// So first get the Proj node, mem_proj, to use it to iterate forward.
- Node *mem_proj = NULL;
+ Node *mem_proj = nullptr;
for (DUIterator_Fast imax, i = mba->fast_outs(imax); i < imax; i++) {
mem_proj = mba->fast_out(i); // Runs out of bounds and asserts if Proj not found.
assert(mem_proj->is_Proj(), "only projections here");
@@ -1270,7 +1270,7 @@ source %{
void CallStubImpl::emit_trampoline_stub(C2_MacroAssembler &_masm, int destination_toc_offset, int insts_call_instruction_offset) {
address stub = __ emit_trampoline_stub(destination_toc_offset, insts_call_instruction_offset);
- if (stub == NULL) {
+ if (stub == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
}
}
@@ -1305,11 +1305,11 @@ EmitCallOffsets emit_call_with_trampoline_stub(C2_MacroAssembler &_masm, address
offsets.insts_call_instruction_offset = __ offset();
// No entry point given, use the current pc.
- if (entry_point == NULL) entry_point = __ pc();
+ if (entry_point == nullptr) entry_point = __ pc();
// Put the entry point as a constant into the constant pool.
const address entry_point_toc_addr = __ address_constant(entry_point, RelocationHolder::none);
- if (entry_point_toc_addr == NULL) {
+ if (entry_point_toc_addr == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return offsets;
}
@@ -1355,8 +1355,8 @@ void MachConstantBaseNode::postalloc_expand(GrowableArray *nodes, Phase
MachNode *m1 = new loadToc_hiNode();
MachNode *m2 = new loadToc_loNode();
- m1->add_req(NULL);
- m2->add_req(NULL, m1);
+ m1->add_req(nullptr);
+ m2->add_req(nullptr, m1);
m1->_opnds[0] = op_dst;
m2->_opnds[0] = op_dst;
m2->_opnds[1] = op_dst;
@@ -1398,7 +1398,7 @@ void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
st->print("push frame %ld\n\t", -framesize);
}
- if (C->stub_function() == NULL) {
+ if (C->stub_function() == nullptr) {
st->print("nmethod entry barrier\n\t");
}
}
@@ -1554,7 +1554,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
__ std(return_pc, _abi0(lr), callers_sp);
}
- if (C->stub_function() == NULL) {
+ if (C->stub_function() == nullptr) {
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->nmethod_entry_barrier(&_masm, push_frame_temp);
}
@@ -1724,7 +1724,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
if (src_lo == dst_lo && src_hi == dst_hi)
return size; // Self copy, no move.
- if (bottom_type()->isa_vect() != NULL && ideal_reg() == Op_VecX) {
+ if (bottom_type()->isa_vect() != nullptr && ideal_reg() == Op_VecX) {
// Memory->Memory Spill.
if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
int src_offset = ra_->reg2offset(src_lo);
@@ -1910,16 +1910,16 @@ void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
if (!ra_)
st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
else
- implementation(NULL, ra_, false, st);
+ implementation(nullptr, ra_, false, st);
}
#endif
void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- implementation(&cbuf, ra_, false, NULL);
+ implementation(&cbuf, ra_, false, nullptr);
}
uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
- return implementation(NULL, ra_, true, NULL);
+ return implementation(nullptr, ra_, true, nullptr);
}
#ifndef PRODUCT
@@ -1978,42 +1978,7 @@ void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
// This is the unverified entry point.
C2_MacroAssembler _masm(&cbuf);
- // Inline_cache contains a klass.
- Register ic_klass = as_Register(Matcher::inline_cache_reg_encode());
- Register receiver_klass = R12_scratch2; // tmp
-
- assert_different_registers(ic_klass, receiver_klass, R11_scratch1, R3_ARG1);
- assert(R11_scratch1 == R11, "need prologue scratch register");
-
- // Check for NULL argument if we don't have implicit null checks.
- if (!ImplicitNullChecks || !os::zero_page_read_protected()) {
- if (TrapBasedNullChecks) {
- __ trap_null_check(R3_ARG1);
- } else {
- Label valid;
- __ cmpdi(CCR0, R3_ARG1, 0);
- __ bne_predict_taken(CCR0, valid);
- // We have a null argument, branch to ic_miss_stub.
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- __ bind(valid);
- }
- }
- // Assume argument is not NULL, load klass from receiver.
- __ load_klass(receiver_klass, R3_ARG1);
-
- if (TrapBasedICMissChecks) {
- __ trap_ic_miss_check(receiver_klass, ic_klass);
- } else {
- Label valid;
- __ cmpd(CCR0, receiver_klass, ic_klass);
- __ beq_predict_taken(CCR0, valid);
- // We have an unexpected klass, branch to ic_miss_stub.
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- __ bind(valid);
- }
-
+ __ ic_check(CodeEntryAlignment);
// Argument is valid and klass is as expected, continue.
}
@@ -2062,7 +2027,10 @@ int HandlerImpl::emit_exception_handler(CodeBuffer &cbuf) {
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_exception_handler());
- if (base == NULL) return 0; // CodeBuffer::expand failed
+ if (base == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
+ return 0; // CodeBuffer::expand failed
+ }
int offset = __ offset();
__ b64_patchable((address)OptoRuntime::exception_blob()->content_begin(),
@@ -2079,7 +2047,10 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_deopt_handler());
- if (base == NULL) return 0; // CodeBuffer::expand failed
+ if (base == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
+ return 0; // CodeBuffer::expand failed
+ }
int offset = __ offset();
__ bl64_patchable((address)SharedRuntime::deopt_blob()->unpack(),
@@ -2173,7 +2144,7 @@ bool Matcher::match_rule_supported(int opcode) {
return true; // Per default match rules are supported.
}
-bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
return match_rule_supported_vector(opcode, vlen, bt);
}
@@ -2193,11 +2164,11 @@ bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
}
const RegMask* Matcher::predicate_reg_mask(void) {
- return NULL;
+ return nullptr;
}
const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return NULL;
+ return nullptr;
}
// Vector calling convention not yet implemented.
@@ -2242,7 +2213,7 @@ int Matcher::min_vector_size(const BasicType bt) {
return max_vector_size(bt); // Same as max.
}
-int Matcher::superword_max_vector_size(const BasicType bt) {
+int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
return Matcher::max_vector_size(bt);
}
@@ -2275,7 +2246,7 @@ bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
/* TODO: PPC port
// Make a new machine dependent decode node (with its operands).
MachTypeNode *Matcher::make_decode_node() {
- assert(CompressedOops::base() == NULL && CompressedOops::shift() == 0,
+ assert(CompressedOops::base() == nullptr && CompressedOops::shift() == 0,
"This method is only implemented for unscaled cOops mode so far");
MachTypeNode *decode = new decodeN_unscaledNode();
decode->set_opnd_array(0, new iRegPdstOper());
@@ -2286,7 +2257,7 @@ MachTypeNode *Matcher::make_decode_node() {
MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
ShouldNotReachHere(); // generic vector operands not supported
- return NULL;
+ return nullptr;
}
bool Matcher::is_reg2reg_move(MachNode* m) {
@@ -2545,7 +2516,7 @@ encode %{
// Create a non-oop constant, no relocation needed.
// If it is an IC, it has a virtual_call_Relocation.
const_toc_addr = __ long_constant((jlong)$src$$constant);
- if (const_toc_addr == NULL) {
+ if (const_toc_addr == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
@@ -2568,7 +2539,7 @@ encode %{
// Create a non-oop constant, no relocation needed.
// If it is an IC, it has a virtual_call_Relocation.
const_toc_addr = __ long_constant((jlong)$src$$constant);
- if (const_toc_addr == NULL) {
+ if (const_toc_addr == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
@@ -2607,8 +2578,8 @@ loadConLNodesTuple loadConLNodesTuple_create(PhaseRegAlloc *ra_, Node *toc, immL
loadConL_loNode *m2 = new loadConL_loNode();
// inputs for new nodes
- m1->add_req(NULL, toc);
- m2->add_req(NULL, m1);
+ m1->add_req(nullptr, toc);
+ m2->add_req(nullptr, m1);
// operands for new nodes
m1->_opnds[0] = new iRegLdstOper(); // dst
@@ -2632,14 +2603,14 @@ loadConLNodesTuple loadConLNodesTuple_create(PhaseRegAlloc *ra_, Node *toc, immL
// Create result.
nodes._large_hi = m1;
nodes._large_lo = m2;
- nodes._small = NULL;
+ nodes._small = nullptr;
nodes._last = nodes._large_lo;
assert(m2->bottom_type()->isa_long(), "must be long");
} else {
loadConLNode *m2 = new loadConLNode();
// inputs for new nodes
- m2->add_req(NULL, toc);
+ m2->add_req(nullptr, toc);
// operands for new nodes
m2->_opnds[0] = new iRegLdstOper(); // dst
@@ -2653,8 +2624,8 @@ loadConLNodesTuple loadConLNodesTuple_create(PhaseRegAlloc *ra_, Node *toc, immL
ra_->set_pair(m2->_idx, reg_second, reg_first);
// Create result.
- nodes._large_hi = NULL;
- nodes._large_lo = NULL;
+ nodes._large_hi = nullptr;
+ nodes._large_lo = nullptr;
nodes._small = m2;
nodes._last = nodes._small;
assert(m2->bottom_type()->isa_long(), "must be long");
@@ -2687,10 +2658,10 @@ loadConLReplicatedNodesTuple loadConLReplicatedNodesTuple_create(Compile *C, Pha
xxspltdNode *m4 = new xxspltdNode();
// inputs for new nodes
- m1->add_req(NULL, toc);
- m2->add_req(NULL, m1);
- m3->add_req(NULL, m2);
- m4->add_req(NULL, m3);
+ m1->add_req(nullptr, toc);
+ m2->add_req(nullptr, m1);
+ m3->add_req(nullptr, m2);
+ m4->add_req(nullptr, m3);
// operands for new nodes
m1->_opnds[0] = new iRegLdstOper(); // dst
@@ -2727,7 +2698,7 @@ loadConLReplicatedNodesTuple loadConLReplicatedNodesTuple_create(Compile *C, Pha
nodes._large_lo = m2;
nodes._moved = m3;
nodes._replicated = m4;
- nodes._small = NULL;
+ nodes._small = nullptr;
nodes._last = nodes._replicated;
assert(m2->bottom_type()->isa_long(), "must be long");
} else {
@@ -2736,7 +2707,7 @@ loadConLReplicatedNodesTuple loadConLReplicatedNodesTuple_create(Compile *C, Pha
xxspltdNode *m4 = new xxspltdNode();
// inputs for new nodes
- m2->add_req(NULL, toc);
+ m2->add_req(nullptr, toc);
// operands for new nodes
m2->_opnds[0] = new iRegLdstOper(); // dst
@@ -2760,8 +2731,8 @@ loadConLReplicatedNodesTuple loadConLReplicatedNodesTuple_create(Compile *C, Pha
ra_->set_pair(m2->_idx, reg_second, reg_first);
// Create result.
- nodes._large_hi = NULL;
- nodes._large_lo = NULL;
+ nodes._large_hi = nullptr;
+ nodes._large_lo = nullptr;
nodes._small = m2;
nodes._moved = m3;
nodes._replicated = m4;
@@ -2801,24 +2772,26 @@ encode %{
intptr_t val = $src$$constant;
relocInfo::relocType constant_reloc = $src->constant_reloc(); // src
address const_toc_addr;
+ RelocationHolder r; // Initializes type to none.
if (constant_reloc == relocInfo::oop_type) {
// Create an oop constant and a corresponding relocation.
- AddressLiteral a = __ allocate_oop_address((jobject)val);
+ AddressLiteral a = __ constant_oop_address((jobject)val);
const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- __ relocate(a.rspec());
+ r = a.rspec();
} else if (constant_reloc == relocInfo::metadata_type) {
+ // Notify OOP recorder (don't need the relocation)
AddressLiteral a = __ constant_metadata_address((Metadata *)val);
const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- __ relocate(a.rspec());
} else {
// Create a non-oop constant, no relocation needed.
const_toc_addr = __ long_constant((jlong)$src$$constant);
}
- if (const_toc_addr == NULL) {
+ if (const_toc_addr == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
+ __ relocate(r); // If set above.
// Get the constant's TOC offset.
toc_offset = __ offset_to_method_toc(const_toc_addr);
@@ -2832,24 +2805,26 @@ encode %{
intptr_t val = $src$$constant;
relocInfo::relocType constant_reloc = $src->constant_reloc(); // src
address const_toc_addr;
+ RelocationHolder r; // Initializes type to none.
if (constant_reloc == relocInfo::oop_type) {
// Create an oop constant and a corresponding relocation.
- AddressLiteral a = __ allocate_oop_address((jobject)val);
+ AddressLiteral a = __ constant_oop_address((jobject)val);
const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- __ relocate(a.rspec());
+ r = a.rspec();
} else if (constant_reloc == relocInfo::metadata_type) {
+ // Notify OOP recorder (don't need the relocation)
AddressLiteral a = __ constant_metadata_address((Metadata *)val);
const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- __ relocate(a.rspec());
} else { // non-oop pointers, e.g. card mark base, heap top
// Create a non-oop constant, no relocation needed.
const_toc_addr = __ long_constant((jlong)$src$$constant);
}
- if (const_toc_addr == NULL) {
+ if (const_toc_addr == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
+ __ relocate(r); // If set above.
// Get the constant's TOC offset.
const int toc_offset = __ offset_to_method_toc(const_toc_addr);
// Store the toc offset of the constant.
@@ -2870,8 +2845,8 @@ encode %{
loadConP_loNode *m2 = new loadConP_loNode();
// inputs for new nodes
- m1->add_req(NULL, n_toc);
- m2->add_req(NULL, m1);
+ m1->add_req(nullptr, n_toc);
+ m2->add_req(nullptr, m1);
// operands for new nodes
m1->_opnds[0] = new iRegPdstOper(); // dst
@@ -2896,7 +2871,7 @@ encode %{
loadConPNode *m2 = new loadConPNode();
// inputs for new nodes
- m2->add_req(NULL, n_toc);
+ m2->add_req(nullptr, n_toc);
// operands for new nodes
m2->_opnds[0] = new iRegPdstOper(); // dst
@@ -2923,7 +2898,7 @@ encode %{
m2 = new loadConFNode();
}
// inputs for new nodes
- m2->add_req(NULL, n_toc);
+ m2->add_req(nullptr, n_toc);
// operands for new nodes
m2->_opnds[0] = op_dst;
@@ -2947,7 +2922,7 @@ encode %{
m2 = new loadConDNode();
}
// inputs for new nodes
- m2->add_req(NULL, n_toc);
+ m2->add_req(nullptr, n_toc);
// operands for new nodes
m2->_opnds[0] = op_dst;
@@ -3258,9 +3233,9 @@ encode %{
Label d; // dummy
__ bind(d);
Label* p = ($lbl$$label);
- // `p' is `NULL' when this encoding class is used only to
+ // `p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
- Label& l = (NULL == p)? d : *(p);
+ Label& l = (nullptr == p)? d : *(p);
int cc = $cmp$$cmpcode;
int flags_reg = $crx$$reg;
assert((Assembler::bcondCRbiIs1 & ~Assembler::bcondCRbiIs0) == 8, "check encoding");
@@ -3287,9 +3262,9 @@ encode %{
Label d; // dummy
__ bind(d);
Label* p = ($lbl$$label);
- // `p' is `NULL' when this encoding class is used only to
+ // `p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
- Label& l = (NULL == p)? d : *(p);
+ Label& l = (nullptr == p)? d : *(p);
int cc = $cmp$$cmpcode;
int flags_reg = $crx$$reg;
int bhint = Assembler::bhintNoHint;
@@ -3431,7 +3406,7 @@ encode %{
// Put the entry point as a constant into the constant pool.
const address entry_point_toc_addr = __ address_constant(entry_point, RelocationHolder::none);
- if (entry_point_toc_addr == NULL) {
+ if (entry_point_toc_addr == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
@@ -3452,8 +3427,8 @@ encode %{
__ bl(__ pc()); // Emits a relocation.
// The stub for call to interpreter.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
- if (stub == NULL) {
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf);
+ if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -3470,7 +3445,7 @@ encode %{
// Create a call trampoline stub for the given method.
const address entry_point = !($meth$$method) ? 0 : (address)$meth$$method;
const address entry_point_const = __ address_constant(entry_point, RelocationHolder::none);
- if (entry_point_const == NULL) {
+ if (entry_point_const == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
@@ -3479,13 +3454,13 @@ encode %{
if (ra_->C->env()->failing()) { return; } // Code cache may be full.
// Build relocation at call site with ic position as data.
- assert((_load_ic_hi_node != NULL && _load_ic_node == NULL) ||
- (_load_ic_hi_node == NULL && _load_ic_node != NULL),
+ assert((_load_ic_hi_node != nullptr && _load_ic_node == nullptr) ||
+ (_load_ic_hi_node == nullptr && _load_ic_node != nullptr),
"must have one, but can't have both");
- assert((_load_ic_hi_node != NULL && _load_ic_hi_node->_cbuf_insts_offset != -1) ||
- (_load_ic_node != NULL && _load_ic_node->_cbuf_insts_offset != -1),
+ assert((_load_ic_hi_node != nullptr && _load_ic_hi_node->_cbuf_insts_offset != -1) ||
+ (_load_ic_node != nullptr && _load_ic_node->_cbuf_insts_offset != -1),
"must contain instruction offset");
- const int virtual_call_oop_addr_offset = _load_ic_hi_node != NULL
+ const int virtual_call_oop_addr_offset = _load_ic_hi_node != nullptr
? _load_ic_hi_node->_cbuf_insts_offset
: _load_ic_node->_cbuf_insts_offset;
const address virtual_call_oop_addr = __ addr_at(virtual_call_oop_addr_offset);
@@ -3507,7 +3482,7 @@ encode %{
// Create the nodes for loading the IC from the TOC.
loadConLNodesTuple loadConLNodes_IC =
- loadConLNodesTuple_create(ra_, n_toc, new immLOper((jlong)Universe::non_oop_word()),
+ loadConLNodesTuple_create(ra_, n_toc, new immLOper((jlong) Universe::non_oop_word()),
OptoReg::Name(R19_H_num), OptoReg::Name(R19_num));
// Create the call node.
@@ -3625,7 +3600,7 @@ encode %{
const address start_pc = __ pc();
#if defined(ABI_ELFv2)
- address entry= !($meth$$method) ? NULL : (address)$meth$$method;
+ address entry= !($meth$$method) ? nullptr : (address)$meth$$method;
__ call_c(entry, relocInfo::runtime_call_type);
__ post_call_nop();
#else
@@ -3682,13 +3657,13 @@ encode %{
// Create nodes and operands for loading the env pointer.
- if (fd->env() != NULL) {
+ if (fd->env() != nullptr) {
loadConLNodes_Env = loadConLNodesTuple_create(ra_, n_toc, new immLOper((jlong) fd->env()),
OptoReg::Name(R11_H_num), OptoReg::Name(R11_num));
} else {
- loadConLNodes_Env._large_hi = NULL;
- loadConLNodes_Env._large_lo = NULL;
- loadConLNodes_Env._small = NULL;
+ loadConLNodes_Env._large_hi = nullptr;
+ loadConLNodes_Env._large_lo = nullptr;
+ loadConLNodes_Env._small = nullptr;
loadConLNodes_Env._last = new loadConL16Node();
loadConLNodes_Env._last->_opnds[0] = new iRegLdstOper();
loadConLNodes_Env._last->_opnds[1] = new immL16Oper(0);
@@ -3702,7 +3677,7 @@ encode %{
// mtctr node
MachNode *mtctr = new CallLeafDirect_mtctrNode();
- assert(loadConLNodes_Entry._last != NULL, "entry must exist");
+ assert(loadConLNodes_Entry._last != nullptr, "entry must exist");
mtctr->add_req(0, loadConLNodes_Entry._last);
mtctr->_opnds[0] = new iRegLdstOper();
@@ -3722,7 +3697,7 @@ encode %{
call->_guaranteed_safepoint = false;
call->_oop_map = _oop_map;
guarantee(!_jvms, "You must clone the jvms and adapt the offsets by fix_jvms().");
- call->_jvms = NULL;
+ call->_jvms = nullptr;
call->_jvmadj = _jvmadj;
call->_in_rms = _in_rms;
call->_nesting = _nesting;
@@ -3826,7 +3801,7 @@ frame %{
// opcodes. This simplifies the register allocator.
c_return_value %{
assert((ideal_reg >= Op_RegI && ideal_reg <= Op_RegL) ||
- (ideal_reg == Op_RegN && CompressedOops::base() == NULL && CompressedOops::shift() == 0),
+ (ideal_reg == Op_RegN && CompressedOops::base() == nullptr && CompressedOops::shift() == 0),
"only return normal values");
// enum names from opcodes.hpp: Op_Node Op_Set Op_RegN Op_RegI Op_RegP Op_RegF Op_RegD Op_RegL
static int typeToRegLo[Op_RegL+1] = { 0, 0, R3_num, R3_num, R3_num, F1_num, F1_num, R3_num };
@@ -3837,7 +3812,7 @@ frame %{
// Location of compiled Java return values. Same as C
return_value %{
assert((ideal_reg >= Op_RegI && ideal_reg <= Op_RegL) ||
- (ideal_reg == Op_RegN && CompressedOops::base() == NULL && CompressedOops::shift() == 0),
+ (ideal_reg == Op_RegN && CompressedOops::base() == nullptr && CompressedOops::shift() == 0),
"only return normal values");
// enum names from opcodes.hpp: Op_Node Op_Set Op_RegN Op_RegI Op_RegP Op_RegF Op_RegD Op_RegL
static int typeToRegLo[Op_RegL+1] = { 0, 0, R3_num, R3_num, R3_num, F1_num, F1_num, R3_num };
@@ -4110,7 +4085,7 @@ operand immN() %{
interface(CONST_INTER);
%}
-// NULL Pointer Immediate
+// Null Pointer Immediate
operand immN_0() %{
predicate(n->get_narrowcon() == 0);
match(ConN);
@@ -4682,7 +4657,7 @@ operand iRegN2P(iRegNsrc reg) %{
%}
operand iRegN2P_klass(iRegNsrc reg) %{
- predicate(CompressedKlassPointers::base() == NULL && CompressedKlassPointers::shift() == 0);
+ predicate(CompressedKlassPointers::base() == nullptr && CompressedKlassPointers::shift() == 0);
constraint(ALLOC_IN_RC(bits32_reg_ro));
match(DecodeNKlass reg);
format %{ "$reg" %}
@@ -4751,7 +4726,7 @@ operand indirectNarrow(iRegNsrc reg) %{
%}
operand indirectNarrow_klass(iRegNsrc reg) %{
- predicate(CompressedKlassPointers::base() == NULL && CompressedKlassPointers::shift() == 0);
+ predicate(CompressedKlassPointers::base() == nullptr && CompressedKlassPointers::shift() == 0);
constraint(ALLOC_IN_RC(bits64_reg_ro));
match(DecodeNKlass reg);
op_cost(100);
@@ -4780,7 +4755,7 @@ operand indOffset16Narrow(iRegNsrc reg, immL16 offset) %{
%}
operand indOffset16Narrow_klass(iRegNsrc reg, immL16 offset) %{
- predicate(CompressedKlassPointers::base() == NULL && CompressedKlassPointers::shift() == 0);
+ predicate(CompressedKlassPointers::base() == nullptr && CompressedKlassPointers::shift() == 0);
constraint(ALLOC_IN_RC(bits64_reg_ro));
match(AddP (DecodeNKlass reg) offset);
op_cost(100);
@@ -4809,7 +4784,7 @@ operand indOffset16NarrowAlg4(iRegNsrc reg, immL16Alg4 offset) %{
%}
operand indOffset16NarrowAlg4_klass(iRegNsrc reg, immL16Alg4 offset) %{
- predicate(CompressedKlassPointers::base() == NULL && CompressedKlassPointers::shift() == 0);
+ predicate(CompressedKlassPointers::base() == nullptr && CompressedKlassPointers::shift() == 0);
constraint(ALLOC_IN_RC(bits64_reg_ro));
match(AddP (DecodeNKlass reg) offset);
op_cost(100);
@@ -5544,7 +5519,7 @@ instruct loadN2P_unscaled(iRegPdst dst, memory mem) %{
instruct loadN2P_klass_unscaled(iRegPdst dst, memory mem) %{
match(Set dst (DecodeNKlass (LoadNKlass mem)));
- predicate(CompressedKlassPointers::base() == NULL && CompressedKlassPointers::shift() == 0 &&
+ predicate(CompressedKlassPointers::base() == nullptr && CompressedKlassPointers::shift() == 0 &&
_kids[0]->_leaf->as_Load()->is_unordered());
ins_cost(MEMORY_REF_COST);
@@ -5954,7 +5929,7 @@ instruct loadConL_Ex(iRegLdst dst, immL src) %{
postalloc_expand( postalloc_expand_load_long_constant(dst, src, constanttablebase) );
%}
-// Load NULL as compressed oop.
+// Load nullptr as compressed oop.
instruct loadConN0(iRegNdst dst, immN_0 src) %{
match(Set dst src);
ins_cost(DEFAULT_COST);
@@ -6045,9 +6020,9 @@ instruct loadConN_Ex(iRegNdst dst, immN src) %{
MachNode *m1 = new loadConN_hiNode();
MachNode *m2 = new loadConN_loNode();
MachNode *m3 = new clearMs32bNode();
- m1->add_req(NULL);
- m2->add_req(NULL, m1);
- m3->add_req(NULL, m2);
+ m1->add_req(nullptr);
+ m2->add_req(nullptr, m1);
+ m3->add_req(nullptr, m2);
m1->_opnds[0] = op_dst;
m1->_opnds[1] = op_src;
m2->_opnds[0] = op_dst;
@@ -6123,7 +6098,7 @@ instruct loadConNKlass_Ex(iRegNdst dst, immNKlass src) %{
postalloc_expand %{
// Load high bits into register. Sign extended.
MachNode *m1 = new loadConNKlass_hiNode();
- m1->add_req(NULL);
+ m1->add_req(nullptr);
m1->_opnds[0] = op_dst;
m1->_opnds[1] = op_src;
ra_->set_pair(m1->_idx, ra_->get_reg_second(this), ra_->get_reg_first(this));
@@ -6133,7 +6108,7 @@ instruct loadConNKlass_Ex(iRegNdst dst, immNKlass src) %{
if (!Assembler::is_uimm((jlong)CompressedKlassPointers::encode((Klass *)op_src->constant()), 31)) {
// Value might be 1-extended. Mask out these bits.
m2 = new loadConNKlass_maskNode();
- m2->add_req(NULL, m1);
+ m2->add_req(nullptr, m1);
m2->_opnds[0] = op_dst;
m2->_opnds[1] = op_src;
m2->_opnds[2] = op_dst;
@@ -6142,7 +6117,7 @@ instruct loadConNKlass_Ex(iRegNdst dst, immNKlass src) %{
}
MachNode *m3 = new loadConNKlass_loNode();
- m3->add_req(NULL, m2);
+ m3->add_req(nullptr, m2);
m3->_opnds[0] = op_dst;
m3->_opnds[1] = op_src;
m3->_opnds[2] = op_dst;
@@ -6243,7 +6218,7 @@ instruct loadConF(regF dst, immF src, iRegLdst toc) %{
size(4);
ins_encode %{
address float_address = __ float_constant($src$$constant);
- if (float_address == NULL) {
+ if (float_address == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
@@ -6267,7 +6242,7 @@ instruct loadConFComp(regF dst, immF src, iRegLdst toc) %{
FloatRegister Rdst = $dst$$FloatRegister;
Register Rtoc = $toc$$Register;
address float_address = __ float_constant($src$$constant);
- if (float_address == NULL) {
+ if (float_address == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
@@ -6305,7 +6280,7 @@ instruct loadConD(regD dst, immD src, iRegLdst toc) %{
size(4);
ins_encode %{
address float_address = __ double_constant($src$$constant);
- if (float_address == NULL) {
+ if (float_address == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
@@ -6330,7 +6305,7 @@ instruct loadConDComp(regD dst, immD src, iRegLdst toc) %{
FloatRegister Rdst = $dst$$FloatRegister;
Register Rtoc = $toc$$Register;
address float_address = __ double_constant($src$$constant);
- if (float_address == NULL) {
+ if (float_address == nullptr) {
ciEnv::current()->record_out_of_memory_failure();
return;
}
@@ -6632,7 +6607,7 @@ instruct cond_sub_base(iRegNdst dst, flagsRegSrc crx, iRegPsrc src1) %{
predicate(false);
format %{ "BEQ $crx, done\n\t"
- "SUB $dst, $src1, heapbase \t// encode: subtract base if != NULL\n"
+ "SUB $dst, $src1, heapbase \t// encode: subtract base if != nullptr\n"
"done:" %}
ins_encode %{
Label done;
@@ -6701,7 +6676,7 @@ instruct encodeP_not_null_base_null(iRegNdst dst, iRegPsrc src) %{
predicate(CompressedOops::shift() != 0 &&
CompressedOops::base() ==0);
- format %{ "SRDI $dst, $src, #3 \t// encodeP, $src != NULL" %}
+ format %{ "SRDI $dst, $src, #3 \t// encodeP, $src != nullptr" %}
size(4);
ins_encode %{
__ srdi($dst$$Register, $src$$Register, CompressedOops::shift() & 0x3f);
@@ -6762,7 +6737,7 @@ instruct cond_add_base(iRegPdst dst, flagsRegSrc crx, iRegPsrc src) %{
predicate(false);
format %{ "BEQ $crx, done\n\t"
- "ADD $dst, $src, heapbase \t// DecodeN: add oop base if $src != NULL\n"
+ "ADD $dst, $src, heapbase \t// DecodeN: add oop base if $src != nullptr\n"
"done:" %}
ins_encode %{
Label done;
@@ -6850,7 +6825,7 @@ instruct decodeN_Disjoint_notNull_Ex(iRegPdst dst, iRegNsrc src) %{
"RLDIMI $dst, $src, shift, 32-shift \t// decode with disjoint base" %}
postalloc_expand %{
loadBaseNode *n1 = new loadBaseNode();
- n1->add_req(NULL);
+ n1->add_req(nullptr);
n1->_opnds[0] = op_dst;
decodeN_mergeDisjointNode *n2 = new decodeN_mergeDisjointNode();
@@ -6882,7 +6857,7 @@ instruct decodeN_Disjoint_isel_Ex(iRegPdst dst, iRegNsrc src, flagsReg crx) %{
format %{ "DecodeN $dst, $src \t// decode with disjoint base using isel" %}
postalloc_expand %{
loadBaseNode *n1 = new loadBaseNode();
- n1->add_req(NULL);
+ n1->add_req(nullptr);
n1->_opnds[0] = op_dst;
cmpN_reg_imm0Node *n_compare = new cmpN_reg_imm0Node();
@@ -6929,7 +6904,7 @@ instruct decodeN_notNull_addBase_Ex(iRegPdst dst, iRegNsrc src) %{
CompressedOops::base() != 0);
ins_cost(2 * DEFAULT_COST);
- format %{ "DecodeN $dst, $src \t// $src != NULL, postalloc expanded" %}
+ format %{ "DecodeN $dst, $src \t// $src != nullptr, postalloc expanded" %}
postalloc_expand( postalloc_expand_decode_oop_not_null(dst, src));
%}
@@ -7086,7 +7061,7 @@ instruct decodeNKlass_notNull_addBase_Ex(iRegPdst dst, iRegLsrc base, iRegNsrc s
//effect(kill src); // We need a register for the immediate result after shifting.
predicate(false);
- format %{ "DecodeNKlass $dst = $base + ($src << 3) \t// $src != NULL, postalloc expanded" %}
+ format %{ "DecodeNKlass $dst = $base + ($src << 3) \t// $src != nullptr, postalloc expanded" %}
postalloc_expand %{
decodeNKlass_add_baseNode *n1 = new decodeNKlass_add_baseNode();
n1->add_req(n_region, n_base, n_src);
@@ -7115,7 +7090,7 @@ instruct decodeNKlass_notNull_addBase_ExEx(iRegPdst dst, iRegNsrc src) %{
// predicate(CompressedKlassPointers::shift() != 0 &&
// CompressedKlassPointers::base() != 0);
- //format %{ "DecodeNKlass $dst, $src \t// $src != NULL, expanded" %}
+ //format %{ "DecodeNKlass $dst, $src \t// $src != nullptr, expanded" %}
ins_cost(DEFAULT_COST*2); // Don't count constant.
expand %{
@@ -7602,7 +7577,7 @@ instruct compareAndSwapL_regP_regL_regL(iRegIdst res, iRegPdst mem_ptr, iRegLsrc
// CmpxchgX sets CCR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
__ cmpxchgd(CCR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
- $res$$Register, NULL, true);
+ $res$$Register, nullptr, true);
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ isync();
} else {
@@ -7621,7 +7596,7 @@ instruct compareAndSwapP_regP_regP_regP(iRegIdst res, iRegPdst mem_ptr, iRegPsrc
// CmpxchgX sets CCR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
__ cmpxchgd(CCR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
- $res$$Register, NULL, true);
+ $res$$Register, nullptr, true);
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ isync();
} else {
@@ -7815,7 +7790,7 @@ instruct weakCompareAndSwapL_regP_regL_regL(iRegIdst res, iRegPdst mem_ptr, iReg
// value is never passed to caller.
__ cmpxchgd(CCR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
MacroAssembler::MemBarNone,
- MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, NULL, true, /*weak*/ true);
+ MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
%}
ins_pipe(pipe_class_default);
%}
@@ -7831,7 +7806,7 @@ instruct weakCompareAndSwapL_acq_regP_regL_regL(iRegIdst res, iRegPdst mem_ptr,
// value is never passed to caller.
__ cmpxchgd(CCR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
support_IRIW_for_not_multiple_copy_atomic_cpu ? MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter,
- MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, NULL, true, /*weak*/ true);
+ MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
%}
ins_pipe(pipe_class_default);
%}
@@ -7845,7 +7820,7 @@ instruct weakCompareAndSwapP_regP_regP_regP(iRegIdst res, iRegPdst mem_ptr, iReg
// CmpxchgX sets CCR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
__ cmpxchgd(CCR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
MacroAssembler::MemBarNone,
- MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, NULL, true, /*weak*/ true);
+ MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
%}
ins_pipe(pipe_class_default);
%}
@@ -7861,7 +7836,7 @@ instruct weakCompareAndSwapP_acq_regP_regP_regP(iRegIdst res, iRegPdst mem_ptr,
// value is never passed to caller.
__ cmpxchgd(CCR0, R0, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
support_IRIW_for_not_multiple_copy_atomic_cpu ? MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter,
- MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, NULL, true, /*weak*/ true);
+ MacroAssembler::cmpxchgx_hint_atomic_update(), $res$$Register, nullptr, true, /*weak*/ true);
%}
ins_pipe(pipe_class_default);
%}
@@ -8081,7 +8056,7 @@ instruct compareAndExchangeL_regP_regL_regL(iRegLdst res, iRegPdst mem_ptr, iReg
// CmpxchgX sets CCR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
__ cmpxchgd(CCR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
- noreg, NULL, true);
+ noreg, nullptr, true);
%}
ins_pipe(pipe_class_default);
%}
@@ -8095,7 +8070,7 @@ instruct compareAndExchangeL_acq_regP_regL_regL(iRegLdst res, iRegPdst mem_ptr,
// CmpxchgX sets CCR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
__ cmpxchgd(CCR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
- noreg, NULL, true);
+ noreg, nullptr, true);
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ isync();
} else {
@@ -8116,7 +8091,7 @@ instruct compareAndExchangeP_regP_regP_regP(iRegPdst res, iRegPdst mem_ptr, iReg
// CmpxchgX sets CCR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
__ cmpxchgd(CCR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
- noreg, NULL, true);
+ noreg, nullptr, true);
%}
ins_pipe(pipe_class_default);
%}
@@ -8131,7 +8106,7 @@ instruct compareAndExchangeP_acq_regP_regP_regP(iRegPdst res, iRegPdst mem_ptr,
// CmpxchgX sets CCR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
__ cmpxchgd(CCR0, $res$$Register, $src1$$Register, $src2$$Register, $mem_ptr$$Register,
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
- noreg, NULL, true);
+ noreg, nullptr, true);
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ isync();
} else {
@@ -12043,9 +12018,9 @@ instruct branch(label labl) %{
Label d; // dummy
__ bind(d);
Label* p = $labl$$label;
- // `p' is `NULL' when this encoding class is used only to
+ // `p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
- Label& l = (NULL == p)? d : *(p);
+ Label& l = (nullptr == p)? d : *(p);
__ b(l);
%}
ins_pipe(pipe_class_default);
@@ -12139,7 +12114,7 @@ instruct partialSubtypeCheck(iRegPdst result, iRegP_N2P subklass, iRegP_N2P supe
format %{ "PartialSubtypeCheck $result = ($subklass instanceOf $superklass) tmp: $tmp_klass, $tmp_arrayptr" %}
ins_encode %{
__ check_klass_subtype_slow_path($subklass$$Register, $superklass$$Register, $tmp_arrayptr$$Register,
- $tmp_klass$$Register, NULL, $result$$Register);
+ $tmp_klass$$Register, nullptr, $result$$Register);
%}
ins_pipe(pipe_class_default);
%}
@@ -12147,6 +12122,7 @@ instruct partialSubtypeCheck(iRegPdst result, iRegP_N2P subklass, iRegP_N2P supe
// inlined locking and unlocking
instruct cmpFastLock(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1, iRegPdst tmp2) %{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set crx (FastLock oop box));
effect(TEMP tmp1, TEMP tmp2);
@@ -12162,6 +12138,7 @@ instruct cmpFastLock(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1,
%}
instruct cmpFastUnlock(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1, iRegPdst tmp2, iRegPdst tmp3) %{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set crx (FastUnlock oop box));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3);
@@ -12176,6 +12153,38 @@ instruct cmpFastUnlock(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp
ins_pipe(pipe_class_compare);
%}
+instruct cmpFastLockLightweight(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1, iRegPdst tmp2) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set crx (FastLock oop box));
+ effect(TEMP tmp1, TEMP tmp2);
+
+ format %{ "FASTLOCK $oop, $box, $tmp1, $tmp2" %}
+ ins_encode %{
+ __ fast_lock_lightweight($crx$$CondRegister, $oop$$Register, $box$$Register,
+ $tmp1$$Register, $tmp2$$Register, /*tmp3*/ R0);
+ // If locking was successful, crx should indicate 'EQ'.
+ // The compiler generates a branch to the runtime call to
+ // _complete_monitor_locking_Java for the case where crx is 'NE'.
+ %}
+ ins_pipe(pipe_class_compare);
+%}
+
+instruct cmpFastUnlockLightweight(flagsRegCR0 crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1, iRegPdst tmp2, iRegPdst tmp3) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set crx (FastUnlock oop box));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3);
+
+ format %{ "FASTUNLOCK $oop, $box, $tmp1, $tmp2" %}
+ ins_encode %{
+ __ fast_unlock_lightweight($crx$$CondRegister, $oop$$Register, $box$$Register,
+ $tmp1$$Register, $tmp2$$Register, $tmp3$$Register);
+ // If unlocking was successful, crx should indicate 'EQ'.
+ // The compiler generates a branch to the runtime call to
+ // _complete_monitor_unlocking_Java for the case where crx is 'NE'.
+ %}
+ ins_pipe(pipe_class_compare);
+%}
+
// Align address.
instruct align_addr(iRegPdst dst, iRegPsrc src, immLnegpow2 mask) %{
match(Set dst (CastX2P (AndL (CastP2X src) mask)));
@@ -12318,21 +12327,6 @@ instruct string_equalsL(rarg1RegP str1, rarg2RegP str2, rarg3RegI cnt, iRegIdst
ins_pipe(pipe_class_default);
%}
-instruct string_equalsU(rarg1RegP str1, rarg2RegP str2, rarg3RegI cnt, iRegIdst result,
- iRegIdst tmp, regCTR ctr, flagsRegCR0 cr0) %{
- predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
- match(Set result (StrEquals (Binary str1 str2) cnt));
- effect(TEMP_DEF result, USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP tmp, KILL ctr, KILL cr0);
- ins_cost(300);
- format %{ "String Equals char[] $str1,$str2,$cnt -> $result \t// KILL $tmp" %}
- ins_encode %{
- __ array_equals(false, $str1$$Register, $str2$$Register,
- $cnt$$Register, $tmp$$Register,
- $result$$Register, false /* byte */);
- %}
- ins_pipe(pipe_class_default);
-%}
-
instruct array_equalsB(rarg1RegP ary1, rarg2RegP ary2, iRegIdst result,
iRegIdst tmp1, iRegIdst tmp2, regCTR ctr, flagsRegCR0 cr0, flagsRegCR0 cr1) %{
predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
@@ -12670,7 +12664,7 @@ instruct indexOf_U(iRegIdst result, iRegPsrc haystack, rscratch1RegI haycnt, iRe
ins_encode %{
__ string_indexof($result$$Register,
$haystack$$Register, $haycnt$$Register,
- $needle$$Register, NULL, $needlecnt$$Register, 0, // needlecnt not constant.
+ $needle$$Register, nullptr, $needlecnt$$Register, 0, // needlecnt not constant.
$tmp1$$Register, $tmp2$$Register, $tmp3$$Register, $tmp4$$Register, StrIntrinsicNode::UU);
%}
ins_pipe(pipe_class_compare);
@@ -12691,7 +12685,7 @@ instruct indexOf_L(iRegIdst result, iRegPsrc haystack, rscratch1RegI haycnt, iRe
ins_encode %{
__ string_indexof($result$$Register,
$haystack$$Register, $haycnt$$Register,
- $needle$$Register, NULL, $needlecnt$$Register, 0, // needlecnt not constant.
+ $needle$$Register, nullptr, $needlecnt$$Register, 0, // needlecnt not constant.
$tmp1$$Register, $tmp2$$Register, $tmp3$$Register, $tmp4$$Register, StrIntrinsicNode::LL);
%}
ins_pipe(pipe_class_compare);
@@ -12712,7 +12706,7 @@ instruct indexOf_UL(iRegIdst result, iRegPsrc haystack, rscratch1RegI haycnt, iR
ins_encode %{
__ string_indexof($result$$Register,
$haystack$$Register, $haycnt$$Register,
- $needle$$Register, NULL, $needlecnt$$Register, 0, // needlecnt not constant.
+ $needle$$Register, nullptr, $needlecnt$$Register, 0, // needlecnt not constant.
$tmp1$$Register, $tmp2$$Register, $tmp3$$Register, $tmp4$$Register, StrIntrinsicNode::UL);
%}
ins_pipe(pipe_class_compare);
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 3382df355da79..66b19794b057e 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -27,7 +27,6 @@
#include "asm/macroAssembler.inline.hpp"
#include "code/debugInfoRec.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "frame_ppc.hpp"
#include "compiler/oopMap.hpp"
@@ -35,7 +34,6 @@
#include "interpreter/interpreter.hpp"
#include "interpreter/interp_masm.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/continuation.hpp"
@@ -1174,8 +1172,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
BLOCK_COMMENT("c2i unverified entry");
c2i_unverified_entry = __ pc();
- // inline_cache contains a compiledICHolder
- const Register ic = R19_method;
+ // inline_cache contains a CompiledICData
+ const Register ic = R19_inline_cache_reg;
const Register ic_klass = R11_scratch1;
const Register receiver_klass = R12_scratch2;
const Register code = R21_tmp1;
@@ -1186,45 +1184,10 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
Label call_interpreter;
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()),
- "klass offset should reach into any page");
- // Check for null argument if we don't have implicit null checks.
- if (!ImplicitNullChecks || !os::zero_page_read_protected()) {
- if (TrapBasedNullChecks) {
- __ trap_null_check(R3_ARG1);
- } else {
- Label valid;
- __ cmpdi(CCR0, R3_ARG1, 0);
- __ bne_predict_taken(CCR0, valid);
- // We have a null argument, branch to ic_miss_stub.
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- __ BIND(valid);
- }
- }
- // Assume argument is not null, load klass from receiver.
- __ load_klass(receiver_klass, R3_ARG1);
-
- __ ld(ic_klass, CompiledICHolder::holder_klass_offset(), ic);
-
- if (TrapBasedICMissChecks) {
- __ trap_ic_miss_check(receiver_klass, ic_klass);
- } else {
- Label valid;
- __ cmpd(CCR0, receiver_klass, ic_klass);
- __ beq_predict_taken(CCR0, valid);
- // We have an unexpected klass, branch to ic_miss_stub.
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- __ BIND(valid);
- }
-
+ __ ic_check(4 /* end_alignment */);
+ __ ld(R19_method, CompiledICData::speculated_method_offset(), ic);
// Argument is valid and klass is as expected, continue.
- // Extract method from inline cache, verified entry point needs it.
- __ ld(R19_method, CompiledICHolder::holder_metadata_offset(), ic);
- assert(R19_method == ic, "the inline cache register is dead here");
-
__ ld(code, method_(code));
__ cmpdi(CCR0, code, 0);
__ ld(ientry, method_(interpreter_entry)); // preloaded
@@ -1798,7 +1761,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
// static stub for the call above
CodeBuffer* cbuf = masm->code_section()->outer();
- stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, c2i_call_pc);
+ stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, c2i_call_pc);
guarantee(stub != nullptr, "no space for static stub");
}
@@ -1891,7 +1854,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
// static stub for the call above
CodeBuffer* cbuf = masm->code_section()->outer();
- stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, call_pc);
+ stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, call_pc);
guarantee(stub != nullptr, "no space for static stub");
}
@@ -2039,6 +2002,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
in_ByteSize(-1),
oop_maps,
exception_offset);
+ if (nm == nullptr) return nm;
if (method->is_continuation_enter_intrinsic()) {
ContinuationEntry::set_enter_code(nm, interpreted_entry_offset);
} else if (method->is_continuation_yield_intrinsic()) {
@@ -2187,7 +2151,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
intptr_t frame_done_pc;
intptr_t oopmap_pc;
- Label ic_miss;
Label handle_pending_exception;
Register r_callers_sp = R21;
@@ -2211,19 +2174,9 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
// Check ic: object class == cached class?
if (!method_is_static) {
- Register ic = R19_inline_cache_reg;
- Register receiver_klass = r_temp_1;
-
- __ cmpdi(CCR0, R3_ARG1, 0);
- __ beq(CCR0, ic_miss);
- __ verify_oop(R3_ARG1, FILE_AND_LINE);
- __ load_klass(receiver_klass, R3_ARG1);
-
- __ cmpd(CCR0, receiver_klass, ic);
- __ bne(CCR0, ic_miss);
+ __ ic_check(4 /* end_alignment */);
}
-
// Generate the Verified Entry Point (VEP).
// --------------------------------------------------------------------------
vep_start_pc = (intptr_t)__ pc();
@@ -2403,8 +2356,13 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
__ addi(r_box, R1_SP, lock_offset);
// Try fastpath for locking.
- // fast_lock kills r_temp_1, r_temp_2, r_temp_3.
- __ compiler_fast_lock_object(CCR0, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3);
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ // fast_lock kills r_temp_1, r_temp_2, r_temp_3.
+ __ compiler_fast_lock_lightweight_object(CCR0, r_oop, r_temp_1, r_temp_2, r_temp_3);
+ } else {
+ // fast_lock kills r_temp_1, r_temp_2, r_temp_3.
+ __ compiler_fast_lock_object(CCR0, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3);
+ }
__ beq(CCR0, locked);
// None of the above fast optimizations worked so we have to get into the
@@ -2614,7 +2572,11 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
__ addi(r_box, R1_SP, lock_offset);
// Try fastpath for unlocking.
- __ compiler_fast_unlock_object(CCR0, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3);
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ __ compiler_fast_unlock_lightweight_object(CCR0, r_oop, r_temp_1, r_temp_2, r_temp_3);
+ } else {
+ __ compiler_fast_unlock_object(CCR0, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3);
+ }
__ beq(CCR0, done);
// Save and restore any potential method result value around the unlocking operation.
@@ -2703,16 +2665,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
__ b64_patchable((address)StubRoutines::forward_exception_entry(),
relocInfo::runtime_call_type);
- // Handler for a cache miss (out-of-line).
- // --------------------------------------------------------------------------
-
- if (!method_is_static) {
- __ bind(ic_miss);
-
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- }
-
// Done.
// --------------------------------------------------------------------------
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index e26f03f52d802..094757ad3e16c 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -3643,8 +3643,6 @@ class StubGenerator: public StubCodeGenerator {
#define VALID_B64 0x80
#define VB64(x) (VALID_B64 | x)
-#define VEC_ALIGN __attribute__ ((aligned(16)))
-
#define BLK_OFFSETOF(x) (offsetof(constant_block, x))
// In little-endian mode, the lxv instruction loads the element at EA into
@@ -3681,7 +3679,7 @@ class StubGenerator: public StubCodeGenerator {
unsigned char pack_permute_val[16];
} constant_block;
- static const constant_block VEC_ALIGN const_block = {
+ alignas(16) static const constant_block const_block = {
.offsetLUT_val = {
ARRAY_TO_LXV_ORDER(
@@ -4263,7 +4261,7 @@ class StubGenerator: public StubCodeGenerator {
unsigned char base64_48_63_URL_val[16];
} constant_block;
- static const constant_block VEC_ALIGN const_block = {
+ alignas(16) static const constant_block const_block = {
.expand_permute_val = {
ARRAY_TO_LXV_ORDER(
0, 4, 5, 6,
diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
index 84ecfc4f934e7..86353aefb36ef 100644
--- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -3803,16 +3803,15 @@ void TemplateTable::_new() {
__ sldi(Roffset, Rindex, LogBytesPerWord);
__ load_resolved_klass_at_offset(Rcpool, Roffset, RinstanceKlass);
- // Make sure klass is fully initialized and get instance_size.
- __ lbz(Rscratch, in_bytes(InstanceKlass::init_state_offset()), RinstanceKlass);
+ // Make sure klass is initialized.
+ assert(VM_Version::supports_fast_class_init_checks(), "Optimization requires support for fast class initialization checks");
+ __ clinit_barrier(RinstanceKlass, R16_thread, nullptr /*L_fast_path*/, &Lslow_case);
+
__ lwz(Rinstance_size, in_bytes(Klass::layout_helper_offset()), RinstanceKlass);
- __ cmpdi(CCR1, Rscratch, InstanceKlass::fully_initialized);
// Make sure klass does not have has_finalizer, or is abstract, or interface or java/lang/Class.
__ andi_(R0, Rinstance_size, Klass::_lh_instance_slow_path_bit); // slow path bit equals 0?
-
- __ crnand(CCR0, Assembler::equal, CCR1, Assembler::equal); // slow path bit set or not fully initialized?
- __ beq(CCR0, Lslow_case);
+ __ bne(CCR0, Lslow_case);
// --------------------------------------------------------------------------
// Fast case:
diff --git a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
index a1c8f46ce2d57..b60fd4f16d163 100644
--- a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
+++ b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
@@ -243,10 +243,14 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ load_const_optimized(R19_method, (intptr_t)entry);
__ std(R19_method, in_bytes(JavaThread::callee_target_offset()), R16_thread);
+ __ push_cont_fastpath();
+
__ ld(call_target_address, in_bytes(Method::from_compiled_offset()), R19_method);
__ mtctr(call_target_address);
__ bctrl();
+ __ pop_cont_fastpath();
+
// return value shuffle
if (!needs_return_buffer) {
// CallArranger can pick a return type that goes in the same reg for both CCs.
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.hpp b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
index a5831ef1590ad..0efde131277b2 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.hpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
@@ -94,6 +94,7 @@ class VM_Version: public Abstract_VM_Version {
// PPC64 supports fast class initialization checks
static bool supports_fast_class_init_checks() { return true; }
constexpr static bool supports_stack_watermark_barrier() { return true; }
+ constexpr static bool supports_recursive_lightweight_locking() { return true; }
static bool is_determine_features_test_running() { return _is_determine_features_test_running; }
// CPU instruction support
diff --git a/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp b/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
index fe4eb3df8f12f..28ba04d833bed 100644
--- a/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
@@ -25,10 +25,10 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_ppc.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klass.inline.hpp"
#include "oops/klassVtable.hpp"
@@ -181,13 +181,13 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
__ load_klass_check_null(rcvr_klass, R3_ARG1);
// Receiver subtype check against REFC.
- __ ld(interface, CompiledICHolder::holder_klass_offset(), R19_method);
+ __ ld(interface, CompiledICData::itable_refc_klass_offset(), R19_method);
__ lookup_interface_method(rcvr_klass, interface, noreg,
R0, tmp1, tmp2,
L_no_such_interface, /*return_method=*/ false);
// Get Method* and entrypoint for compiler
- __ ld(interface, CompiledICHolder::holder_metadata_offset(), R19_method);
+ __ ld(interface, CompiledICData::itable_defc_klass_offset(), R19_method);
__ lookup_interface_method(rcvr_klass, interface, itable_index,
R19_method, tmp1, tmp2,
L_no_such_interface, /*return_method=*/ true);
diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp
index f44840d9f8cc7..38804ff2ed6a8 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp
@@ -815,6 +815,8 @@ enum operand_size { int8, int16, int32, uint32, int64 };
INSN(fsqrt_s, 0b1010011, 0b00000, 0b0101100);
INSN(fsqrt_d, 0b1010011, 0b00000, 0b0101101);
+ INSN(fcvt_s_h, 0b1010011, 0b00010, 0b0100000);
+ INSN(fcvt_h_s, 0b1010011, 0b00000, 0b0100010);
INSN(fcvt_s_d, 0b1010011, 0b00001, 0b0100000);
INSN(fcvt_d_s, 0b1010011, 0b00000, 0b0100001);
#undef INSN
@@ -1071,6 +1073,7 @@ enum operand_size { int8, int16, int32, uint32, int64 };
emit(insn); \
}
+ INSN(fmv_h_x, 0b1010011, 0b000, 0b00000, 0b1111010);
INSN(fmv_w_x, 0b1010011, 0b000, 0b00000, 0b1111000);
INSN(fmv_d_x, 0b1010011, 0b000, 0b00000, 0b1111001);
@@ -1108,8 +1111,10 @@ enum fclass_mask {
emit(insn); \
}
+ INSN(fclass_h, 0b1010011, 0b001, 0b00000, 0b1110010);
INSN(fclass_s, 0b1010011, 0b001, 0b00000, 0b1110000);
INSN(fclass_d, 0b1010011, 0b001, 0b00000, 0b1110001);
+ INSN(fmv_x_h, 0b1010011, 0b000, 0b00000, 0b1110010);
INSN(fmv_x_w, 0b1010011, 0b000, 0b00000, 0b1110000);
INSN(fmv_x_d, 0b1010011, 0b000, 0b00000, 0b1110001);
@@ -2651,7 +2656,7 @@ enum Nf {
}
bool do_compress_zcb(Register reg1 = noreg, Register reg2 = noreg) const {
- return do_compress() && VM_Version::ext_Zcb.enabled() &&
+ return do_compress() && UseZcb &&
(reg1 == noreg || reg1->is_compressed_valid()) && (reg2 == noreg || reg2->is_compressed_valid());
}
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index 0bbf3771a04bc..5d0fa3fad3cec 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -51,7 +51,6 @@
#endif
NEEDS_CLEANUP // remove this definitions ?
-const Register IC_Klass = t1; // where the IC klass is cached
const Register SYNC_header = x10; // synchronization header
const Register SHIFT_count = x10; // where count for shift operations must be
@@ -265,26 +264,7 @@ void LIR_Assembler::osr_entry() {
// inline cache check; done before the frame is built.
int LIR_Assembler::check_icache() {
- Register receiver = FrameMap::receiver_opr->as_register();
- Register ic_klass = IC_Klass;
- int start_offset = __ offset();
- Label dont;
- __ inline_cache_check(receiver, ic_klass, dont);
-
- // if icache check fails, then jump to runtime routine
- // Note: RECEIVER must still contain the receiver!
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- // We align the verified entry point unless the method body
- // (including its inline cache check) will fit in a single 64-byte
- // icache line.
- if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
- // force alignment after the cache check.
- __ align(CodeEntryAlignment);
- }
-
- __ bind(dont);
- return start_offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::jobject2reg(jobject o, Register reg) {
@@ -1040,7 +1020,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -1398,7 +1378,7 @@ void LIR_Assembler::emit_static_call_stub() {
__ relocate(static_stub_Relocation::spec(call_pc));
__ emit_static_call_stub();
- assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
+ assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
<= call_stub_size(), "stub too big");
__ end_a_stub();
}
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
index b088498e6fc08..ce23213776c08 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
@@ -68,7 +68,7 @@ friend class ArrayCopyStub;
enum {
// See emit_static_call_stub for detail
- // CompiledStaticCall::to_interp_stub_size() (14) + CompiledStaticCall::to_trampoline_stub_size() (1 + 3 + address)
+ // CompiledDirectCall::to_interp_stub_size() (14) + CompiledDirectCall::to_trampoline_stub_size() (1 + 3 + address)
_call_stub_size = 14 * NativeInstruction::instruction_size +
(NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size),
// See emit_exception_handler for detail
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
index 6c1dce0de1598..f4c05b9efd651 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -69,13 +69,12 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
bnez(temp, slow_case, true /* is_far */);
}
- // Load object header
- ld(hdr, Address(obj, hdr_offset));
-
if (LockingMode == LM_LIGHTWEIGHT) {
lightweight_lock(obj, hdr, temp, t1, slow_case);
} else if (LockingMode == LM_LEGACY) {
Label done;
+ // Load object header
+ ld(hdr, Address(obj, hdr_offset));
// and mark it as unlocked
ori(hdr, hdr, markWord::unlocked_value);
// save unlocked object header into the displaced header location on the stack
@@ -134,9 +133,6 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
verify_oop(obj);
if (LockingMode == LM_LIGHTWEIGHT) {
- ld(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
- test_bit(temp, hdr, exact_log2(markWord::monitor_value));
- bnez(temp, slow_case, /* is_far */ true);
lightweight_unlock(obj, hdr, temp, t1, slow_case);
} else if (LockingMode == LM_LEGACY) {
// test if object header is pointing to the displaced header, and if so, restore
@@ -280,7 +276,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, tmp1, tmp2, klass);
// determine alignment mask
@@ -292,7 +288,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
const Register arr_size = tmp2; // okay to be the same
// align object end
- mv(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ mv(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
shadd(arr_size, len, arr_size, t0, f);
andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask);
@@ -300,9 +296,20 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
initialize_header(obj, klass, len, tmp1, tmp2);
+ // Clear leading 4 bytes, if necessary.
+ // TODO: This could perhaps go into initialize_body() and also clear the leading 4 bytes
+ // for non-array objects, thereby replacing the klass-gap clearing code in initialize_header().
+ int base_offset = base_offset_in_bytes;
+ if (!is_aligned(base_offset, BytesPerWord)) {
+ assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+ sw(zr, Address(obj, base_offset));
+ base_offset += BytesPerInt;
+ }
+ assert(is_aligned(base_offset, BytesPerWord), "must be word-aligned");
+
// clear rest of allocated space
const Register len_zero = len;
- initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
+ initialize_body(obj, arr_size, base_offset, len_zero);
membar(MacroAssembler::StoreStore);
@@ -314,15 +321,6 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
verify_oop(obj);
}
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache, Label &L) {
- verify_oop(receiver);
- // explicit null check not needed since load from [klass_offset] causes a trap
- // check against inline cache
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
- assert_different_registers(receiver, iCache, t0, t2);
- cmp_klass(receiver, iCache, t0, t2 /* call-clobbered t2 as a tmp */, L);
-}
-
void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
// Make sure there is enough stack space for this method's activation.
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
index b737a438511c8..2d7f8d7485d4f 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -101,7 +101,7 @@ using MacroAssembler::null_check;
// header_size: size of object header in words
// f : element scale factor
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case);
+ void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
int rsp_offset() const { return _rsp_offset; }
diff --git a/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp b/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp
index b76163a30841d..9fa8939837a85 100644
--- a/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp
@@ -37,7 +37,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_riscv.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_riscv.hpp"
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index 711eb2100912b..767d4111e76a6 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -32,6 +32,7 @@
#include "opto/output.hpp"
#include "opto/subnode.hpp"
#include "runtime/stubRoutines.hpp"
+#include "utilities/globalDefinitions.hpp"
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -51,30 +52,35 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg,
Register box = boxReg;
Register disp_hdr = tmp1Reg;
Register tmp = tmp2Reg;
- Label cont;
Label object_has_monitor;
- Label count, no_count;
+ // Finish fast lock successfully. MUST branch to with flag == 0
+ Label locked;
+ // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
+ Label slow_path;
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
assert_different_registers(oop, box, tmp, disp_hdr, flag, tmp3Reg, t0);
+ mv(flag, 1);
+
// Load markWord from object into displaced_header.
ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
if (DiagnoseSyncOnValueBasedClasses != 0) {
- load_klass(flag, oop);
- lwu(flag, Address(flag, Klass::access_flags_offset()));
- test_bit(flag, flag, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
- bnez(flag, cont, true /* is_far */);
+ load_klass(tmp, oop);
+ lwu(tmp, Address(tmp, Klass::access_flags_offset()));
+ test_bit(tmp, tmp, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
+ bnez(tmp, slow_path);
}
// Check for existing monitor
- test_bit(t0, disp_hdr, exact_log2(markWord::monitor_value));
- bnez(t0, object_has_monitor);
+ test_bit(tmp, disp_hdr, exact_log2(markWord::monitor_value));
+ bnez(tmp, object_has_monitor);
if (LockingMode == LM_MONITOR) {
- mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
- j(cont);
- } else if (LockingMode == LM_LEGACY) {
+ j(slow_path);
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Set tmp to be (markWord of object | UNLOCK_VALUE).
ori(tmp, disp_hdr, markWord::unlocked_value);
@@ -84,77 +90,72 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg,
// Compare object markWord with an unlocked value (tmp) and if
// equal exchange the stack address of our box with object markWord.
// On failure disp_hdr contains the possibly locked markWord.
- cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64, Assembler::aq,
- Assembler::rl, /*result*/disp_hdr);
- mv(flag, zr);
- beq(disp_hdr, tmp, cont); // prepare zero flag and goto cont if we won the cas
+ cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64,
+ Assembler::aq, Assembler::rl, /*result*/disp_hdr);
+ beq(disp_hdr, tmp, locked);
assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
// If the compare-and-exchange succeeded, then we found an unlocked
- // object, will have now locked it will continue at label cont
+ // object, will have now locked it will continue at label locked
// We did not see an unlocked object so try the fast recursive case.
// Check if the owner is self by comparing the value in the
// markWord of object (disp_hdr) with the stack pointer.
sub(disp_hdr, disp_hdr, sp);
mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place));
- // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto cont,
+ // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto label locked,
// hence we can store 0 as the displaced header in the box, which indicates that it is a
// recursive lock.
andr(tmp/*==0?*/, disp_hdr, tmp);
sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
- mv(flag, tmp); // we can use the value of tmp as the result here
- j(cont);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "");
- Label slow;
- lightweight_lock(oop, disp_hdr, tmp, tmp3Reg, slow);
-
- // Indicate success on completion.
- mv(flag, zr);
- j(count);
- bind(slow);
- mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path
- j(no_count);
+ beqz(tmp, locked);
+ j(slow_path);
}
// Handle existing monitor.
bind(object_has_monitor);
- // The object's monitor m is unlocked iff m->owner == NULL,
+ // The object's monitor m is unlocked iff m->owner == nullptr,
// otherwise m->owner may contain a thread or a stack address.
//
- // Try to CAS m->owner from NULL to current thread.
+ // Try to CAS m->owner from null to current thread.
add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value));
- cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, Assembler::aq,
- Assembler::rl, /*result*/flag); // cas succeeds if flag == zr(expected)
-
- if (LockingMode != LM_LIGHTWEIGHT) {
- // Store a non-null value into the box to avoid looking like a re-entrant
- // lock. The fast-path monitor unlock code checks for
- // markWord::monitor_value so use markWord::unused_mark which has the
- // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
- mv(tmp, (address)markWord::unused_mark().value());
- sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
- }
+ cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64,
+ Assembler::aq, Assembler::rl, /*result*/tmp3Reg); // cas succeeds if tmp3Reg == zr(expected)
+
+ // Store a non-null value into the box to avoid looking like a re-entrant
+ // lock. The fast-path monitor unlock code checks for
+ // markWord::monitor_value so use markWord::unused_mark which has the
+ // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
+ mv(tmp, (address)markWord::unused_mark().value());
+ sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
- beqz(flag, cont); // CAS success means locking succeeded
+ beqz(tmp3Reg, locked); // CAS success means locking succeeded
- bne(flag, xthread, cont); // Check for recursive locking
+ bne(tmp3Reg, xthread, slow_path); // Check for recursive locking
// Recursive lock case
- mv(flag, zr);
- increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1, t0, tmp);
+ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1, tmp2Reg, tmp3Reg);
- bind(cont);
- // zero flag indicates success
- // non-zero flag indicates failure
- bnez(flag, no_count);
+ bind(locked);
+ mv(flag, zr);
+ increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2Reg, tmp3Reg);
- bind(count);
- increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp);
+#ifdef ASSERT
+ // Check that locked label is reached with flag == 0.
+ Label flag_correct;
+ beqz(flag, flag_correct);
+ stop("Fast Lock Flag != 0");
+#endif
- bind(no_count);
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with flag != 0.
+ bnez(flag, flag_correct);
+ stop("Fast Lock Flag == 0");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of flag (0 vs !0) to determine the continuation.
}
void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
@@ -165,19 +166,23 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
Register box = boxReg;
Register disp_hdr = tmp1Reg;
Register tmp = tmp2Reg;
- Label cont;
Label object_has_monitor;
- Label count, no_count;
+ // Finish fast lock successfully. MUST branch to with flag == 0
+ Label unlocked;
+ // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
+ Label slow_path;
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
assert_different_registers(oop, box, tmp, disp_hdr, flag, t0);
+ mv(flag, 1);
+
if (LockingMode == LM_LEGACY) {
// Find the lock address and load the displaced header from the stack.
ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
// If the displaced header is 0, we have a recursive unlock.
- mv(flag, disp_hdr);
- beqz(disp_hdr, cont);
+ beqz(disp_hdr, unlocked);
}
// Handle existing monitor.
@@ -186,28 +191,17 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
bnez(t0, object_has_monitor);
if (LockingMode == LM_MONITOR) {
- mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
- j(cont);
- } else if (LockingMode == LM_LEGACY) {
+ j(slow_path);
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Check if it is still a light weight lock, this is true if we
// see the stack address of the basicLock in the markWord of the
// object.
- cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64, Assembler::relaxed,
- Assembler::rl, /*result*/tmp);
- xorr(flag, box, tmp); // box == tmp if cas succeeds
- j(cont);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "");
- Label slow;
- lightweight_unlock(oop, tmp, box, disp_hdr, slow);
-
- // Indicate success on completion.
- mv(flag, zr);
- j(count);
- bind(slow);
- mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path
- j(no_count);
+ cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64,
+ Assembler::relaxed, Assembler::rl, /*result*/tmp);
+ beq(box, tmp, unlocked); // box == tmp if cas succeeds
+ j(slow_path);
}
assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
@@ -217,17 +211,6 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
add(tmp, tmp, -(int)markWord::monitor_value); // monitor
- if (LockingMode == LM_LIGHTWEIGHT) {
- // If the owner is anonymous, we need to fix it -- in an outline stub.
- Register tmp2 = disp_hdr;
- ld(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
- test_bit(t0, tmp2, exact_log2(ObjectMonitor::ANONYMOUS_OWNER));
- C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
- Compile::current()->output()->add_stub(stub);
- bnez(t0, stub->entry(), /* is_far */ true);
- bind(stub->continuation());
- }
-
ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
Label notRecursive;
@@ -236,28 +219,304 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
// Recursive lock
addi(disp_hdr, disp_hdr, -1);
sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
- mv(flag, zr);
- j(cont);
+ j(unlocked);
bind(notRecursive);
- ld(flag, Address(tmp, ObjectMonitor::EntryList_offset()));
+ ld(t0, Address(tmp, ObjectMonitor::EntryList_offset()));
ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
- orr(flag, flag, disp_hdr); // Will be 0 if both are 0.
- bnez(flag, cont);
+ orr(t0, t0, disp_hdr); // Will be 0 if both are 0.
+ bnez(t0, slow_path);
+
// need a release store here
la(tmp, Address(tmp, ObjectMonitor::owner_offset()));
membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
sd(zr, Address(tmp)); // set unowned
- bind(cont);
- // zero flag indicates success
- // non-zero flag indicates failure
- bnez(flag, no_count);
+ bind(unlocked);
+ mv(flag, zr);
+ decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp1Reg, tmp2Reg);
+
+#ifdef ASSERT
+ // Check that unlocked label is reached with flag == 0.
+ Label flag_correct;
+ beqz(flag, flag_correct);
+ stop("Fast Lock Flag != 0");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with flag != 0.
+ bnez(flag, flag_correct);
+ stop("Fast Lock Flag == 0");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of flag (0 vs !0) to determine the continuation.
+}
+
+void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register tmp1, Register tmp2, Register tmp3) {
+ // Flag register, zero for success; non-zero for failure.
+ Register flag = t1;
+
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert_different_registers(obj, tmp1, tmp2, tmp3, flag, t0);
- bind(count);
- decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp);
+ mv(flag, 1);
- bind(no_count);
+ // Handle inflated monitor.
+ Label inflated;
+ // Finish fast lock successfully. MUST branch to with flag == 0
+ Label locked;
+ // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
+ Label slow_path;
+
+ if (DiagnoseSyncOnValueBasedClasses != 0) {
+ load_klass(tmp1, obj);
+ lwu(tmp1, Address(tmp1, Klass::access_flags_offset()));
+ test_bit(tmp1, tmp1, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
+ bnez(tmp1, slow_path);
+ }
+
+ const Register tmp1_mark = tmp1;
+
+ { // Lightweight locking
+
+ // Push lock to the lock stack and finish successfully. MUST branch to with flag == 0
+ Label push;
+
+ const Register tmp2_top = tmp2;
+ const Register tmp3_t = tmp3;
+
+ // Check if lock-stack is full.
+ lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
+ mv(tmp3_t, (unsigned)LockStack::end_offset());
+ bge(tmp2_top, tmp3_t, slow_path);
+
+ // Check if recursive.
+ add(tmp3_t, xthread, tmp2_top);
+ ld(tmp3_t, Address(tmp3_t, -oopSize));
+ beq(obj, tmp3_t, push);
+
+ // Relaxed normal load to check for monitor. Optimization for monitor case.
+ ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
+ bnez(tmp3_t, inflated);
+
+ // Not inflated
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
+
+ // Try to lock. Transition lock-bits 0b01 => 0b00
+ ori(tmp1_mark, tmp1_mark, markWord::unlocked_value);
+ xori(tmp3_t, tmp1_mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_t);
+ bne(tmp1_mark, tmp3_t, slow_path);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ add(tmp3_t, xthread, tmp2_top);
+ sd(obj, Address(tmp3_t));
+ addw(tmp2_top, tmp2_top, oopSize);
+ sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
+ j(locked);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated);
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register tmp1_tagged_monitor = tmp1_mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+ const Register tmp2_owner_addr = tmp2;
+ const Register tmp3_owner = tmp3;
+
+ // Compute owner address.
+ la(tmp2_owner_addr, Address(tmp1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
+
+ // CAS owner (null => current thread).
+ cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ xthread, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner);
+ beqz(tmp3_owner, locked);
+
+ // Check if recursive.
+ bne(tmp3_owner, xthread, slow_path);
+
+ // Recursive.
+ increment(Address(tmp1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1, tmp2, tmp3);
+ }
+
+ bind(locked);
+ mv(flag, zr);
+ increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
+
+#ifdef ASSERT
+ // Check that locked label is reached with flag == 0.
+ Label flag_correct;
+ beqz(flag, flag_correct);
+ stop("Fast Lock Flag != 0");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with flag != 0.
+ bnez(flag, flag_correct);
+ stop("Fast Lock Flag == 0");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of flag (0 vs !0) to determine the continuation.
+}
+
+void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register tmp1, Register tmp2,
+ Register tmp3) {
+ // Flag register, zero for success; non-zero for failure.
+ Register flag = t1;
+
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert_different_registers(obj, tmp1, tmp2, tmp3, flag, t0);
+
+ mv(flag, 1);
+
+ // Handle inflated monitor.
+ Label inflated, inflated_load_monitor;
+ // Finish fast unlock successfully. unlocked MUST branch to with flag == 0
+ Label unlocked;
+ // Finish fast unlock unsuccessfully. MUST branch to with flag != 0
+ Label slow_path;
+
+ const Register tmp1_mark = tmp1;
+ const Register tmp2_top = tmp2;
+ const Register tmp3_t = tmp3;
+
+ { // Lightweight unlock
+
+ // Check if obj is top of lock-stack.
+ lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
+ subw(tmp2_top, tmp2_top, oopSize);
+ add(tmp3_t, xthread, tmp2_top);
+ ld(tmp3_t, Address(tmp3_t));
+ // Top of lock stack was not obj. Must be monitor.
+ bne(obj, tmp3_t, inflated_load_monitor);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
+ DEBUG_ONLY(sd(zr, Address(tmp3_t));)
+ sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
+
+ // Check if recursive.
+ add(tmp3_t, xthread, tmp2_top);
+ ld(tmp3_t, Address(tmp3_t, -oopSize));
+ beq(obj, tmp3_t, unlocked);
+
+ // Not recursive.
+ // Load Mark.
+ ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check header for monitor (0b10).
+ test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
+ bnez(tmp3_t, inflated);
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ ori(tmp3_t, tmp1_mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ tmp3_t);
+ beq(tmp1_mark, tmp3_t, unlocked);
+
+ // Compare and exchange failed.
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
+ DEBUG_ONLY(sd(obj, Address(tmp3_t));)
+ addw(tmp2_top, tmp2_top, oopSize);
+ sd(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
+ j(slow_path);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated_load_monitor);
+ ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+#ifdef ASSERT
+ test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
+ bnez(tmp3_t, inflated);
+ stop("Fast Unlock not monitor");
+#endif
+
+ bind(inflated);
+
+#ifdef ASSERT
+ Label check_done;
+ subw(tmp2_top, tmp2_top, oopSize);
+ mv(tmp3_t, in_bytes(JavaThread::lock_stack_base_offset()));
+ blt(tmp2_top, tmp3_t, check_done);
+ add(tmp3_t, xthread, tmp2_top);
+ ld(tmp3_t, Address(tmp3_t));
+ bne(obj, tmp3_t, inflated);
+ stop("Fast Unlock lock on stack");
+ bind(check_done);
+#endif
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register tmp1_monitor = tmp1_mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+
+ // Untag the monitor.
+ sub(tmp1_monitor, tmp1_mark, monitor_tag);
+
+ const Register tmp2_recursions = tmp2;
+ Label not_recursive;
+
+ // Check if recursive.
+ ld(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
+ beqz(tmp2_recursions, not_recursive);
+
+ // Recursive unlock.
+ addi(tmp2_recursions, tmp2_recursions, -1);
+ sd(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
+ j(unlocked);
+
+ bind(not_recursive);
+
+ Label release;
+ const Register tmp2_owner_addr = tmp2;
+
+ // Compute owner address.
+ la(tmp2_owner_addr, Address(tmp1_monitor, ObjectMonitor::owner_offset()));
+
+ // Check if the entry lists are empty.
+ ld(t0, Address(tmp1_monitor, ObjectMonitor::EntryList_offset()));
+ ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::cxq_offset()));
+ orr(t0, t0, tmp3_t);
+ beqz(t0, release);
+
+ // The owner may be anonymous and we removed the last obj entry in
+ // the lock-stack. This loses the information about the owner.
+ // Write the thread to the owner field so the runtime knows the owner.
+ sd(xthread, Address(tmp2_owner_addr));
+ j(slow_path);
+
+ bind(release);
+ // Set owner to null.
+ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
+ sd(zr, Address(tmp2_owner_addr));
+ }
+
+ bind(unlocked);
+ mv(flag, zr);
+ decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
+
+#ifdef ASSERT
+ // Check that unlocked label is reached with flag == 0.
+ Label flag_correct;
+ beqz(flag, flag_correct);
+ stop("Fast Lock Flag != 0");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with flag != 0.
+ bnez(flag, flag_correct);
+ stop("Fast Lock Flag == 0");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of flag (0 vs !0) to determine the continuation.
}
// short string
@@ -1358,7 +1617,6 @@ void C2_MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
// For Strings we're passed the address of the first characters in a1
// and a2 and the length in cnt1.
-// elem_size is the element size in bytes: either 1 or 2.
// There are two implementations. For arrays >= 8 bytes, all
// comparisons (for hw supporting unaligned access: including the final one,
// which may overlap) are performed 8 bytes at a time.
@@ -1367,13 +1625,12 @@ void C2_MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
// halfword, then a short, and then a byte.
void C2_MacroAssembler::string_equals(Register a1, Register a2,
- Register result, Register cnt1, int elem_size)
+ Register result, Register cnt1)
{
Label SAME, DONE, SHORT, NEXT_WORD;
Register tmp1 = t0;
Register tmp2 = t1;
- assert(elem_size == 1 || elem_size == 2, "must be 2 or 1 byte");
assert_different_registers(a1, a2, result, cnt1, tmp1, tmp2);
BLOCK_COMMENT("string_equals {");
@@ -1439,15 +1696,13 @@ void C2_MacroAssembler::string_equals(Register a1, Register a2,
}
bind(TAIL01);
- if (elem_size == 1) { // Only needed when comparing 1-byte elements
- // 0-1 bytes left.
- test_bit(tmp1, cnt1, 0);
- beqz(tmp1, SAME);
- {
- lbu(tmp1, Address(a1, 0));
- lbu(tmp2, Address(a2, 0));
- bne(tmp1, tmp2, DONE);
- }
+ // 0-1 bytes left.
+ test_bit(tmp1, cnt1, 0);
+ beqz(tmp1, SAME);
+ {
+ lbu(tmp1, Address(a1, 0));
+ lbu(tmp2, Address(a2, 0));
+ bne(tmp1, tmp2, DONE);
}
// Arrays are equal.
@@ -1783,6 +2038,95 @@ void C2_MacroAssembler::signum_fp(FloatRegister dst, FloatRegister one, bool is_
bind(done);
}
+static void float16_to_float_slow_path(C2_MacroAssembler& masm, C2GeneralStub& stub) {
+#define __ masm.
+ FloatRegister dst = stub.data<0>();
+ Register src = stub.data<1>();
+ Register tmp = stub.data<2>();
+ __ bind(stub.entry());
+
+ // following instructions mainly focus on NaN, as riscv does not handle
+ // NaN well with fcvt, but the code also works for Inf at the same time.
+
+ // construct a NaN in 32 bits from the NaN in 16 bits,
+ // we need the payloads of non-canonical NaNs to be preserved.
+ __ mv(tmp, 0x7f800000);
+ // sign-bit was already set via sign-extension if necessary.
+ __ slli(t0, src, 13);
+ __ orr(tmp, t0, tmp);
+ __ fmv_w_x(dst, tmp);
+
+ __ j(stub.continuation());
+#undef __
+}
+
+// j.l.Float.float16ToFloat
+void C2_MacroAssembler::float16_to_float(FloatRegister dst, Register src, Register tmp) {
+ auto stub = C2CodeStub::make(dst, src, tmp, 20, float16_to_float_slow_path);
+
+ // in riscv, NaN needs a special process as fcvt does not work in that case.
+ // in riscv, Inf does not need a special process as fcvt can handle it correctly.
+ // but we consider to get the slow path to process NaN and Inf at the same time,
+ // as both of them are rare cases, and if we try to get the slow path to handle
+ // only NaN case it would sacrifise the performance for normal cases,
+ // i.e. non-NaN and non-Inf cases.
+
+ // check whether it's a NaN or +/- Inf.
+ mv(t0, 0x7c00);
+ andr(tmp, src, t0);
+ // jump to stub processing NaN and Inf cases.
+ beq(t0, tmp, stub->entry());
+
+ // non-NaN or non-Inf cases, just use built-in instructions.
+ fmv_h_x(dst, src);
+ fcvt_s_h(dst, dst);
+
+ bind(stub->continuation());
+}
+
+static void float_to_float16_slow_path(C2_MacroAssembler& masm, C2GeneralStub& stub) {
+#define __ masm.
+ Register dst = stub.data<0>();
+ FloatRegister src = stub.data<1>();
+ Register tmp = stub.data<2>();
+ __ bind(stub.entry());
+
+ __ fmv_x_w(dst, src);
+
+ // preserve the payloads of non-canonical NaNs.
+ __ srai(dst, dst, 13);
+ // preserve the sign bit.
+ __ srai(tmp, dst, 13);
+ __ slli(tmp, tmp, 10);
+ __ mv(t0, 0x3ff);
+ __ orr(tmp, tmp, t0);
+
+ // get the result by merging sign bit and payloads of preserved non-canonical NaNs.
+ __ andr(dst, dst, tmp);
+
+ __ j(stub.continuation());
+#undef __
+}
+
+// j.l.Float.floatToFloat16
+void C2_MacroAssembler::float_to_float16(Register dst, FloatRegister src, FloatRegister ftmp, Register xtmp) {
+ auto stub = C2CodeStub::make(dst, src, xtmp, 130, float_to_float16_slow_path);
+
+ // in riscv, NaN needs a special process as fcvt does not work in that case.
+
+ // check whether it's a NaN.
+ // replace fclass with feq as performance optimization.
+ feq_s(t0, src, src);
+ // jump to stub processing NaN cases.
+ beqz(t0, stub->entry());
+
+ // non-NaN cases, just use built-in instructions.
+ fcvt_h_s(ftmp, src);
+ fmv_x_h(dst, ftmp);
+
+ bind(stub->continuation());
+}
+
void C2_MacroAssembler::signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen) {
vsetvli_helper(bt, vlen);
@@ -1896,7 +2240,7 @@ void C2_MacroAssembler::element_compare(Register a1, Register a2, Register resul
mv(result, true);
}
-void C2_MacroAssembler::string_equals_v(Register a1, Register a2, Register result, Register cnt, int elem_size) {
+void C2_MacroAssembler::string_equals_v(Register a1, Register a2, Register result, Register cnt) {
Label DONE;
Register tmp1 = t0;
Register tmp2 = t1;
@@ -1905,11 +2249,7 @@ void C2_MacroAssembler::string_equals_v(Register a1, Register a2, Register resul
mv(result, false);
- if (elem_size == 2) {
- srli(cnt, cnt, 1);
- }
-
- element_compare(a1, a2, result, cnt, tmp1, tmp2, v2, v4, v2, elem_size == 1, DONE);
+ element_compare(a1, a2, result, cnt, tmp1, tmp2, v2, v4, v2, true, DONE);
bind(DONE);
BLOCK_COMMENT("} string_equals_v");
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
index 4940ce5fe9e94..25ba66387f18a 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
@@ -44,9 +44,11 @@
public:
// Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
- // See full description in macroAssembler_riscv.cpp.
void fast_lock(Register object, Register box, Register tmp1, Register tmp2, Register tmp3);
void fast_unlock(Register object, Register box, Register tmp1, Register tmp2);
+ // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in .ad file.
+ void fast_lock_lightweight(Register object, Register tmp1, Register tmp2, Register tmp3);
+ void fast_unlock_lightweight(Register object, Register tmp1, Register tmp2, Register tmp3);
void string_compare(Register str1, Register str2,
Register cnt1, Register cnt2, Register result,
@@ -92,8 +94,7 @@
void arrays_hashcode_elload(Register dst, Address src, BasicType eltype);
void string_equals(Register r1, Register r2,
- Register result, Register cnt1,
- int elem_size);
+ Register result, Register cnt1);
// refer to conditional_branches and float_conditional_branches
static const int bool_test_bits = 3;
@@ -172,8 +173,12 @@
void signum_fp(FloatRegister dst, FloatRegister one, bool is_double);
+ void float16_to_float(FloatRegister dst, Register src, Register tmp);
+ void float_to_float16(Register dst, FloatRegister src, FloatRegister ftmp, Register xtmp);
+
void signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen);
+
// intrinsic methods implemented by rvv instructions
// compress bits, i.e. j.l.Integer/Long::compress.
@@ -184,8 +189,7 @@
void expand_bits_l_v(Register dst, Register src, Register mask);
void string_equals_v(Register r1, Register r2,
- Register result, Register cnt1,
- int elem_size);
+ Register result, Register cnt1);
void arrays_equals_v(Register r1, Register r2,
Register result, Register cnt1,
diff --git a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
index e29dee56de8d8..fdb2bcb06ff97 100644
--- a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
+++ b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
@@ -27,7 +27,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
@@ -37,7 +36,7 @@
// ----------------------------------------------------------------------------
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
precond(cbuf.stubs()->start() != badAddress);
precond(cbuf.stubs()->end() != badAddress);
// Stub is fixed up when the corresponding call is converted from
@@ -69,11 +68,11 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
}
#undef __
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return MacroAssembler::static_call_stub_size();
}
-int CompiledStaticCall::to_trampoline_stub_size() {
+int CompiledDirectCall::to_trampoline_stub_size() {
// Somewhat pessimistically, we count 4 instructions here (although
// there are only 3) because we sometimes emit an alignment nop.
// Trampoline stubs are always word aligned.
@@ -81,21 +80,14 @@ int CompiledStaticCall::to_trampoline_stub_size() {
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 4; // 3 in emit_to_interp_stub + 1 in emit_call
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder
= nativeMovConstReg_at(stub);
@@ -112,7 +104,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -129,7 +121,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
index 5d0b0fb472934..fa7df32d7e944 100644
--- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
@@ -29,8 +29,8 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/javaThread.hpp"
diff --git a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
index e368bbdc9141f..68cd51ece5f70 100644
--- a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
@@ -50,6 +50,9 @@ const bool CCallingConventionRequiresIntsAsLongs = false;
#define USE_POINTERS_TO_REGISTER_IMPL_ARRAY
+// auipc useable for all cc -> cc calls and jumps
+#define CODE_CACHE_SIZE_LIMIT ((2*G)-(2*K))
+
// The expected size in bytes of a cache line.
#define DEFAULT_CACHE_LINE_SIZE 64
diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp
index aa95cebec14cd..8461948752bae 100644
--- a/src/hotspot/cpu/riscv/globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globals_riscv.hpp
@@ -105,7 +105,9 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(bool, UseZba, false, "Use Zba instructions") \
product(bool, UseZbb, false, "Use Zbb instructions") \
product(bool, UseZbs, false, "Use Zbs instructions") \
+ product(bool, UseZfh, false, "Use Zfh instructions") \
product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \
+ product(bool, UseZcb, false, EXPERIMENTAL, "Use Zcb instructions") \
product(bool, UseZic64b, false, EXPERIMENTAL, "Use Zic64b instructions") \
product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions") \
product(bool, UseZicbop, false, EXPERIMENTAL, "Use Zicbop instructions") \
diff --git a/src/hotspot/cpu/riscv/icBuffer_riscv.cpp b/src/hotspot/cpu/riscv/icBuffer_riscv.cpp
deleted file mode 100644
index ab904817816fc..0000000000000
--- a/src/hotspot/cpu/riscv/icBuffer_riscv.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_riscv.hpp"
-#include "oops/oop.inline.hpp"
-
-int InlineCacheBuffer::ic_stub_code_size() {
- // 6: auipc + ld + auipc + jalr + address(2 * instruction_size)
- return 6 * NativeInstruction::instruction_size;
-}
-
-#define __ masm->
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- assert_cond(code_begin != nullptr && entry_point != nullptr);
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler* masm = new MacroAssembler(&code);
- // Note: even though the code contains an embedded value, we do not need reloc info
- // because
- // (1) the value is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear
-
- address start = __ pc();
- Label l;
- __ ld(t1, l);
- __ far_jump(ExternalAddress(entry_point));
- __ align(wordSize);
- __ bind(l);
- __ emit_int64((intptr_t)cached_value);
- // Only need to invalidate the 1st two instructions - not the whole ic stub
- ICache::invalidate_range(code_begin, InlineCacheBuffer::ic_stub_code_size());
- assert(__ pc() - start == ic_stub_code_size(), "must be");
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- NativeJump* jump = nativeJump_at(move->next_instruction_address());
- return jump->jump_destination();
-}
-
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- // The word containing the cached value is at the end of this IC buffer
- uintptr_t *p = (uintptr_t *)(code_begin + ic_stub_code_size() - wordSize);
- void* o = (void*)*p;
- return o;
-}
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
index 19d665bd421d0..497918e6c05c6 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
@@ -763,7 +763,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
}
if (LockingMode == LM_LIGHTWEIGHT) {
- ld(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
lightweight_lock(obj_reg, tmp, tmp2, tmp3, slow_case);
j(count);
} else if (LockingMode == LM_LEGACY) {
@@ -860,24 +859,6 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
if (LockingMode == LM_LIGHTWEIGHT) {
Label slow_case;
-
- // Check for non-symmetric locking. This is allowed by the spec and the interpreter
- // must handle it.
- Register tmp1 = t0;
- Register tmp2 = header_reg;
- // First check for lock-stack underflow.
- lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
- mv(tmp2, (unsigned)LockStack::start_offset());
- ble(tmp1, tmp2, slow_case);
- // Then check if the top of the lock-stack matches the unlocked object.
- subw(tmp1, tmp1, oopSize);
- add(tmp1, xthread, tmp1);
- ld(tmp1, Address(tmp1, 0));
- bne(tmp1, obj_reg, slow_case);
-
- ld(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- test_bit(t0, header_reg, exact_log2(markWord::monitor_value));
- bnez(t0, slow_case);
lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
j(count);
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index ce336c16aa718..4459356d3b132 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -27,6 +27,7 @@
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -48,6 +49,7 @@
#include "runtime/jniHandles.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER2
#include "opto/compile.hpp"
@@ -634,8 +636,8 @@ void MacroAssembler::unimplemented(const char* what) {
}
void MacroAssembler::emit_static_call_stub() {
- IncompressibleRegion ir(this); // Fixed length: see CompiledStaticCall::to_interp_stub_size().
- // CompiledDirectStaticCall::set_to_interpreted knows the
+ IncompressibleRegion ir(this); // Fixed length: see CompiledDirectCall::to_interp_stub_size().
+ // CompiledDirectCall::set_to_interpreted knows the
// exact layout of this stub.
mov_metadata(xmethod, (Metadata*)nullptr);
@@ -2542,7 +2544,7 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
}
// Look up the method for a megamorphic invokeinterface call in a single pass over itable:
-// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
// - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
// The target method is determined by .
// The receiver klass is in recv_klass.
@@ -3542,6 +3544,48 @@ address MacroAssembler::ic_call(address entry, jint method_index) {
return trampoline_call(Address(entry, rh));
}
+int MacroAssembler::ic_check_size() {
+ // No compressed
+ return (NativeInstruction::instruction_size * (2 /* 2 loads */ + 1 /* branch */)) +
+ far_branch_size();
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ IncompressibleRegion ir(this);
+ Register receiver = j_rarg0;
+ Register data = t1;
+
+ Register tmp1 = t0; // t0 always scratch
+ // t2 is saved on call, thus should have been saved before this check.
+ // Hence we can clobber it.
+ Register tmp2 = t2;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, ic_check_size());
+ int uep_offset = offset();
+
+ if (UseCompressedClassPointers) {
+ lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+ } else {
+ ld(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ ld(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+ }
+
+ Label ic_hit;
+ beq(tmp1, tmp2, ic_hit);
+ // Note, far_jump is not fixed size.
+ // Is this ever generates a movptr alignment/size will be off.
+ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ bind(ic_hit);
+
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point.");
+ return uep_offset;
+}
+
// Emit a trampoline stub for a call to a target which is too far away.
//
// code sequences:
@@ -4999,98 +5043,124 @@ void MacroAssembler::test_bit(Register Rd, Register Rs, uint32_t bit_pos) {
}
// Implements lightweight-locking.
-// Branches to slow upon failure to lock the object.
-// Falls through upon success.
//
// - obj: the object to be locked
-// - hdr: the header, already loaded from obj, will be destroyed
-// - tmp1, tmp2: temporary registers, will be destroyed
-void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow) {
+// - tmp1, tmp2, tmp3: temporary registers, will be destroyed
+// - slow: branched to if locking fails
+void MacroAssembler::lightweight_lock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, tmp1, tmp2, t0);
-
- // Check if we would have space on lock-stack for the object.
- lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
- mv(tmp2, (unsigned)LockStack::end_offset());
- bge(tmp1, tmp2, slow, /* is_far */ true);
-
- // Load (object->mark() | 1) into hdr
- ori(hdr, hdr, markWord::unlocked_value);
- // Clear lock-bits, into tmp2
- xori(tmp2, hdr, markWord::unlocked_value);
-
- // Try to swing header from unlocked to locked
- Label success;
- cmpxchgptr(hdr, tmp2, obj, tmp1, success, &slow);
- bind(success);
-
- // After successful lock, push object on lock-stack
- lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
- add(tmp2, xthread, tmp1);
- sd(obj, Address(tmp2, 0));
- addw(tmp1, tmp1, oopSize);
- sw(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+ assert_different_registers(obj, tmp1, tmp2, tmp3, t0);
+
+ Label push;
+ const Register top = tmp1;
+ const Register mark = tmp2;
+ const Register t = tmp3;
+
+ // Preload the markWord. It is important that this is the first
+ // instruction emitted as it is part of C1's null check semantics.
+ ld(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check if the lock-stack is full.
+ lwu(top, Address(xthread, JavaThread::lock_stack_top_offset()));
+ mv(t, (unsigned)LockStack::end_offset());
+ bge(top, t, slow, /* is_far */ true);
+
+ // Check for recursion.
+ add(t, xthread, top);
+ ld(t, Address(t, -oopSize));
+ beq(obj, t, push);
+
+ // Check header for monitor (0b10).
+ test_bit(t, mark, exact_log2(markWord::monitor_value));
+ bnez(t, slow, /* is_far */ true);
+
+ // Try to lock. Transition lock-bits 0b01 => 0b00
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
+ ori(mark, mark, markWord::unlocked_value);
+ xori(t, mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t);
+ bne(mark, t, slow, /* is_far */ true);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ add(t, xthread, top);
+ sd(obj, Address(t));
+ addw(top, top, oopSize);
+ sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
}
// Implements ligthweight-unlocking.
-// Branches to slow upon failure.
-// Falls through upon success.
//
// - obj: the object to be unlocked
-// - hdr: the (pre-loaded) header of the object
-// - tmp1, tmp2: temporary registers
-void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow) {
+// - tmp1, tmp2, tmp3: temporary registers
+// - slow: branched to if unlocking fails
+void MacroAssembler::lightweight_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, tmp1, tmp2, t0);
+ assert_different_registers(obj, tmp1, tmp2, tmp3, t0);
#ifdef ASSERT
{
- // The following checks rely on the fact that LockStack is only ever modified by
- // its owning thread, even if the lock got inflated concurrently; removal of LockStack
- // entries after inflation will happen delayed in that case.
-
// Check for lock-stack underflow.
Label stack_ok;
lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
mv(tmp2, (unsigned)LockStack::start_offset());
- bgt(tmp1, tmp2, stack_ok);
+ bge(tmp1, tmp2, stack_ok);
STOP("Lock-stack underflow");
bind(stack_ok);
}
- {
- // Check if the top of the lock-stack matches the unlocked object.
- Label tos_ok;
- subw(tmp1, tmp1, oopSize);
- add(tmp1, xthread, tmp1);
- ld(tmp1, Address(tmp1, 0));
- beq(tmp1, obj, tos_ok);
- STOP("Top of lock-stack does not match the unlocked object");
- bind(tos_ok);
- }
- {
- // Check that hdr is fast-locked.
- Label hdr_ok;
- andi(tmp1, hdr, markWord::lock_mask_in_place);
- beqz(tmp1, hdr_ok);
- STOP("Header is not fast-locked");
- bind(hdr_ok);
- }
#endif
- // Load the new header (unlocked) into tmp1
- ori(tmp1, hdr, markWord::unlocked_value);
+ Label unlocked, push_and_slow;
+ const Register top = tmp1;
+ const Register mark = tmp2;
+ const Register t = tmp3;
+
+ // Check if obj is top of lock-stack.
+ lwu(top, Address(xthread, JavaThread::lock_stack_top_offset()));
+ subw(top, top, oopSize);
+ add(t, xthread, top);
+ ld(t, Address(t));
+ bne(obj, t, slow, /* is_far */ true);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(add(t, xthread, top);)
+ DEBUG_ONLY(sd(zr, Address(t));)
+ sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
+
+ // Check if recursive.
+ add(t, xthread, top);
+ ld(t, Address(t, -oopSize));
+ beq(obj, t, unlocked);
+
+ // Not recursive. Check header for monitor (0b10).
+ ld(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ test_bit(t, mark, exact_log2(markWord::monitor_value));
+ bnez(t, push_and_slow);
- // Try to swing header from locked to unlocked
- Label success;
- cmpxchgptr(hdr, tmp1, obj, tmp2, success, &slow);
- bind(success);
-
- // After successful unlock, pop object from lock-stack
- lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
- subw(tmp1, tmp1, oopSize);
#ifdef ASSERT
- add(tmp2, xthread, tmp1);
- sd(zr, Address(tmp2, 0));
+ // Check header not unlocked (0b01).
+ Label not_unlocked;
+ test_bit(t, mark, exact_log2(markWord::unlocked_value));
+ beqz(t, not_unlocked);
+ stop("lightweight_unlock already unlocked");
+ bind(not_unlocked);
#endif
- sw(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ ori(t, mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ t);
+ beq(mark, t, unlocked);
+
+ bind(push_and_slow);
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(add(t, xthread, top);)
+ DEBUG_ONLY(sd(obj, Address(t));)
+ addw(top, top, oopSize);
+ sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
+ j(slow);
+
+ bind(unlocked);
}
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index d283654e6e179..75c0f9ba30596 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -1193,7 +1193,10 @@ class MacroAssembler: public Assembler {
//
// Return: the call PC or null if CodeCache is full.
address trampoline_call(Address entry);
+
address ic_call(address entry, jint method_index = 0);
+ static int ic_check_size();
+ int ic_check(int end_alignment = NativeInstruction::instruction_size);
// Support for memory inc/dec
// n.b. increment/decrement calls with an Address destination will
@@ -1516,8 +1519,8 @@ class MacroAssembler: public Assembler {
void store_conditional(Register dst, Register new_val, Register addr, enum operand_size size, Assembler::Aqrl release);
public:
- void lightweight_lock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow);
- void lightweight_unlock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow);
+ void lightweight_lock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow);
+ void lightweight_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow);
};
#ifdef ASSERT
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
index c29ac1a04fae1..c4048f66e0d3e 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
@@ -27,7 +27,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "code/compiledIC.hpp"
-#include "memory/resourceArea.hpp"
#include "nativeInst_riscv.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/handles.hpp"
@@ -157,7 +156,6 @@ void NativeCall::set_destination_mt_safe(address dest, bool assert_lock) {
CompiledICLocker::is_safe(addr_at(0)),
"concurrent code patching");
- ResourceMark rm;
address addr_call = addr_at(0);
assert(NativeCall::is_call_at(addr_call), "unexpected code at call site");
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index 7e1291f49d74c..86d0ad7a05929 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
// Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1196,13 +1196,13 @@ bool is_CAS(int opcode, bool maybe_volatile)
// returns true if CAS needs to use an acquiring load otherwise false
bool needs_acquiring_load_reserved(const Node *n)
{
- assert(n != NULL && is_CAS(n->Opcode(), true), "expecting a compare and swap");
+ assert(n != nullptr && is_CAS(n->Opcode(), true), "expecting a compare and swap");
LoadStoreNode* ldst = n->as_LoadStore();
- if (n != NULL && is_CAS(n->Opcode(), false)) {
- assert(ldst != NULL && ldst->trailing_membar() != NULL, "expected trailing membar");
+ if (n != nullptr && is_CAS(n->Opcode(), false)) {
+ assert(ldst != nullptr && ldst->trailing_membar() != nullptr, "expected trailing membar");
} else {
- return ldst != NULL && ldst->trailing_membar() != NULL;
+ return ldst != nullptr && ldst->trailing_membar() != nullptr;
}
// so we can just return true here
return true;
@@ -1247,7 +1247,7 @@ int MachCallRuntimeNode::ret_addr_offset() {
// sd(t1, Address(sp, wordSize)) -> sd
// jalr(t0) -> jalr
CodeBlob *cb = CodeCache::find_blob(_entry_point);
- if (cb != NULL) {
+ if (cb != nullptr) {
return 1 * NativeInstruction::instruction_size;
} else {
return 11 * NativeInstruction::instruction_size;
@@ -1286,7 +1286,7 @@ int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
#ifndef PRODUCT
void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
- assert_cond(st != NULL);
+ assert_cond(st != nullptr);
st->print("BREAKPOINT");
}
#endif
@@ -1342,14 +1342,14 @@ uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
#ifndef PRODUCT
void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
- assert_cond(st != NULL);
+ assert_cond(st != nullptr);
st->print("-- \t// MachConstantBaseNode (empty encoding)");
}
#endif
#ifndef PRODUCT
void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
- assert_cond(st != NULL && ra_ != NULL);
+ assert_cond(st != nullptr && ra_ != nullptr);
Compile* C = ra_->C;
int framesize = C->output()->frame_slots() << LogBytesPerInt;
@@ -1363,7 +1363,7 @@ void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
if (PreserveFramePointer) { st->print("sub fp, sp, #%d\n\t", 2 * wordSize); }
st->print("sub sp, sp, #%d\n\t", framesize);
- if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
+ if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
st->print("ld t0, [guard]\n\t");
st->print("membar LoadLoad\n\t");
st->print("ld t1, [xthread, #thread_disarmed_guard_value_offset]\n\t");
@@ -1377,7 +1377,7 @@ void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
#endif
void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- assert_cond(ra_ != NULL);
+ assert_cond(ra_ != nullptr);
Compile* C = ra_->C;
C2_MacroAssembler _masm(&cbuf);
@@ -1392,7 +1392,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
__ nop(); // 4 bytes
}
- assert_cond(C != NULL);
+ assert_cond(C != nullptr);
if (C->clinit_barrier_on_entry()) {
assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
@@ -1412,9 +1412,9 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
__ build_frame(framesize);
- if (C->stub_function() == NULL) {
+ if (C->stub_function() == nullptr) {
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
- if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
+ if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
// Dummy labels for just measuring the code size
Label dummy_slow_path;
Label dummy_continuation;
@@ -1451,7 +1451,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
uint MachPrologNode::size(PhaseRegAlloc* ra_) const
{
- assert_cond(ra_ != NULL);
+ assert_cond(ra_ != nullptr);
return MachNode::size(ra_); // too many variables; just compute it
// the hard way
}
@@ -1465,9 +1465,9 @@ int MachPrologNode::reloc() const
#ifndef PRODUCT
void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
- assert_cond(st != NULL && ra_ != NULL);
+ assert_cond(st != nullptr && ra_ != nullptr);
Compile* C = ra_->C;
- assert_cond(C != NULL);
+ assert_cond(C != nullptr);
int framesize = C->output()->frame_size_in_bytes();
st->print("# pop frame %d\n\t", framesize);
@@ -1491,10 +1491,10 @@ void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
#endif
void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- assert_cond(ra_ != NULL);
+ assert_cond(ra_ != nullptr);
Compile* C = ra_->C;
C2_MacroAssembler _masm(&cbuf);
- assert_cond(C != NULL);
+ assert_cond(C != nullptr);
int framesize = C->output()->frame_size_in_bytes();
__ remove_frame(framesize);
@@ -1517,7 +1517,7 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
}
uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
- assert_cond(ra_ != NULL);
+ assert_cond(ra_ != nullptr);
// Variable size. Determine dynamically.
return MachNode::size(ra_);
}
@@ -1568,7 +1568,7 @@ static enum RC rc_class(OptoReg::Name reg) {
}
uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
- assert_cond(ra_ != NULL);
+ assert_cond(ra_ != nullptr);
Compile* C = ra_->C;
// Get registers to move.
@@ -1599,7 +1599,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
int src_offset = ra_->reg2offset(src_lo);
int dst_offset = ra_->reg2offset(dst_lo);
- if (bottom_type()->isa_vect() != NULL) {
+ if (bottom_type()->isa_vect() != nullptr) {
uint ireg = ideal_reg();
if (ireg == Op_VecA && cbuf) {
C2_MacroAssembler _masm(cbuf);
@@ -1640,7 +1640,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
ShouldNotReachHere();
}
}
- } else if (cbuf != NULL) {
+ } else if (cbuf != nullptr) {
C2_MacroAssembler _masm(cbuf);
switch (src_lo_rc) {
case rc_int:
@@ -1711,7 +1711,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
}
}
- if (st != NULL) {
+ if (st != nullptr) {
st->print("spill ");
if (src_lo_rc == rc_stack) {
st->print("[sp, #%d] -> ", src_offset);
@@ -1745,16 +1745,16 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
#ifndef PRODUCT
void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
- if (ra_ == NULL) {
+ if (ra_ == nullptr) {
st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
} else {
- implementation(NULL, ra_, false, st);
+ implementation(nullptr, ra_, false, st);
}
}
#endif
void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- implementation(&cbuf, ra_, false, NULL);
+ implementation(&cbuf, ra_, false, nullptr);
}
uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
@@ -1765,7 +1765,7 @@ uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
#ifndef PRODUCT
void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
- assert_cond(ra_ != NULL && st != NULL);
+ assert_cond(ra_ != nullptr && st != nullptr);
int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
int reg = ra_->get_reg_first(this);
st->print("add %s, sp, #%d\t# box lock",
@@ -1777,7 +1777,7 @@ void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
C2_MacroAssembler _masm(&cbuf);
Assembler::IncompressibleRegion ir(&_masm); // Fixed length: see BoxLockNode::size()
- assert_cond(ra_ != NULL);
+ assert_cond(ra_ != nullptr);
int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
int reg = ra_->get_encode(this);
@@ -1805,17 +1805,16 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
#ifndef PRODUCT
void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
- assert_cond(st != NULL);
+ assert_cond(st != nullptr);
st->print_cr("# MachUEPNode");
if (UseCompressedClassPointers) {
- st->print_cr("\tlwu t0, [j_rarg0, oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- if (CompressedKlassPointers::shift() != 0) {
- st->print_cr("\tdecode_klass_not_null t0, t0");
- }
+ st->print_cr("\tlwu t0, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tlwu t2, [t1 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
} else {
- st->print_cr("\tld t0, [j_rarg0, oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tld t0, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tld t2, [t1 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
}
- st->print_cr("\tbeq t0, t1, ic_hit");
+ st->print_cr("\tbeq t0, t2, ic_hit");
st->print_cr("\tj, SharedRuntime::_ic_miss_stub\t # Inline cache check");
st->print_cr("\tic_hit:");
}
@@ -1825,20 +1824,16 @@ void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
{
// This is the unverified entry point.
C2_MacroAssembler _masm(&cbuf);
+ __ ic_check(CodeEntryAlignment);
- Label skip;
- __ cmp_klass(j_rarg0, t1, t0, t2 /* call-clobbered t2 as a tmp */, skip);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- __ bind(skip);
-
- // These NOPs are critical so that verified entry point is properly
- // 4 bytes aligned for patching by NativeJump::patch_verified_entry()
- __ align(NativeInstruction::instruction_size);
+ // Verified entry point must be properly 4 bytes aligned for patching by NativeJump::patch_verified_entry().
+ // ic_check() aligns to CodeEntryAlignment >= InteriorEntryAlignment(min 16) > NativeInstruction::instruction_size(4).
+ assert(((__ offset()) % CodeEntryAlignment) == 0, "Misaligned verified entry point");
}
uint MachUEPNode::size(PhaseRegAlloc* ra_) const
{
- assert_cond(ra_ != NULL);
+ assert_cond(ra_ != nullptr);
return MachNode::size(ra_);
}
@@ -1855,7 +1850,7 @@ int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
// That's why we must use the macroassembler to generate a handler.
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_exception_handler());
- if (base == NULL) {
+ if (base == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -1873,7 +1868,7 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
// That's why we must use the macroassembler to generate a handler.
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_deopt_handler());
- if (base == NULL) {
+ if (base == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -1932,6 +1927,10 @@ bool Matcher::match_rule_supported(int opcode) {
case Op_FmaVF:
case Op_FmaVD:
return UseFMA;
+
+ case Op_ConvHF2F:
+ case Op_ConvF2HF:
+ return UseZfh;
}
return true; // Per default match rules are supported.
@@ -2013,7 +2012,7 @@ int Matcher::min_vector_size(const BasicType bt) {
return MIN2(size, max_size);
}
-int Matcher::superword_max_vector_size(const BasicType bt) {
+int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
return Matcher::max_vector_size(bt);
}
@@ -2034,7 +2033,7 @@ int Matcher::scalable_vector_reg_size(const BasicType bt) {
MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
ShouldNotReachHere(); // generic vector operands not supported
- return NULL;
+ return nullptr;
}
bool Matcher::is_reg2reg_move(MachNode* m) {
@@ -2137,10 +2136,10 @@ const RegMask Matcher::method_handle_invoke_SP_save_mask() {
}
bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
- assert_cond(addp != NULL);
+ assert_cond(addp != nullptr);
for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
Node* u = addp->fast_out(i);
- if (u != NULL && u->is_Mem()) {
+ if (u != nullptr && u->is_Mem()) {
int opsize = u->as_Mem()->memory_size();
assert(opsize > 0, "unexpected memory operand size");
if (u->as_Mem()->memory_size() != (1 << shift)) {
@@ -2153,7 +2152,7 @@ bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
// Should the Matcher clone input 'm' of node 'n'?
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
- assert_cond(m != NULL);
+ assert_cond(m != nullptr);
if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
mstack.push(m, Visit); // m = ShiftCntV
return true;
@@ -2212,7 +2211,7 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Register dst_reg = as_Register($dst$$reg);
address con = (address)$src$$constant;
- if (con == NULL || con == (address)1) {
+ if (con == nullptr || con == (address)1) {
ShouldNotReachHere();
} else {
relocInfo::relocType rtype = $src->constant_reloc();
@@ -2242,7 +2241,7 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Register dst_reg = as_Register($dst$$reg);
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
ShouldNotReachHere();
} else {
relocInfo::relocType rtype = $src->constant_reloc();
@@ -2261,7 +2260,7 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Register dst_reg = as_Register($dst$$reg);
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
ShouldNotReachHere();
} else {
relocInfo::relocType rtype = $src->constant_reloc();
@@ -2347,7 +2346,7 @@ encode %{
Label done;
C2_MacroAssembler _masm(&cbuf);
__ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
- NULL, &miss);
+ nullptr, &miss);
if ($primary) {
__ mv(result_reg, zr);
} else {
@@ -2368,12 +2367,12 @@ encode %{
Assembler::IncompressibleRegion ir(&_masm); // Fixed length: see ret_addr_offset
address addr = (address)$meth$$method;
- address call = NULL;
- assert_cond(addr != NULL);
+ address call = nullptr;
+ assert_cond(addr != nullptr);
if (!_method) {
// A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
- if (call == NULL) {
+ if (call == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -2387,7 +2386,7 @@ encode %{
RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
: static_call_Relocation::spec(method_index);
call = __ trampoline_call(Address(addr, rspec));
- if (call == NULL) {
+ if (call == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -2398,8 +2397,8 @@ encode %{
cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
} else {
// Emit stub for static call
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
- if (stub == NULL) {
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, call);
+ if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -2414,7 +2413,7 @@ encode %{
Assembler::IncompressibleRegion ir(&_masm); // Fixed length: see ret_addr_offset
int method_index = resolved_method_index(cbuf);
address call = __ ic_call((address)$meth$$method, method_index);
- if (call == NULL) {
+ if (call == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -2440,9 +2439,9 @@ encode %{
// which loads the absolute address into a register.
address entry = (address)$meth$$method;
CodeBlob *cb = CodeCache::find_blob(entry);
- if (cb != NULL) {
+ if (cb != nullptr) {
address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
- if (call == NULL) {
+ if (call == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -2890,7 +2889,7 @@ operand immP()
interface(CONST_INTER);
%}
-// NULL Pointer Immediate
+// Null Pointer Immediate
operand immP0()
%{
predicate(n->get_ptr() == 0);
@@ -3011,7 +3010,7 @@ operand immN()
interface(CONST_INTER);
%}
-// Narrow NULL Pointer Immediate
+// Narrow Null Pointer Immediate
operand immN0()
%{
predicate(n->get_narrowcon() == 0);
@@ -4889,7 +4888,7 @@ instruct loadConP0(iRegPNoSp dst, immP0 con)
match(Set dst con);
ins_cost(ALU_COST);
- format %{ "mv $dst, $con\t# NULL ptr, #@loadConP0" %}
+ format %{ "mv $dst, $con\t# null pointer, #@loadConP0" %}
ins_encode(riscv_enc_mov_zero(dst));
@@ -4940,7 +4939,7 @@ instruct loadConN0(iRegNNoSp dst, immN0 con)
match(Set dst con);
ins_cost(ALU_COST);
- format %{ "mv $dst, $con\t# compressed NULL ptr, #@loadConN0" %}
+ format %{ "mv $dst, $con\t# compressed null pointer, #@loadConN0" %}
ins_encode(riscv_enc_mov_zero(dst));
@@ -8275,6 +8274,32 @@ instruct convD2F_reg(fRegF dst, fRegD src) %{
ins_pipe(fp_d2f);
%}
+// single <-> half precision
+
+instruct convHF2F_reg_reg(fRegF dst, iRegINoSp src, iRegINoSp tmp) %{
+ match(Set dst (ConvHF2F src));
+ effect(TEMP tmp);
+ format %{ "fmv.h.x $dst, $src\t# move source from $src to $dst\n\t"
+ "fcvt.s.h $dst, $dst\t# convert half to single precision"
+ %}
+ ins_encode %{
+ __ float16_to_float($dst$$FloatRegister, $src$$Register, $tmp$$Register);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct convF2HF_reg_reg(iRegINoSp dst, fRegF src, fRegF ftmp, iRegINoSp xtmp) %{
+ match(Set dst (ConvF2HF src));
+ effect(TEMP_DEF dst, TEMP ftmp, TEMP xtmp);
+ format %{ "fcvt.h.s $ftmp, $src\t# convert single precision to half\n\t"
+ "fmv.x.h $dst, $ftmp\t# move result from $ftmp to $dst"
+ %}
+ ins_encode %{
+ __ float_to_float16($dst$$Register, $src$$FloatRegister, $ftmp$$FloatRegister, $xtmp$$Register);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// float <-> int
instruct convF2I_reg_reg(iRegINoSp dst, fRegF src) %{
@@ -10308,7 +10333,7 @@ instruct clearArray_reg_reg(iRegL_R29 cnt, iRegP_R28 base, iRegP_R30 tmp1,
ins_encode %{
address tpc = __ zero_words($base$$Register, $cnt$$Register);
- if (tpc == NULL) {
+ if (tpc == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -10345,23 +10370,7 @@ instruct string_equalsL(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt,
ins_encode %{
// Count is in 8-bit bytes; non-Compact chars are 16 bits.
__ string_equals($str1$$Register, $str2$$Register,
- $result$$Register, $cnt$$Register, 1);
- %}
- ins_pipe(pipe_class_memory);
-%}
-
-instruct string_equalsU(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt,
- iRegI_R10 result, rFlagsReg cr)
-%{
- predicate(!UseRVV && ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
- match(Set result (StrEquals (Binary str1 str2) cnt));
- effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
-
- format %{ "String Equals $str1, $str2, $cnt -> $result\t#@string_equalsU" %}
- ins_encode %{
- // Count is in 8-bit bytes; non-Compact chars are 16 bits.
- __ string_equals($str1$$Register, $str2$$Register,
- $result$$Register, $cnt$$Register, 2);
+ $result$$Register, $cnt$$Register);
%}
ins_pipe(pipe_class_memory);
%}
@@ -10460,10 +10469,11 @@ instruct tlsLoadP(javaThread_RegP dst)
// using t1 as the 'flag' register to bridge the BoolNode producers and consumers
instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3)
%{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastLock object box));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3);
- ins_cost(LOAD_COST * 2 + STORE_COST * 3 + ALU_COST * 6 + BRANCH_COST * 3);
+ ins_cost(10 * DEFAULT_COST);
format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2,$tmp3, #@cmpFastLock" %}
ins_encode %{
@@ -10476,10 +10486,11 @@ instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iReg
// using t1 as the 'flag' register to bridge the BoolNode producers and consumers
instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2)
%{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastUnlock object box));
effect(TEMP tmp1, TEMP tmp2);
- ins_cost(LOAD_COST * 2 + STORE_COST + ALU_COST * 2 + BRANCH_COST * 4);
+ ins_cost(10 * DEFAULT_COST);
format %{ "fastunlock $object,$box\t! kills $tmp1, $tmp2, #@cmpFastUnlock" %}
ins_encode %{
@@ -10489,6 +10500,38 @@ instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iR
ins_pipe(pipe_serial);
%}
+instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP_R10 box, iRegPNoSp tmp1, iRegPNoSp tmp2)
+%{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastLock object box));
+ effect(TEMP tmp1, TEMP tmp2, USE_KILL box);
+
+ ins_cost(10 * DEFAULT_COST);
+ format %{ "fastlock $object,$box\t! kills $box,$tmp1,$tmp2 #@cmpFastLockLightweight" %}
+
+ ins_encode %{
+ __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
+instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP_R10 box, iRegPNoSp tmp1, iRegPNoSp tmp2)
+%{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastUnlock object box));
+ effect(TEMP tmp1, TEMP tmp2, USE_KILL box);
+
+ ins_cost(10 * DEFAULT_COST);
+ format %{ "fastunlock $object,$box\t! kills $box,$tmp1,$tmp2, #@cmpFastUnlockLightweight" %}
+
+ ins_encode %{
+ __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
// Tail Call; Jump from runtime stub to Java code.
// Also known as an 'interprocedural jump'.
// Target of jump will eventually return to caller.
diff --git a/src/hotspot/cpu/riscv/riscv_v.ad b/src/hotspot/cpu/riscv/riscv_v.ad
index c163325fc815f..d07f4dc7c9a2c 100644
--- a/src/hotspot/cpu/riscv/riscv_v.ad
+++ b/src/hotspot/cpu/riscv/riscv_v.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2023, Arm Limited. All rights reserved.
// Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -46,7 +46,7 @@ source %{
}
}
- bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+ bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
return match_rule_supported_vector(opcode, vlen, bt);
}
@@ -2614,24 +2614,7 @@ instruct vstring_equalsL(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt,
ins_encode %{
// Count is in 8-bit bytes; non-Compact chars are 16 bits.
__ string_equals_v($str1$$Register, $str2$$Register,
- $result$$Register, $cnt$$Register, 1);
- %}
- ins_pipe(pipe_class_memory);
-%}
-
-instruct vstring_equalsU(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt,
- iRegI_R10 result, vReg_V2 v2,
- vReg_V3 v3, vReg_V4 v4, vReg_V5 v5, rFlagsReg cr)
-%{
- predicate(UseRVV && ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
- match(Set result (StrEquals (Binary str1 str2) cnt));
- effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP v2, TEMP v3, TEMP v4, TEMP v5, KILL cr);
-
- format %{ "String Equals $str1, $str2, $cnt -> $result\t#@string_equalsU" %}
- ins_encode %{
- // Count is in 8-bit bytes; non-Compact chars are 16 bits.
- __ string_equals_v($str1$$Register, $str2$$Register,
- $result$$Register, $cnt$$Register, 2);
+ $result$$Register, $cnt$$Register);
%}
ins_pipe(pipe_class_memory);
%}
@@ -3217,7 +3200,7 @@ instruct vcvtBtoX(vReg dst, vReg src) %{
__ csrwi(CSR_FRM, C2_MacroAssembler::rne);
__ vfcvt_f_x_v(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg));
} else {
- __ integer_extend_v(as_VectorRegister($dst$$reg), bt,
+ __ integer_extend_v(as_VectorRegister($dst$$reg), bt,
Matcher::vector_length(this), as_VectorRegister($src$$reg), T_BYTE);
}
%}
diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
index 01fe307bc27c3..5945f9d5fe2ab 100644
--- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
+++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
@@ -29,7 +29,6 @@
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -38,7 +37,6 @@
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_riscv.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "oops/method.inline.hpp"
#include "prims/methodHandles.hpp"
@@ -622,10 +620,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
address c2i_unverified_entry = __ pc();
Label skip_fixup;
- Label ok;
-
- const Register holder = t1;
const Register receiver = j_rarg0;
+ const Register data = t1;
const Register tmp = t2; // A call-clobbered register not used for arg passing
// -------------------------------------------------------------------------
@@ -639,16 +635,10 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
{
__ block_comment("c2i_unverified_entry {");
- __ load_klass(t0, receiver, tmp);
- __ ld(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
- __ ld(xmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
- __ beq(t0, tmp, ok);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- __ bind(ok);
- // Method might have been compiled since the call site was patched to
- // interpreted; if that is the case treat it as a miss so we can get
- // the call site corrected.
+ __ ic_check();
+ __ ld(xmethod, Address(data, CompiledICData::speculated_method_offset()));
+
__ ld(t0, Address(xmethod, in_bytes(Method::code_offset())));
__ beqz(t0, skip_fixup);
__ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
@@ -985,7 +975,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ j(exit);
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, tr_call);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1051,7 +1041,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
}
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, tr_call);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1271,6 +1261,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
in_ByteSize(-1),
oop_maps,
exception_offset);
+ if (nm == nullptr) return nm;
if (method->is_continuation_enter_intrinsic()) {
ContinuationEntry::set_enter_code(nm, interpreted_entry_offset);
} else if (method->is_continuation_yield_intrinsic()) {
@@ -1424,19 +1415,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
const Register ic_reg = t1;
const Register receiver = j_rarg0;
- Label hit;
- Label exception_pending;
-
__ verify_oop(receiver);
- assert_different_registers(ic_reg, receiver, t0, t2);
- __ cmp_klass(receiver, ic_reg, t0, t2 /* call-clobbered t2 as a tmp */, hit);
-
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- // Verified entry point must be aligned
- __ align(8);
+ assert_different_registers(receiver, t0, t1);
- __ bind(hit);
+ __ ic_check();
int vep_offset = ((intptr_t)__ pc()) - start;
@@ -1697,8 +1679,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ sd(swap_reg, Address(lock_reg, mark_word_offset));
__ bnez(swap_reg, slow_path_lock);
} else {
- assert(LockingMode == LM_LIGHTWEIGHT, "");
- __ ld(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
__ lightweight_lock(obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock);
}
@@ -1824,9 +1805,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "");
- __ ld(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- __ test_bit(t0, old_hdr, exact_log2(markWord::monitor_value));
- __ bnez(t0, slow_path_unlock);
__ lightweight_unlock(obj_reg, old_hdr, swap_reg, lock_tmp, slow_path_unlock);
__ decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
}
@@ -1871,6 +1849,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ leave();
// Any exception pending?
+ Label exception_pending;
__ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset())));
__ bnez(t0, exception_pending);
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index 4bd33d08f8928..bbdafb922cc46 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -4809,6 +4809,348 @@ class StubGenerator: public StubCodeGenerator {
return (address) start;
}
+
+ // ------------------------ SHA-1 intrinsic ------------------------
+
+ // K't =
+ // 5a827999, 0 <= t <= 19
+ // 6ed9eba1, 20 <= t <= 39
+ // 8f1bbcdc, 40 <= t <= 59
+ // ca62c1d6, 60 <= t <= 79
+ void sha1_prepare_k(Register cur_k, int round) {
+ assert(round >= 0 && round < 80, "must be");
+
+ static const int64_t ks[] = {0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6};
+ if ((round % 20) == 0) {
+ __ mv(cur_k, ks[round/20]);
+ }
+ }
+
+ // W't =
+ // M't, 0 <= t <= 15
+ // ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
+ void sha1_prepare_w(Register cur_w, Register ws[], Register buf, int round) {
+ assert(round >= 0 && round < 80, "must be");
+
+ if (round < 16) {
+ // in the first 16 rounds, in ws[], every register contains 2 W't, e.g.
+ // in ws[0], high part contains W't-0, low part contains W't-1,
+ // in ws[1], high part contains W't-2, low part contains W't-3,
+ // ...
+ // in ws[7], high part contains W't-14, low part contains W't-15.
+
+ if ((round % 2) == 0) {
+ __ ld(ws[round/2], Address(buf, (round/2) * 8));
+ // reverse bytes, as SHA-1 is defined in big-endian.
+ __ revb(ws[round/2], ws[round/2]);
+ __ srli(cur_w, ws[round/2], 32);
+ } else {
+ __ mv(cur_w, ws[round/2]);
+ }
+
+ return;
+ }
+
+ if ((round % 2) == 0) {
+ int idx = 16;
+ // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
+ __ srli(t1, ws[(idx-8)/2], 32);
+ __ xorr(t0, ws[(idx-3)/2], t1);
+
+ __ srli(t1, ws[(idx-14)/2], 32);
+ __ srli(cur_w, ws[(idx-16)/2], 32);
+ __ xorr(cur_w, cur_w, t1);
+
+ __ xorr(cur_w, cur_w, t0);
+ __ rolw_imm(cur_w, cur_w, 1, t0);
+
+ // copy the cur_w value to ws[8].
+ // now, valid w't values are at:
+ // w0: ws[0]'s lower 32 bits
+ // w1 ~ w14: ws[1] ~ ws[7]
+ // w15: ws[8]'s higher 32 bits
+ __ slli(ws[idx/2], cur_w, 32);
+
+ return;
+ }
+
+ int idx = 17;
+ // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
+ __ srli(t1, ws[(idx-3)/2], 32);
+ __ xorr(t0, t1, ws[(idx-8)/2]);
+
+ __ xorr(cur_w, ws[(idx-16)/2], ws[(idx-14)/2]);
+
+ __ xorr(cur_w, cur_w, t0);
+ __ rolw_imm(cur_w, cur_w, 1, t0);
+
+ // copy the cur_w value to ws[8]
+ __ zero_extend(cur_w, cur_w, 32);
+ __ orr(ws[idx/2], ws[idx/2], cur_w);
+
+ // shift the w't registers, so they start from ws[0] again.
+ // now, valid w't values are at:
+ // w0 ~ w15: ws[0] ~ ws[7]
+ Register ws_0 = ws[0];
+ for (int i = 0; i < 16/2; i++) {
+ ws[i] = ws[i+1];
+ }
+ ws[8] = ws_0;
+ }
+
+ // f't(x, y, z) =
+ // Ch(x, y, z) = (x & y) ^ (~x & z) , 0 <= t <= 19
+ // Parity(x, y, z) = x ^ y ^ z , 20 <= t <= 39
+ // Maj(x, y, z) = (x & y) ^ (x & z) ^ (y & z) , 40 <= t <= 59
+ // Parity(x, y, z) = x ^ y ^ z , 60 <= t <= 79
+ void sha1_f(Register dst, Register x, Register y, Register z, int round) {
+ assert(round >= 0 && round < 80, "must be");
+ assert_different_registers(dst, x, y, z, t0, t1);
+
+ if (round < 20) {
+ // (x & y) ^ (~x & z)
+ __ andr(t0, x, y);
+ __ andn(dst, z, x);
+ __ xorr(dst, dst, t0);
+ } else if (round >= 40 && round < 60) {
+ // (x & y) ^ (x & z) ^ (y & z)
+ __ andr(t0, x, y);
+ __ andr(t1, x, z);
+ __ andr(dst, y, z);
+ __ xorr(dst, dst, t0);
+ __ xorr(dst, dst, t1);
+ } else {
+ // x ^ y ^ z
+ __ xorr(dst, x, y);
+ __ xorr(dst, dst, z);
+ }
+ }
+
+ // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't
+ // e = d
+ // d = c
+ // c = ROTL'30(b)
+ // b = a
+ // a = T
+ void sha1_process_round(Register a, Register b, Register c, Register d, Register e,
+ Register cur_k, Register cur_w, Register tmp, int round) {
+ assert(round >= 0 && round < 80, "must be");
+ assert_different_registers(a, b, c, d, e, cur_w, cur_k, tmp, t0);
+
+ // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't
+
+ // cur_w will be recalculated at the beginning of each round,
+ // so, we can reuse it as a temp register here.
+ Register tmp2 = cur_w;
+
+ // reuse e as a temporary register, as we will mv new value into it later
+ Register tmp3 = e;
+ __ add(tmp2, cur_k, tmp2);
+ __ add(tmp3, tmp3, tmp2);
+ __ rolw_imm(tmp2, a, 5, t0);
+
+ sha1_f(tmp, b, c, d, round);
+
+ __ add(tmp2, tmp2, tmp);
+ __ add(tmp2, tmp2, tmp3);
+
+ // e = d
+ // d = c
+ // c = ROTL'30(b)
+ // b = a
+ // a = T
+ __ mv(e, d);
+ __ mv(d, c);
+
+ __ rolw_imm(c, b, 30);
+ __ mv(b, a);
+ __ mv(a, tmp2);
+ }
+
+ // H(i)0 = a + H(i-1)0
+ // H(i)1 = b + H(i-1)1
+ // H(i)2 = c + H(i-1)2
+ // H(i)3 = d + H(i-1)3
+ // H(i)4 = e + H(i-1)4
+ void sha1_calculate_im_hash(Register a, Register b, Register c, Register d, Register e,
+ Register prev_ab, Register prev_cd, Register prev_e) {
+ assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e);
+
+ __ add(a, a, prev_ab);
+ __ srli(prev_ab, prev_ab, 32);
+ __ add(b, b, prev_ab);
+
+ __ add(c, c, prev_cd);
+ __ srli(prev_cd, prev_cd, 32);
+ __ add(d, d, prev_cd);
+
+ __ add(e, e, prev_e);
+ }
+
+ void sha1_preserve_prev_abcde(Register a, Register b, Register c, Register d, Register e,
+ Register prev_ab, Register prev_cd, Register prev_e) {
+ assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e, t0);
+
+ __ slli(t0, b, 32);
+ __ zero_extend(prev_ab, a, 32);
+ __ orr(prev_ab, prev_ab, t0);
+
+ __ slli(t0, d, 32);
+ __ zero_extend(prev_cd, c, 32);
+ __ orr(prev_cd, prev_cd, t0);
+
+ __ mv(prev_e, e);
+ }
+
+ // Intrinsic for:
+ // void sun.security.provider.SHA.implCompress0(byte[] buf, int ofs)
+ // void sun.security.provider.DigestBase.implCompressMultiBlock0(byte[] b, int ofs, int limit)
+ //
+ // Arguments:
+ //
+ // Inputs:
+ // c_rarg0: byte[] src array + offset
+ // c_rarg1: int[] SHA.state
+ // - - - - - - below are only for implCompressMultiBlock0 - - - - - -
+ // c_rarg2: int offset
+ // c_rarg3: int limit
+ //
+ // Outputs:
+ // - - - - - - below are only for implCompressMultiBlock0 - - - - - -
+ // c_rarg0: int offset, when (multi_block == true)
+ //
+ address generate_sha1_implCompress(bool multi_block, const char *name) {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", name);
+
+ address start = __ pc();
+ __ enter();
+
+ RegSet saved_regs = RegSet::range(x18, x27);
+ if (multi_block) {
+ // use x9 as src below.
+ saved_regs += RegSet::of(x9);
+ }
+ __ push_reg(saved_regs, sp);
+
+ // c_rarg0 - c_rarg3: x10 - x13
+ Register buf = c_rarg0;
+ Register state = c_rarg1;
+ Register offset = c_rarg2;
+ Register limit = c_rarg3;
+ // use src to contain the original start point of the array.
+ Register src = x9;
+
+ if (multi_block) {
+ __ sub(limit, limit, offset);
+ __ add(limit, limit, buf);
+ __ sub(src, buf, offset);
+ }
+
+ // [args-reg]: x14 - x17
+ // [temp-reg]: x28 - x31
+ // [saved-reg]: x18 - x27
+
+ // h0/1/2/3/4
+ const Register a = x14, b = x15, c = x16, d = x17, e = x28;
+ // w0, w1, ... w15
+ // put two adjecent w's in one register:
+ // one at high word part, another at low word part
+ // at different round (even or odd), w't value reside in different items in ws[].
+ // w0 ~ w15, either reside in
+ // ws[0] ~ ws[7], where
+ // w0 at higher 32 bits of ws[0],
+ // w1 at lower 32 bits of ws[0],
+ // ...
+ // w14 at higher 32 bits of ws[7],
+ // w15 at lower 32 bits of ws[7].
+ // or, reside in
+ // w0: ws[0]'s lower 32 bits
+ // w1 ~ w14: ws[1] ~ ws[7]
+ // w15: ws[8]'s higher 32 bits
+ Register ws[9] = {x29, x30, x31, x18,
+ x19, x20, x21, x22,
+ x23}; // auxiliary register for calculating w's value
+ // current k't's value
+ const Register cur_k = x24;
+ // current w't's value
+ const Register cur_w = x25;
+ // values of a, b, c, d, e in the previous round
+ const Register prev_ab = x26, prev_cd = x27;
+ const Register prev_e = offset; // reuse offset/c_rarg2
+
+ // load 5 words state into a, b, c, d, e.
+ //
+ // To minimize the number of memory operations, we apply following
+ // optimization: read the states (a/b/c/d) of 4-byte values in pairs,
+ // with a single ld, and split them into 2 registers.
+ //
+ // And, as the core algorithm of SHA-1 works on 32-bits words, so
+ // in the following code, it does not care about the content of
+ // higher 32-bits in a/b/c/d/e. Based on this observation,
+ // we can apply further optimization, which is to just ignore the
+ // higher 32-bits in a/c/e, rather than set the higher
+ // 32-bits of a/c/e to zero explicitly with extra instructions.
+ __ ld(a, Address(state, 0));
+ __ srli(b, a, 32);
+ __ ld(c, Address(state, 8));
+ __ srli(d, c, 32);
+ __ lw(e, Address(state, 16));
+
+ Label L_sha1_loop;
+ if (multi_block) {
+ __ BIND(L_sha1_loop);
+ }
+
+ sha1_preserve_prev_abcde(a, b, c, d, e, prev_ab, prev_cd, prev_e);
+
+ for (int round = 0; round < 80; round++) {
+ // prepare K't value
+ sha1_prepare_k(cur_k, round);
+
+ // prepare W't value
+ sha1_prepare_w(cur_w, ws, buf, round);
+
+ // one round process
+ sha1_process_round(a, b, c, d, e, cur_k, cur_w, t2, round);
+ }
+
+ // compute the intermediate hash value
+ sha1_calculate_im_hash(a, b, c, d, e, prev_ab, prev_cd, prev_e);
+
+ if (multi_block) {
+ int64_t block_bytes = 16 * 4;
+ __ addi(buf, buf, block_bytes);
+
+ __ bge(limit, buf, L_sha1_loop, true);
+ }
+
+ // store back the state.
+ __ zero_extend(a, a, 32);
+ __ slli(b, b, 32);
+ __ orr(a, a, b);
+ __ sd(a, Address(state, 0));
+ __ zero_extend(c, c, 32);
+ __ slli(d, d, 32);
+ __ orr(c, c, d);
+ __ sd(c, Address(state, 8));
+ __ sw(e, Address(state, 16));
+
+ // return offset
+ if (multi_block) {
+ __ sub(c_rarg0, buf, src);
+ }
+
+ __ pop_reg(saved_regs, sp);
+
+ __ leave();
+ __ ret();
+
+ return (address) start;
+ }
+
+
+
#ifdef COMPILER2
static const int64_t right_2_bits = right_n_bits(2);
@@ -5273,6 +5615,11 @@ static const int64_t right_3_bits = right_n_bits(3);
StubRoutines::_chacha20Block = generate_chacha20Block();
}
+ if (UseSHA1Intrinsics) {
+ StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
+ StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
+ }
+
#endif // COMPILER2_OR_JVMCI
}
diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
index 7c604e8c11cc2..90a7e0967b240 100644
--- a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
+++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
@@ -39,7 +39,7 @@ enum platform_dependent_constants {
// simply increase sizes if too small (assembler will crash if too small)
_initial_stubs_code_size = 10000,
_continuation_stubs_code_size = 2000,
- _compiler_stubs_code_size = 15000 ZGC_ONLY(+5000),
+ _compiler_stubs_code_size = 25000 ZGC_ONLY(+5000),
_final_stubs_code_size = 20000 ZGC_ONLY(+10000)
};
diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
index 73b4d1e28ccef..58f57f32b2f65 100644
--- a/src/hotspot/cpu/riscv/templateTable_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
@@ -3547,11 +3547,10 @@ void TemplateTable::_new() {
// get InstanceKlass
__ load_resolved_klass_at_offset(x14, x13, x14, t0);
- // make sure klass is initialized & doesn't have finalizer
- // make sure klass is fully initialized
- __ lbu(t0, Address(x14, InstanceKlass::init_state_offset()));
- __ sub(t1, t0, (u1)InstanceKlass::fully_initialized);
- __ bnez(t1, slow_case);
+ // make sure klass is initialized
+ assert(VM_Version::supports_fast_class_init_checks(),
+ "Optimization requires support for fast class initialization checks");
+ __ clinit_barrier(x14, t0, nullptr /*L_fast_path*/, &slow_case);
// get instance_size in InstanceKlass (scaled to a count of bytes)
__ lwu(x13, Address(x14, Klass::layout_helper_offset()));
diff --git a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
index 142d9d636938d..8a6557dde93a1 100644
--- a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
+++ b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
@@ -267,9 +267,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ mov_metadata(xmethod, entry);
__ sd(xmethod, Address(xthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+ __ push_cont_fastpath(xthread);
+
__ ld(t0, Address(xmethod, Method::from_compiled_offset()));
__ jalr(t0);
+ __ pop_cont_fastpath(xthread);
+
// return value shuffle
if (!needs_return_buffer) {
#ifdef ASSERT
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
index 9a72b8d75a136..e1711dc5592f9 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
@@ -81,6 +81,9 @@ void VM_Version::initialize() {
if (FLAG_IS_DEFAULT(UseZbs)) {
FLAG_SET_DEFAULT(UseZbs, true);
}
+ if (FLAG_IS_DEFAULT(UseZfh)) {
+ FLAG_SET_DEFAULT(UseZfh, true);
+ }
if (FLAG_IS_DEFAULT(UseZic64b)) {
FLAG_SET_DEFAULT(UseZic64b, true);
}
@@ -146,16 +149,6 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
}
- if (UseSHA1Intrinsics) {
- warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
- FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
- }
-
- if (UseSHA3Intrinsics) {
- warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
- FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
- }
-
if (UseCRC32Intrinsics) {
warning("CRC32 intrinsics are not available on this CPU.");
FLAG_SET_DEFAULT(UseCRC32Intrinsics, false);
@@ -257,11 +250,8 @@ void VM_Version::initialize() {
// NOTE: Make sure codes dependent on UseRVV are put after c2_initialize(),
// as there are extra checks inside it which could disable UseRVV
// in some situations.
- if (UseZvkn && !UseRVV) {
- FLAG_SET_DEFAULT(UseZvkn, false);
- warning("Cannot enable Zvkn on cpu without RVV support.");
- }
+ // ChaCha20
if (UseRVV) {
if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
FLAG_SET_DEFAULT(UseChaCha20Intrinsics, true);
@@ -273,29 +263,65 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
}
- if (!UseZvkn && UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- } else if (UseZvkn && FLAG_IS_DEFAULT(UseSHA)) {
+ // SHA's
+ if (FLAG_IS_DEFAULT(UseSHA)) {
FLAG_SET_DEFAULT(UseSHA, true);
}
- if (!UseSHA) {
+ // SHA-1, no RVV required though.
+ if (UseSHA) {
+ if (FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA1Intrinsics, true);
+ }
+ } else if (UseSHA1Intrinsics) {
+ warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
+ FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
+ }
+
+ // UseZvkn (depends on RVV) and SHA-2.
+ if (UseZvkn && !UseRVV) {
+ FLAG_SET_DEFAULT(UseZvkn, false);
+ warning("Cannot enable Zvkn on cpu without RVV support.");
+ }
+ // SHA-2, depends on Zvkn.
+ if (UseSHA) {
+ if (UseZvkn) {
+ if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
+ }
+ if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
+ }
+ } else {
+ if (UseSHA256Intrinsics) {
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU, UseZvkn needed.");
+ FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
+ }
+ if (UseSHA512Intrinsics) {
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU, UseZvkn needed.");
+ FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
+ }
+ }
+ } else {
if (UseSHA256Intrinsics) {
- warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU, UseZvkn needed.");
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU, as UseSHA disabled.");
FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
}
if (UseSHA512Intrinsics) {
- warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU, UseZvkn needed.");
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU, as UseSHA disabled.");
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
- } else {
- if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
- }
- if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
- }
+ }
+
+ // SHA-3
+ if (UseSHA3Intrinsics) {
+ warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
+ }
+
+ // UseSHA
+ if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA3Intrinsics || UseSHA512Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA, false);
}
}
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
index 1ea853284ff15..e6f79cd6379d0 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
@@ -105,6 +105,8 @@ class VM_Version : public Abstract_VM_Version {
// Zbc Carry-less multiplication
// Zbs Single-bit instructions
//
+ // Zfh Half-Precision Floating-Point instructions
+ //
// Zicsr Control and Status Register (CSR) Instructions
// Zifencei Instruction-Fetch Fence
// Zic64b Cache blocks must be 64 bytes in size, naturally aligned in the address space.
@@ -142,7 +144,8 @@ class VM_Version : public Abstract_VM_Version {
decl(ext_Zbb , "Zbb" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbb)) \
decl(ext_Zbc , "Zbc" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
decl(ext_Zbs , "Zbs" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbs)) \
- decl(ext_Zcb , "Zcb" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
+ decl(ext_Zcb , "Zcb" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZcb)) \
+ decl(ext_Zfh , "Zfh" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfh)) \
decl(ext_Zicsr , "Zicsr" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
decl(ext_Zifencei , "Zifencei" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
decl(ext_Zic64b , "Zic64b" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b)) \
@@ -214,6 +217,8 @@ class VM_Version : public Abstract_VM_Version {
constexpr static bool supports_stack_watermark_barrier() { return true; }
+ constexpr static bool supports_recursive_lightweight_locking() { return true; }
+
static bool supports_on_spin_wait() { return UseZihintpause; }
// RISCV64 supports fast class initialization checks
diff --git a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
index 9d08796681f3f..5d945dbc32309 100644
--- a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
@@ -27,10 +27,10 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_riscv.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -171,22 +171,22 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// Entry arguments:
- // t1: CompiledICHolder
+ // t1: CompiledICData
// j_rarg0: Receiver
// This stub is called from compiled code which has no callee-saved registers,
// so all registers except arguments are free at this point.
const Register recv_klass_reg = x18;
- const Register holder_klass_reg = x19; // declaring interface klass (DECC)
+ const Register holder_klass_reg = x19; // declaring interface klass (DEFC)
const Register resolved_klass_reg = x30; // resolved interface klass (REFC)
const Register temp_reg = x28;
const Register temp_reg2 = x29;
- const Register icholder_reg = t1;
+ const Register icdata_reg = t1;
Label L_no_such_interface;
- __ ld(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
- __ ld(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
+ __ ld(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+ __ ld(holder_klass_reg, Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
start_pc = __ pc();
diff --git a/src/hotspot/cpu/s390/assembler_s390.hpp b/src/hotspot/cpu/s390/assembler_s390.hpp
index 9bb143001b944..91cc7e611bfd1 100644
--- a/src/hotspot/cpu/s390/assembler_s390.hpp
+++ b/src/hotspot/cpu/s390/assembler_s390.hpp
@@ -107,7 +107,7 @@ class RelAddr {
static bool is_in_range_of_RelAddr(address target, address pc, bool shortForm) {
// Guard against illegal branch targets, e.g. -1. Occurrences in
- // CompiledStaticCall and ad-file. Do not assert (it's a test
+ // CompiledDirectCall and ad-file. Do not assert (it's a test
// function!). Just return false in case of illegal operands.
if ((((uint64_t)target) & 0x0001L) != 0) return false;
if ((((uint64_t)pc) & 0x0001L) != 0) return false;
diff --git a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
index 200f7ee978dcb..b7f1d3605681a 100644
--- a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -428,6 +428,7 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
"must be aligned");
ce->emit_static_call_stub();
+ CHECK_BAILOUT();
// Prepend each BRASL with a nop.
__ relocate(relocInfo::static_call_type);
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index 542ade8ed0ec3..503440a5fcc01 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -76,10 +76,7 @@ int LIR_Assembler::initial_frame_size_in_bytes() const {
// We fetch the class of the receiver and compare it with the cached class.
// If they do not match we jump to the slow case.
int LIR_Assembler::check_icache() {
- Register receiver = receiverOpr()->as_register();
- int offset = __ offset();
- __ inline_cache_check(receiver, Z_inline_cache);
- return offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -2385,7 +2382,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->len()->as_register(),
op->tmp1()->as_register(),
op->tmp2()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -2480,23 +2477,30 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
assert_different_registers(obj, k_RInfo, klass_RInfo);
if (op->should_profile()) {
+ Register mdo = klass_RInfo;
+ metadata2reg(md->constant_encoding(), mdo);
NearLabel not_null;
__ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
// Object is null; update MDO and exit.
- Register mdo = klass_RInfo;
- metadata2reg(md->constant_encoding(), mdo);
Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
__ or2mem_8(data_addr, header_bits);
__ branch_optimized(Assembler::bcondAlways, *obj_is_null);
__ bind(not_null);
+
+ NearLabel update_done;
+ Register recv = k_RInfo;
+ __ load_klass(recv, obj);
+ type_profile_helper(mdo, md, data, recv, Rtmp1, &update_done);
+ Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
+ __ add2mem_64(counter_addr, DataLayout::counter_increment, Rtmp1);
+ __ bind(update_done);
} else {
__ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondEqual, *obj_is_null);
}
- NearLabel profile_cast_failure, profile_cast_success;
- Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
- Label *success_target = op->should_profile() ? &profile_cast_success : success;
+ Label *failure_target = failure;
+ Label *success_target = success;
// Patching may screw with our temporaries,
// so let's do it before loading the class.
@@ -2536,28 +2540,12 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
store_parameter(klass_RInfo, 0); // sub
store_parameter(k_RInfo, 1); // super
emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
- CHECK_BAILOUT2(profile_cast_failure, profile_cast_success);
__ branch_optimized(Assembler::bcondNotEqual, *failure_target);
// Fall through to success case.
}
}
- if (op->should_profile()) {
- Register mdo = klass_RInfo, recv = k_RInfo;
- assert_different_registers(obj, mdo, recv);
- __ bind(profile_cast_success);
- metadata2reg(md->constant_encoding(), mdo);
- __ load_klass(recv, obj);
- type_profile_helper(mdo, md, data, recv, Rtmp1, success);
- __ branch_optimized(Assembler::bcondAlways, *success);
-
- __ bind(profile_cast_failure);
- metadata2reg(md->constant_encoding(), mdo);
- __ add2mem_64(Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())), -(int)DataLayout::counter_increment, Rtmp1);
- __ branch_optimized(Assembler::bcondAlways, *failure);
- } else {
- __ branch_optimized(Assembler::bcondAlways, *success);
- }
+ __ branch_optimized(Assembler::bcondAlways, *success);
}
void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
@@ -2587,21 +2575,29 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
assert(data != nullptr, "need data for type check");
assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
}
- NearLabel profile_cast_success, profile_cast_failure, done;
- Label *success_target = op->should_profile() ? &profile_cast_success : &done;
- Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
+ NearLabel done;
+ Label *success_target = &done;
+ Label *failure_target = stub->entry();
if (op->should_profile()) {
+ Register mdo = klass_RInfo;
+ metadata2reg(md->constant_encoding(), mdo);
NearLabel not_null;
__ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
// Object is null; update MDO and exit.
- Register mdo = klass_RInfo;
- metadata2reg(md->constant_encoding(), mdo);
Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
__ or2mem_8(data_addr, header_bits);
__ branch_optimized(Assembler::bcondAlways, done);
__ bind(not_null);
+
+ NearLabel update_done;
+ Register recv = k_RInfo;
+ __ load_klass(recv, value);
+ type_profile_helper(mdo, md, data, recv, Rtmp1, &update_done);
+ Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
+ __ add2mem_64(counter_addr, DataLayout::counter_increment, Rtmp1);
+ __ bind(update_done);
} else {
__ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondEqual, done);
}
@@ -2619,25 +2615,9 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
store_parameter(klass_RInfo, 0); // sub
store_parameter(k_RInfo, 1); // super
emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
- CHECK_BAILOUT3(profile_cast_success, profile_cast_failure, done);
__ branch_optimized(Assembler::bcondNotEqual, *failure_target);
// Fall through to success case.
- if (op->should_profile()) {
- Register mdo = klass_RInfo, recv = k_RInfo;
- assert_different_registers(value, mdo, recv);
- __ bind(profile_cast_success);
- metadata2reg(md->constant_encoding(), mdo);
- __ load_klass(recv, value);
- type_profile_helper(mdo, md, data, recv, Rtmp1, &done);
- __ branch_optimized(Assembler::bcondAlways, done);
-
- __ bind(profile_cast_failure);
- metadata2reg(md->constant_encoding(), mdo);
- __ add2mem_64(Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())), -(int)DataLayout::counter_increment, Rtmp1);
- __ branch_optimized(Assembler::bcondAlways, *stub->entry());
- }
-
__ bind(done);
} else {
if (code == lir_checkcast) {
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.hpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.hpp
index 229216ef20d44..c8815f3a729a4 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.hpp
@@ -45,7 +45,7 @@
}
enum {
- _call_stub_size = 512, // See Compile::MAX_stubs_size and CompiledStaticCall::emit_to_interp_stub.
+ _call_stub_size = 512, // See Compile::MAX_stubs_size and CompiledDirectCall::emit_to_interp_stub.
_exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(128),
_deopt_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(64)
};
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
index 40edca6559aa4..58bdcee5d5f8f 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -40,31 +40,6 @@
#include "runtime/stubRoutines.hpp"
#include "utilities/macros.hpp"
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- Label ic_miss, ic_hit;
- verify_oop(receiver, FILE_AND_LINE);
- int klass_offset = oopDesc::klass_offset_in_bytes();
-
- if (!ImplicitNullChecks || MacroAssembler::needs_explicit_null_check(klass_offset)) {
- if (VM_Version::has_CompareBranch()) {
- z_cgij(receiver, 0, Assembler::bcondEqual, ic_miss);
- } else {
- z_ltgr(receiver, receiver);
- z_bre(ic_miss);
- }
- }
-
- compare_klass_ptr(iCache, klass_offset, receiver, false);
- z_bre(ic_hit);
-
- // If icache check fails, then jump to runtime routine.
- // Note: RECEIVER must still contain the receiver!
- load_const_optimized(Z_R1_scratch, AddressLiteral(SharedRuntime::get_ic_miss_stub()));
- z_br(Z_R1_scratch);
- align(CodeEntryAlignment);
- bind(ic_hit);
-}
-
void C1_MacroAssembler::explicit_null_check(Register base) {
ShouldNotCallThis(); // unused
}
@@ -296,7 +271,7 @@ void C1_MacroAssembler::allocate_array(
Register len, // array length
Register t1, // temp register
Register t2, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // Continuation point if fast allocation fails.
@@ -322,8 +297,8 @@ void C1_MacroAssembler::allocate_array(
case 8: z_sllg(arr_size, len, 3); break;
default: ShouldNotReachHere();
}
- add2reg(arr_size, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
- z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size.
+ add2reg(arr_size, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
+ z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size.
try_allocate(obj, arr_size, 0, t1, slow_case);
@@ -333,9 +308,9 @@ void C1_MacroAssembler::allocate_array(
Label done;
Register object_fields = t1;
Register Rzero = Z_R1_scratch;
- z_aghi(arr_size, -(hdr_size * BytesPerWord));
+ z_aghi(arr_size, -base_offset_in_bytes);
z_bre(done); // Jump if size of fields is zero.
- z_la(object_fields, hdr_size * BytesPerWord, obj);
+ z_la(object_fields, base_offset_in_bytes, obj);
z_xgr(Rzero, Rzero);
initialize_body(object_fields, arr_size, Rzero);
bind(done);
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp
index 7a4f76af1546e..c77258509e1a5 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -86,7 +86,7 @@
Register len, // array length
Register t1, // temp register
Register t2, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // Continuation point if fast allocation fails.
diff --git a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
index 257148827be4e..decb3a1cafc31 100644
--- a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
@@ -35,7 +35,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_s390.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_s390.hpp"
diff --git a/src/hotspot/cpu/s390/compiledIC_s390.cpp b/src/hotspot/cpu/s390/compiledIC_s390.cpp
index 7ea90c1de7c69..3adcfbc85f185 100644
--- a/src/hotspot/cpu/s390/compiledIC_s390.cpp
+++ b/src/hotspot/cpu/s390/compiledIC_s390.cpp
@@ -26,7 +26,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/mutexLocker.hpp"
@@ -40,7 +39,7 @@
#undef __
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/* = nullptr*/) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/* = nullptr*/) {
#ifdef COMPILER2
// Stub is fixed up when the corresponding call is converted from calling
// compiled code to calling interpreted code.
@@ -54,7 +53,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
// That's why we must use the macroassembler to generate a stub.
MacroAssembler _masm(&cbuf);
- address stub = __ start_a_stub(CompiledStaticCall::to_interp_stub_size());
+ address stub = __ start_a_stub(CompiledDirectCall::to_interp_stub_size());
if (stub == nullptr) {
return nullptr; // CodeBuffer::expand failed.
}
@@ -81,27 +80,20 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
#undef __
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return 2 * MacroAssembler::load_const_from_toc_size() +
2; // branch
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 5; // 4 in emit_java_to_interp + 1 in Java_Static_Call
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder = nativeMovConstReg_at(stub + NativeCall::get_IC_pos_in_java_to_interp_stub());
NativeJump* jump = nativeJump_at(method_holder->next_instruction_address());
@@ -115,7 +107,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -131,7 +123,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
index 3ed99f68c475c..8ce9305a865e5 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
@@ -31,9 +31,9 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/sharedRuntime.hpp"
diff --git a/src/hotspot/cpu/s390/icBuffer_s390.cpp b/src/hotspot/cpu/s390/icBuffer_s390.cpp
deleted file mode 100644
index 0dc936d6fad0c..0000000000000
--- a/src/hotspot/cpu/s390/icBuffer_s390.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_s390.hpp"
-#include "oops/oop.inline.hpp"
-
-#define __ masm.
-
-int InlineCacheBuffer::ic_stub_code_size() {
- return MacroAssembler::load_const_size() + Assembler::z_brul_size();
-}
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_oop, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler masm(&code);
- // Note: even though the code contains an embedded oop, we do not need reloc info
- // because
- // (1) the oop is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear.
-
- // Load the oop,
- __ load_const(Z_method, (address) cached_oop); // inline cache reg = Z_method
- // and do a tail-call (pc-relative).
- __ z_brul((address) entry_point);
- __ flush();
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // Creation also verifies the object.
- return MacroAssembler::get_target_addr_pcrel(move->next_instruction_address());
-}
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // Creation also verifies the object.
- return (void*)move->data();
-}
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp
index ea691342b2397..9ee38c619f0b0 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.cpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -447,9 +447,6 @@ void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
// Do the check.
check_klass_subtype(Rsub_klass, Rsuper_klass, Rtmp1, Rtmp2, ok_is_subtype);
-
- // Profile the failure of the check.
- profile_typecheck_failed(Rtmp1, Rtmp2);
}
// Pop topmost element from stack. It just disappears.
@@ -1425,7 +1422,7 @@ void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
}
// Record the receiver type.
- record_klass_in_profile(receiver, mdp, reg2, true);
+ record_klass_in_profile(receiver, mdp, reg2);
bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
@@ -1448,11 +1445,9 @@ void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
void InterpreterMacroAssembler::record_klass_in_profile_helper(
Register receiver, Register mdp,
Register reg2, int start_row,
- Label& done, bool is_virtual_call) {
+ Label& done) {
if (TypeProfileWidth == 0) {
- if (is_virtual_call) {
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- }
+ increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
return;
}
@@ -1487,23 +1482,19 @@ void InterpreterMacroAssembler::record_klass_in_profile_helper(
z_ltgr(reg2, reg2);
if (start_row == last_row) {
// The only thing left to do is handle the null case.
- if (is_virtual_call) {
- z_brz(found_null);
- // Receiver did not match any saved receiver and there is no empty row for it.
- // Increment total counter to indicate polymorphic case.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- z_bru(done);
- bind(found_null);
- } else {
- z_brnz(done);
- }
+ z_brz(found_null);
+ // Receiver did not match any saved receiver and there is no empty row for it.
+ // Increment total counter to indicate polymorphic case.
+ increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
+ z_bru(done);
+ bind(found_null);
break;
}
// Since null is rare, make it be the branch-taken case.
z_brz(found_null);
// Put all the "Case 3" tests here.
- record_klass_in_profile_helper(receiver, mdp, reg2, start_row + 1, done, is_virtual_call);
+ record_klass_in_profile_helper(receiver, mdp, reg2, start_row + 1, done);
// Found a null. Keep searching for a matching receiver,
// but remember that this is an empty (unused) slot.
@@ -1550,12 +1541,11 @@ void InterpreterMacroAssembler::record_klass_in_profile_helper(
// done:
void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
- Register mdp, Register reg2,
- bool is_virtual_call) {
+ Register mdp, Register reg2) {
assert(ProfileInterpreter, "must be profiling");
Label done;
- record_klass_in_profile_helper(receiver, mdp, reg2, 0, done, is_virtual_call);
+ record_klass_in_profile_helper(receiver, mdp, reg2, 0, done);
bind (done);
}
@@ -1615,24 +1605,6 @@ void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
}
}
-void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp, Register tmp) {
- if (ProfileInterpreter && TypeProfileCasts) {
- Label profile_continue;
-
- // If no method data exists, go to profile_continue.
- test_method_data_pointer(mdp, profile_continue);
-
- int count_offset = in_bytes(CounterData::count_offset());
- // Back up the address, since we have already bumped the mdp.
- count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
-
- // *Decrement* the counter. We expect to see zero or small negatives.
- increment_mdp_data_at(mdp, count_offset, tmp, true);
-
- bind (profile_continue);
- }
-}
-
void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) {
if (ProfileInterpreter) {
Label profile_continue;
@@ -1646,7 +1618,7 @@ void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass,
mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
// Record the object type.
- record_klass_in_profile(klass, mdp, reg2, false);
+ record_klass_in_profile(klass, mdp, reg2);
}
update_mdp_by_constant(mdp, mdp_delta);
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.hpp b/src/hotspot/cpu/s390/interp_masm_s390.hpp
index 56abe295876aa..f94473b1700b7 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.hpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -281,10 +281,10 @@ class InterpreterMacroAssembler: public MacroAssembler {
Label& not_equal_continue);
void record_klass_in_profile(Register receiver, Register mdp,
- Register reg2, bool is_virtual_call);
+ Register reg2);
void record_klass_in_profile_helper(Register receiver, Register mdp,
Register reg2, int start_row,
- Label& done, bool is_virtual_call);
+ Label& done);
void update_mdp_by_offset(Register mdp_in, int offset_of_offset);
void update_mdp_by_offset(Register mdp_in, Register dataidx, int offset_of_disp);
@@ -301,7 +301,6 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass, Register scratch);
- void profile_typecheck_failed(Register mdp, Register tmp);
void profile_switch_default(Register mdp);
void profile_switch_case(Register index_in_scratch, Register mdp,
Register scratch1, Register scratch2);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index d95a0b3a3c5bd..0226d494c8958 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -26,6 +26,7 @@
#include "precompiled.hpp"
#include "asm/codeBuffer.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -1097,7 +1098,13 @@ void MacroAssembler::clear_mem(const Address& addr, unsigned int size) {
}
void MacroAssembler::align(int modulus) {
- while (offset() % modulus != 0) z_nop();
+ align(modulus, offset());
+}
+
+void MacroAssembler::align(int modulus, int target) {
+ assert(((modulus % 2 == 0) && (target % 2 == 0)), "needs to be even");
+ int delta = target - offset();
+ while ((offset() + delta) % modulus != 0) z_nop();
}
// Special version for non-relocateable code if required alignment
@@ -2150,6 +2157,45 @@ void MacroAssembler::call_VM_leaf_base(address entry_point) {
call_VM_leaf_base(entry_point, allow_relocation);
}
+int MacroAssembler::ic_check_size() {
+ return 30 + (ImplicitNullChecks ? 0 : 6);
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ Register R2_receiver = Z_ARG1;
+ Register R0_scratch = Z_R0_scratch;
+ Register R1_scratch = Z_R1_scratch;
+ Register R9_data = Z_inline_cache;
+ Label success, failure;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, offset() + ic_check_size());
+
+ int uep_offset = offset();
+ if (!ImplicitNullChecks) {
+ z_cgij(R2_receiver, 0, Assembler::bcondEqual, failure);
+ }
+
+ if (UseCompressedClassPointers) {
+ z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
+ } else {
+ z_lg(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
+ }
+ z_cg(R1_scratch, Address(R9_data, in_bytes(CompiledICData::speculated_klass_offset())));
+ z_bre(success);
+
+ bind(failure);
+ load_const(R1_scratch, AddressLiteral(SharedRuntime::get_ic_miss_stub()));
+ z_br(R1_scratch);
+ bind(success);
+
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point, offset() = %d, end_alignment = %d", offset(), end_alignment);
+ return uep_offset;
+}
+
void MacroAssembler::call_VM_base(Register oop_result,
Register last_java_sp,
address entry_point,
@@ -3744,7 +3790,7 @@ void MacroAssembler::compare_klass_ptr(Register Rop1, int64_t disp, Register Rba
Register current = Rop1;
Label done;
- if (maybenull) { // null ptr must be preserved!
+ if (maybenull) { // null pointer must be preserved!
z_ltgr(Z_R0, current);
z_bre(done);
current = Z_R0;
@@ -3883,7 +3929,7 @@ void MacroAssembler::compare_heap_oop(Register Rop1, Address mem, bool maybenull
Label done;
int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
- if (maybenull) { // null ptr must be preserved!
+ if (maybenull) { // null pointer must be preserved!
z_ltgr(Z_R0, Rop1);
z_bre(done);
}
@@ -4123,7 +4169,7 @@ void MacroAssembler::oop_decoder(Register Rdst, Register Rsrc, bool maybenull, R
Label done;
// Rsrc contains a narrow oop. Thus we are sure the leftmost bits will never be set.
- if (maybenull) { // null ptr must be preserved!
+ if (maybenull) { // null pointer must be preserved!
z_slag(Rdst, Rsrc, oop_shift); // Arithmetic shift sets the condition code.
z_bre(done);
} else {
@@ -4185,7 +4231,7 @@ void MacroAssembler::oop_decoder(Register Rdst, Register Rsrc, bool maybenull, R
// Scale oop and check for null.
// Rsrc contains a narrow oop. Thus we are sure the leftmost bits will never be set.
- if (maybenull) { // null ptr must be preserved!
+ if (maybenull) { // null pointer must be preserved!
z_slag(Rdst_tmp, Rsrc, oop_shift); // Arithmetic shift sets the condition code.
z_bre(done);
} else {
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index bf14b42e2d1b3..924583abdf563 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -257,6 +257,7 @@ class MacroAssembler: public Assembler {
// nop padding
void align(int modulus);
+ void align(int modulus, int target);
void align_address(int modulus);
//
@@ -566,6 +567,9 @@ class MacroAssembler: public Assembler {
// Get the pc where the last call will return to. Returns _last_calls_return_pc.
inline address last_calls_return_pc();
+ static int ic_check_size();
+ int ic_check(int end_alignment);
+
private:
static bool is_call_far_patchable_variant0_at(address instruction_addr); // Dynamic TOC: load target addr from CP and call.
static bool is_call_far_patchable_variant2_at(address instruction_addr); // PC-relative call, prefixed with NOPs.
diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad
index 9f4e182a9e4b7..5fcb885cdc3ab 100644
--- a/src/hotspot/cpu/s390/s390.ad
+++ b/src/hotspot/cpu/s390/s390.ad
@@ -1,6 +1,6 @@
//
-// Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2017, 2022 SAP SE. All rights reserved.
+// Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2017, 2024 SAP SE. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -740,7 +740,7 @@ int emit_call_reloc(C2_MacroAssembler &_masm, intptr_t entry_point, relocInfo::r
if (rtype == relocInfo::runtime_call_w_cp_type) {
assert((__ offset() & 2) == 0, "misaligned emit_call_reloc");
address call_addr = __ call_c_opt((address)entry_point);
- if (call_addr == NULL) {
+ if (call_addr == nullptr) {
Compile::current()->env()->record_out_of_memory_failure();
return -1;
}
@@ -835,7 +835,7 @@ void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
st->print_cr("push_frame %d", (int)-framesize);
st->print("\t");
- if (C->stub_function() == NULL) {
+ if (C->stub_function() == nullptr) {
st->print("nmethod entry barrier\n\t");
}
}
@@ -890,7 +890,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
}
- if (C->stub_function() == NULL) {
+ if (C->stub_function() == nullptr) {
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->nmethod_entry_barrier(&_masm);
}
@@ -1056,7 +1056,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
bool src12 = Immediate::is_uimm12(src_offset);
bool dst12 = Immediate::is_uimm12(dst_offset);
- const char *mnemo = NULL;
+ const char *mnemo = nullptr;
unsigned long opc = 0;
// Memory->Memory Spill. Use Z_R0 to hold the value.
@@ -1199,7 +1199,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
#if !defined(PRODUCT)
void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *os) const {
if (ra_ && ra_->node_regs_max_index() > 0) {
- implementation(NULL, ra_, false, os);
+ implementation(nullptr, ra_, false, os);
} else {
if (req() == 2 && in(1)) {
os->print("N%d = N%d\n", _idx, in(1)->_idx);
@@ -1217,11 +1217,11 @@ void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *os) const {
#endif
void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- implementation(&cbuf, ra_, false, NULL);
+ implementation(&cbuf, ra_, false, nullptr);
}
uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
- return implementation(NULL, ra_, true, NULL);
+ return implementation(nullptr, ra_, true, nullptr);
}
//=============================================================================
@@ -1341,51 +1341,9 @@ void MachUEPNode::format(PhaseRegAlloc *ra_, outputStream *os) const {
#endif
void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
+ // This is Unverified Entry Point
C2_MacroAssembler _masm(&cbuf);
- const int ic_miss_offset = 2;
-
- // Inline_cache contains a klass.
- Register ic_klass = as_Register(Matcher::inline_cache_reg_encode());
- // ARG1 is the receiver oop.
- Register R2_receiver = Z_ARG1;
- int klass_offset = oopDesc::klass_offset_in_bytes();
- AddressLiteral icmiss(SharedRuntime::get_ic_miss_stub());
- Register R1_ic_miss_stub_addr = Z_R1_scratch;
-
- // Null check of receiver.
- // This is the null check of the receiver that actually should be
- // done in the caller. It's here because in case of implicit null
- // checks we get it for free.
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()),
- "second word in oop should not require explicit null check.");
- if (!ImplicitNullChecks) {
- Label valid;
- if (VM_Version::has_CompareBranch()) {
- __ z_cgij(R2_receiver, 0, Assembler::bcondNotEqual, valid);
- } else {
- __ z_ltgr(R2_receiver, R2_receiver);
- __ z_bre(valid);
- }
- // The ic_miss_stub will handle the null pointer exception.
- __ load_const_optimized(R1_ic_miss_stub_addr, icmiss);
- __ z_br(R1_ic_miss_stub_addr);
- __ bind(valid);
- }
-
- // Check whether this method is the proper implementation for the class of
- // the receiver (ic miss check).
- {
- Label valid;
- // Compare cached class against klass from receiver.
- // This also does an implicit null check!
- __ compare_klass_ptr(ic_klass, klass_offset, R2_receiver, false);
- __ z_bre(valid);
- // The inline cache points to the wrong method. Call the
- // ic_miss_stub to find the proper method.
- __ load_const_optimized(R1_ic_miss_stub_addr, icmiss);
- __ z_br(R1_ic_miss_stub_addr);
- __ bind(valid);
- }
+ __ ic_check(CodeEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
@@ -1446,7 +1404,8 @@ int HandlerImpl::emit_exception_handler(CodeBuffer &cbuf) {
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_exception_handler());
- if (base == NULL) {
+ if (base == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -1467,7 +1426,8 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_deopt_handler());
- if (base == NULL) {
+ if (base == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -1513,7 +1473,7 @@ bool Matcher::match_rule_supported(int opcode) {
return true; // Per default match rules are supported.
}
-bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
return match_rule_supported_vector(opcode, vlen, bt);
}
@@ -1533,11 +1493,11 @@ bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
}
const RegMask* Matcher::predicate_reg_mask(void) {
- return NULL;
+ return nullptr;
}
const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return NULL;
+ return nullptr;
}
// Vector calling convention not yet implemented.
@@ -1574,7 +1534,7 @@ int Matcher::min_vector_size(const BasicType bt) {
return max_vector_size(bt); // Same as max.
}
-int Matcher::superword_max_vector_size(const BasicType bt) {
+int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
return Matcher::max_vector_size(bt);
}
@@ -1602,7 +1562,7 @@ bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
ShouldNotReachHere(); // generic vector operands not supported
- return NULL;
+ return nullptr;
}
bool Matcher::is_reg2reg_move(MachNode* m) {
@@ -1948,12 +1908,12 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Label* p = $lbl$$label;
- // 'p' is `NULL' when this encoding class is used only to
+ // 'p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
// Use a bound dummy label in that case.
Label d;
__ bind(d);
- Label& l = (NULL == p) ? d : *(p);
+ Label& l = (nullptr == p) ? d : *(p);
__ z_brul(l);
%}
@@ -1961,12 +1921,12 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Label* p = $lbl$$label;
- // 'p' is `NULL' when this encoding class is used only to
+ // 'p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
// Use a bound dummy label in that case.
Label d;
__ bind(d);
- Label& l = (NULL == p) ? d : *(p);
+ Label& l = (nullptr == p) ? d : *(p);
__ z_bru(l);
%}
@@ -1974,12 +1934,12 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Label* p = $lbl$$label;
- // 'p' is `NULL' when this encoding class is used only to
+ // 'p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
// Use a bound dummy label in that case.
Label d;
__ bind(d);
- Label& l = (NULL == p) ? d : *(p);
+ Label& l = (nullptr == p) ? d : *(p);
__ z_brcl((Assembler::branch_condition)$cmp$$cmpcode, l);
%}
@@ -1987,12 +1947,12 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Label* p = $lbl$$label;
- // 'p' is `NULL' when this encoding class is used only to
+ // 'p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
// Use a bound dummy label in that case.
Label d;
__ bind(d);
- Label& l = (NULL == p) ? d : *(p);
+ Label& l = (nullptr == p) ? d : *(p);
__ z_brc((Assembler::branch_condition)$cmp$$cmpcode, l);
%}
@@ -2000,12 +1960,12 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Label* p = $lbl$$label;
- // 'p' is `NULL' when this encoding class is used only to
+ // 'p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
// Use a bound dummy label in that case.
Label d;
__ bind(d);
- Label& l = (NULL == p) ? d : *(p);
+ Label& l = (nullptr == p) ? d : *(p);
Assembler::branch_condition cc = (Assembler::branch_condition)$cmp$$cmpcode;
unsigned long instr = $primary;
if (instr == CRJ_ZOPC) {
@@ -2024,12 +1984,12 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Label* p = $lbl$$label;
- // 'p' is `NULL' when this encoding class is used only to
+ // 'p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
// Use a bound dummy label in that case.
Label d;
__ bind(d);
- Label& l = (NULL == p) ? d : *(p);
+ Label& l = (nullptr == p) ? d : *(p);
unsigned long instr = $primary;
if (instr == CR_ZOPC) {
@@ -2050,12 +2010,12 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Label* p = $lbl$$label;
- // 'p' is `NULL' when this encoding class is used only to
+ // 'p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
// Use a bound dummy label in that case.
Label d;
__ bind(d);
- Label& l = (NULL == p) ? d : *(p);
+ Label& l = (nullptr == p) ? d : *(p);
Assembler::branch_condition cc = (Assembler::branch_condition)$cmp$$cmpcode;
unsigned long instr = $primary;
@@ -2075,12 +2035,12 @@ encode %{
C2_MacroAssembler _masm(&cbuf);
Label* p = $lbl$$label;
- // 'p' is `NULL' when this encoding class is used only to
+ // 'p' is `nullptr' when this encoding class is used only to
// determine the size of the encoded instruction.
// Use a bound dummy label in that case.
Label d;
__ bind(d);
- Label& l = (NULL == p) ? d : *(p);
+ Label& l = (nullptr == p) ? d : *(p);
unsigned long instr = $primary;
if (instr == CHI_ZOPC) {
@@ -2112,7 +2072,7 @@ encode %{
assert((__ offset() & 2) == 0, "misaligned z_enc_java_to_runtime_call");
address call_addr = __ call_c_opt((address)$meth$$method);
- if (call_addr == NULL) {
+ if (call_addr == nullptr) {
Compile::current()->env()->record_out_of_memory_failure();
return;
}
@@ -2143,11 +2103,11 @@ encode %{
static_call_Relocation::spec(method_index));
}
}
- assert(__ inst_mark() != NULL, "emit_call_reloc must set_inst_mark()");
+ assert(__ inst_mark() != nullptr, "emit_call_reloc must set_inst_mark()");
if (_method) { // Emit stub for static call.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
- if (stub == NULL) {
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf);
+ if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -2162,7 +2122,7 @@ encode %{
int vtable_index = this->_vtable_index;
if (vtable_index == -4) {
Register ic_reg = reg_to_register_object(Matcher::inline_cache_reg_encode());
- address virtual_call_oop_addr = NULL;
+ address virtual_call_oop_addr = nullptr;
AddressLiteral empty_ic((address) Universe::non_oop_word());
virtual_call_oop_addr = __ pc();
@@ -2929,7 +2889,7 @@ operand immP8() %{
// POINTER specific values
//-----------------------------------
-// Pointer Immediate: NULL
+// Pointer Immediate: nullptr
operand immP0() %{
predicate(n->get_ptr() == 0);
match(ConP);
@@ -2966,7 +2926,7 @@ operand immN8() %{
interface(CONST_INTER);
%}
-// Narrow NULL Pointer Immediate
+// Narrow Null Pointer Immediate
operand immN0() %{
predicate(n->get_narrowcon() == 0);
match(ConN);
@@ -3383,7 +3343,7 @@ operand inline_cache_regP(iRegP reg) %{
// Operands to remove register moves in unscaled mode.
// Match read/write registers with an EncodeP node if neither shift nor add are required.
operand iRegP2N(iRegP reg) %{
- predicate(CompressedOops::shift() == 0 && _leaf->as_EncodeP()->in(0) == NULL);
+ predicate(CompressedOops::shift() == 0 && _leaf->as_EncodeP()->in(0) == nullptr);
constraint(ALLOC_IN_RC(z_memory_ptr_reg));
match(EncodeP reg);
format %{ "$reg" %}
@@ -3391,8 +3351,8 @@ operand iRegP2N(iRegP reg) %{
%}
operand iRegN2P(iRegN reg) %{
- predicate(CompressedOops::base() == NULL && CompressedOops::shift() == 0 &&
- _leaf->as_DecodeN()->in(0) == NULL);
+ predicate(CompressedOops::base() == nullptr && CompressedOops::shift() == 0 &&
+ _leaf->as_DecodeN()->in(0) == nullptr);
constraint(ALLOC_IN_RC(z_memory_ptr_reg));
match(DecodeN reg);
format %{ "$reg" %}
@@ -4236,7 +4196,7 @@ instruct loadConL_pcrelTOC(iRegL dst, immL src) %{
format %{ "LGRL $dst,[pcrelTOC]\t # load long $src from table" %}
ins_encode %{
address long_address = __ long_constant($src$$constant);
- if (long_address == NULL) {
+ if (long_address == nullptr) {
Compile::current()->env()->record_out_of_memory_failure();
return;
}
@@ -4292,14 +4252,14 @@ instruct loadConP_pcrelTOC(iRegP dst, immP src) %{
} else if (constant_reloc == relocInfo::metadata_type) {
AddressLiteral a = __ constant_metadata_address((Metadata *)$src$$constant);
address const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- if (const_toc_addr == NULL) {
+ if (const_toc_addr == nullptr) {
Compile::current()->env()->record_out_of_memory_failure();
return;
}
__ load_long_pcrelative($dst$$Register, const_toc_addr);
} else { // Non-oop pointers, e.g. card mark base, heap top.
address long_address = __ long_constant((jlong)$src$$constant);
- if (long_address == NULL) {
+ if (long_address == nullptr) {
Compile::current()->env()->record_out_of_memory_failure();
return;
}
@@ -4314,7 +4274,7 @@ instruct loadConP0(iRegP dst, immP0 src, flagsReg cr) %{
match(Set dst src);
effect(KILL cr);
size(4);
- format %{ "XGR $dst,$dst\t # NULL ptr" %}
+ format %{ "XGR $dst,$dst\t # null pointer" %}
opcode(XGR_ZOPC);
ins_encode(z_rreform(dst, dst));
ins_pipe(pipe_class_dummy);
@@ -4660,7 +4620,7 @@ instruct loadConNKlass(iRegN dst, immNKlass src) %{
instruct decodeLoadN(iRegP dst, memory mem) %{
match(Set dst (DecodeN (LoadN mem)));
- predicate(false && (CompressedOops::base()==NULL)&&(CompressedOops::shift()==0));
+ predicate(false && (CompressedOops::base()==nullptr)&&(CompressedOops::shift()==0));
ins_cost(MEMORY_REF_COST);
size(Z_DISP3_SIZE);
format %{ "DecodeLoadN $dst,$mem\t # (cOop Load+Decode)" %}
@@ -4671,7 +4631,7 @@ instruct decodeLoadN(iRegP dst, memory mem) %{
instruct decodeLoadNKlass(iRegP dst, memory mem) %{
match(Set dst (DecodeNKlass (LoadNKlass mem)));
- predicate(false && (CompressedKlassPointers::base()==NULL)&&(CompressedKlassPointers::shift()==0));
+ predicate(false && (CompressedKlassPointers::base()==nullptr)&&(CompressedKlassPointers::shift()==0));
ins_cost(MEMORY_REF_COST);
size(Z_DISP3_SIZE);
format %{ "DecodeLoadNKlass $dst,$mem\t # (load/decode NKlass)" %}
@@ -4699,7 +4659,7 @@ instruct decodeLoadConNKlass(iRegP dst, immNKlass src) %{
instruct decodeN(iRegP dst, iRegN src, flagsReg cr) %{
match(Set dst (DecodeN src));
effect(KILL cr);
- predicate(CompressedOops::base() == NULL || !ExpandLoadingBaseDecode);
+ predicate(CompressedOops::base() == nullptr || !ExpandLoadingBaseDecode);
ins_cost(MEMORY_REF_COST+3 * DEFAULT_COST + BRANCH_COST);
// TODO: s390 port size(VARIABLE_SIZE);
format %{ "decodeN $dst,$src\t # (decode cOop)" %}
@@ -4723,7 +4683,7 @@ instruct decodeN_NN(iRegP dst, iRegN src, flagsReg cr) %{
effect(KILL cr);
predicate((n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull ||
n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant) &&
- (CompressedOops::base()== NULL || !ExpandLoadingBaseDecode_NN));
+ (CompressedOops::base()== nullptr || !ExpandLoadingBaseDecode_NN));
ins_cost(MEMORY_REF_COST+2 * DEFAULT_COST);
// TODO: s390 port size(VARIABLE_SIZE);
format %{ "decodeN $dst,$src\t # (decode cOop NN)" %}
@@ -4749,7 +4709,7 @@ instruct decodeN_NN(iRegP dst, iRegN src, flagsReg cr) %{
effect(KILL cr);
predicate(false);
// TODO: s390 port size(VARIABLE_SIZE);
- format %{ "decodeN $dst = ($src == 0) ? NULL : ($src << 3) + $base + pow2_offset\t # (decode cOop)" %}
+ format %{ "decodeN $dst = ($src == 0) ? nullptr : ($src << 3) + $base + pow2_offset\t # (decode cOop)" %}
ins_encode %{
__ oop_decoder($dst$$Register, $src$$Register, true, $base$$Register,
(jlong)MacroAssembler::get_oop_base_pow2_offset((uint64_t)(intptr_t)CompressedOops::base()));
@@ -4774,7 +4734,7 @@ instruct decodeN_NN(iRegP dst, iRegN src, flagsReg cr) %{
// Decoder for heapbased mode peeling off loading the base.
instruct decodeN_Ex(iRegP dst, iRegN src, flagsReg cr) %{
match(Set dst (DecodeN src));
- predicate(CompressedOops::base() != NULL && ExpandLoadingBaseDecode);
+ predicate(CompressedOops::base() != nullptr && ExpandLoadingBaseDecode);
ins_cost(MEMORY_REF_COST+3 * DEFAULT_COST + BRANCH_COST);
// TODO: s390 port size(VARIABLE_SIZE);
expand %{
@@ -4790,7 +4750,7 @@ instruct decodeN_NN_Ex(iRegP dst, iRegN src, flagsReg cr) %{
match(Set dst (DecodeN src));
predicate((n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull ||
n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant) &&
- CompressedOops::base() != NULL && ExpandLoadingBaseDecode_NN);
+ CompressedOops::base() != nullptr && ExpandLoadingBaseDecode_NN);
ins_cost(MEMORY_REF_COST+2 * DEFAULT_COST);
// TODO: s390 port size(VARIABLE_SIZE);
expand %{
@@ -6069,7 +6029,7 @@ instruct addP_reg_reg_imm12(iRegP dst, memoryRegP src1, iRegL src2, uimmL12 con)
instruct addP_regN_reg_imm12(iRegP dst, iRegP_N2P src1, iRegL src2, uimmL12 con) %{
match(Set dst (AddP (AddP src1 src2) con));
- predicate( PreferLAoverADD && CompressedOops::base() == NULL && CompressedOops::shift() == 0);
+ predicate( PreferLAoverADD && CompressedOops::base() == nullptr && CompressedOops::shift() == 0);
ins_cost(DEFAULT_COST_LOW);
size(4);
format %{ "LA $dst,$con($src1,$src2)\t # ptr d12(x,b)" %}
@@ -6091,7 +6051,7 @@ instruct addP_reg_reg_imm20(iRegP dst, memoryRegP src1, iRegL src2, immL20 con)
instruct addP_regN_reg_imm20(iRegP dst, iRegP_N2P src1, iRegL src2, immL20 con) %{
match(Set dst (AddP (AddP src1 src2) con));
- predicate( PreferLAoverADD && CompressedOops::base() == NULL && CompressedOops::shift() == 0);
+ predicate( PreferLAoverADD && CompressedOops::base() == nullptr && CompressedOops::shift() == 0);
ins_cost(DEFAULT_COST);
// TODO: s390 port size(FIXED_SIZE);
format %{ "LAY $dst,$con($src1,$src2)\t # ptr d20(x,b)" %}
@@ -8427,7 +8387,7 @@ instruct compP_reg_imm0(flagsReg cr, iRegP_N2P op1, immP0 op2) %{
// Don't use LTGFR which performs sign extend.
instruct compP_decode_reg_imm0(flagsReg cr, iRegN op1, immP0 op2) %{
match(Set cr (CmpP (DecodeN op1) op2));
- predicate(CompressedOops::base() == NULL && CompressedOops::shift() == 0);
+ predicate(CompressedOops::base() == nullptr && CompressedOops::shift() == 0);
ins_cost(DEFAULT_COST_LOW);
size(2);
format %{ "LTR $op1, $op1\t # ptr" %}
@@ -9843,24 +9803,10 @@ instruct string_equalsL(iRegP str1, iRegP str2, iRegI cnt, iRegI result, roddReg
ins_pipe(pipe_class_dummy);
%}
-instruct string_equalsU(iRegP str1, iRegP str2, iRegI cnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
- match(Set result (StrEquals (Binary str1 str2) cnt));
- effect(TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
- predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU || ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::none);
- ins_cost(300);
- format %{ "String Equals char[] $str1,$str2,$cnt -> $result" %}
- ins_encode %{
- __ array_equals(false, $str1$$Register, $str2$$Register,
- $cnt$$Register, $oddReg$$Register, $evenReg$$Register,
- $result$$Register, false /* byte */);
- %}
- ins_pipe(pipe_class_dummy);
-%}
-
instruct string_equals_imm(iRegP str1, iRegP str2, uimmI8 cnt, iRegI result, flagsReg cr) %{
match(Set result (StrEquals (Binary str1 str2) cnt));
effect(KILL cr); // R0 is killed, too.
- predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL || ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
+ predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
ins_cost(100);
format %{ "String Equals byte[] $str1,$str2,$cnt -> $result" %}
ins_encode %{
diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
index ed1795cfa339f..11e1e617d8e3a 100644
--- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
+++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
@@ -26,8 +26,8 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shared/gcLocker.hpp"
@@ -35,7 +35,6 @@
#include "interpreter/interp_masm.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_s390.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "registerSaver_s390.hpp"
@@ -1500,17 +1499,15 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
unsigned int wrapper_FrameDone;
unsigned int wrapper_CRegsSet;
Label handle_pending_exception;
- Label ic_miss;
//---------------------------------------------------------------------
// Unverified entry point (UEP)
//---------------------------------------------------------------------
- wrapper_UEPStart = __ offset();
// check ic: object class <-> cached class
- if (!method_is_static) __ nmethod_UEP(ic_miss);
- // Fill with nops (alignment of verified entry point).
- __ align(CodeEntryAlignment);
+ if (!method_is_static) {
+ wrapper_UEPStart = __ ic_check(CodeEntryAlignment /* end_alignment */);
+ }
//---------------------------------------------------------------------
// Verified entry point (VEP)
@@ -2026,13 +2023,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
__ restore_return_pc();
__ z_br(Z_R1_scratch);
- //---------------------------------------------------------------------
- // Handler for a cache miss (out-of-line)
- //---------------------------------------------------------------------
- __ call_ic_miss_handler(ic_miss, 0x77, 0, Z_R1_scratch);
__ flush();
-
-
//////////////////////////////////////////////////////////////////////
// end of code generation
//////////////////////////////////////////////////////////////////////
@@ -2318,9 +2309,6 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
Label skip_fixup;
{
Label ic_miss;
- const int klass_offset = oopDesc::klass_offset_in_bytes();
- const int holder_klass_offset = in_bytes(CompiledICHolder::holder_klass_offset());
- const int holder_metadata_offset = in_bytes(CompiledICHolder::holder_metadata_offset());
// Out-of-line call to ic_miss handler.
__ call_ic_miss_handler(ic_miss, 0x11, 0, Z_R1_scratch);
@@ -2329,27 +2317,11 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
__ align(CodeEntryAlignment);
c2i_unverified_entry = __ pc();
- // Check the pointers.
- if (!ImplicitNullChecks || MacroAssembler::needs_explicit_null_check(klass_offset)) {
- __ z_ltgr(Z_ARG1, Z_ARG1);
- __ z_bre(ic_miss);
- }
- __ verify_oop(Z_ARG1, FILE_AND_LINE);
-
- // Check ic: object class <-> cached class
- // Compress cached class for comparison. That's more efficient.
- if (UseCompressedClassPointers) {
- __ z_lg(Z_R11, holder_klass_offset, Z_method); // Z_R11 is overwritten a few instructions down anyway.
- __ compare_klass_ptr(Z_R11, klass_offset, Z_ARG1, false); // Cached class can't be zero.
- } else {
- __ z_clc(klass_offset, sizeof(void *)-1, Z_ARG1, holder_klass_offset, Z_method);
- }
- __ z_brne(ic_miss); // Cache miss: call runtime to handle this.
-
+ __ ic_check(2);
+ __ z_lg(Z_method, Address(Z_inline_cache, CompiledICData::speculated_method_offset()));
// This def MUST MATCH code in gen_c2i_adapter!
const Register code = Z_R11;
- __ z_lg(Z_method, holder_metadata_offset, Z_method);
__ load_and_test_long(Z_R0, method_(code));
__ z_brne(ic_miss); // Cache miss: call runtime to handle this.
diff --git a/src/hotspot/cpu/s390/templateTable_s390.cpp b/src/hotspot/cpu/s390/templateTable_s390.cpp
index 62d62a9842f83..02b9405ad31d5 100644
--- a/src/hotspot/cpu/s390/templateTable_s390.cpp
+++ b/src/hotspot/cpu/s390/templateTable_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2868,7 +2868,7 @@ void TemplateTable::jvmti_post_field_mod(Register cache,
__ z_lgr(fieldEntry, cache);
if (is_static) {
- // Life is simple. NULL the object pointer.
+ // Life is simple. Null the object pointer.
__ clear_reg(obj, true, false); // Don't set CC.
} else {
// Life is harder. The stack holds the value on top, followed by
@@ -3914,20 +3914,15 @@ void TemplateTable::_new() {
__ z_cli(0, tmp, JVM_CONSTANT_Class);
__ z_brne(slow_case);
- __ z_sllg(offset, offset, LogBytesPerWord); // Convert to to offset.
+ __ z_sllg(offset, offset, LogBytesPerWord); // Convert to offset.
// Get InstanceKlass.
Register iklass = cpool;
__ load_resolved_klass_at_offset(cpool, offset, iklass);
- // Make sure klass is initialized & doesn't have finalizer.
- // Make sure klass is fully initialized.
- const int state_offset = in_bytes(InstanceKlass::init_state_offset());
- if (Immediate::is_uimm12(state_offset)) {
- __ z_cli(state_offset, iklass, InstanceKlass::fully_initialized);
- } else {
- __ z_cliy(state_offset, iklass, InstanceKlass::fully_initialized);
- }
- __ z_brne(slow_case);
+ // make sure klass is initialized
+ assert(VM_Version::supports_fast_class_init_checks(),
+ "Optimization requires support for fast class initialization checks");
+ __ clinit_barrier(iklass, Z_thread, nullptr /*L_fast_path*/, &slow_case);
// Get instance_size in InstanceKlass (scaled to a count of bytes).
Register Rsize = offset;
diff --git a/src/hotspot/cpu/s390/vm_version_s390.hpp b/src/hotspot/cpu/s390/vm_version_s390.hpp
index 93d5b11c473d1..7ac60a10ae7f8 100644
--- a/src/hotspot/cpu/s390/vm_version_s390.hpp
+++ b/src/hotspot/cpu/s390/vm_version_s390.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -567,7 +567,6 @@ class VM_Version: public Abstract_VM_Version {
static unsigned long z_SIGSEGV();
static void initialize_cpu_information(void);
- static bool profile_all_receivers_at_type_check() { return false; }
};
#endif // CPU_S390_VM_VERSION_S390_HPP
diff --git a/src/hotspot/cpu/s390/vtableStubs_s390.cpp b/src/hotspot/cpu/s390/vtableStubs_s390.cpp
index 5a79369ceab47..573c23d796708 100644
--- a/src/hotspot/cpu/s390/vtableStubs_s390.cpp
+++ b/src/hotspot/cpu/s390/vtableStubs_s390.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2021 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,10 +25,10 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_s390.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klass.inline.hpp"
#include "oops/klassVtable.hpp"
@@ -197,12 +197,12 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
__ load_klass(rcvr_klass, Z_ARG1);
// Receiver subtype check against REFC.
- __ z_lg(interface, Address(Z_method, CompiledICHolder::holder_klass_offset()));
+ __ z_lg(interface, Address(Z_method, CompiledICData::itable_refc_klass_offset()));
__ lookup_interface_method(rcvr_klass, interface, noreg,
noreg, Z_R1, no_such_interface, /*return_method=*/ false);
// Get Method* and entrypoint for compiler
- __ z_lg(interface, Address(Z_method, CompiledICHolder::holder_metadata_offset()));
+ __ z_lg(interface, Address(Z_method, CompiledICData::itable_defc_klass_offset()));
__ lookup_interface_method(rcvr_klass, interface, itable_index,
Z_method, Z_R1, no_such_interface, /*return_method=*/ true);
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index 63d6ec9b00f1b..9482537d84f90 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -1089,7 +1089,7 @@ address Assembler::locate_operand(address inst, WhichOperand which) {
break;
case 0x62: // EVEX_4bytes
- assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
+ assert(VM_Version::cpu_supports_evex(), "shouldn't have EVEX prefix");
assert(ip == inst+1, "no prefixes allowed");
// no EVEX collisions, all instructions that have 0x62 opcodes
// have EVEX versions and are subopcodes of 0x66
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index 7b907218f3516..8b512fac6bc05 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -816,8 +816,8 @@ class Assembler : public AbstractAssembler {
void check_relocation(RelocationHolder const& rspec, int format);
#endif
- void emit_data(jint data, relocInfo::relocType rtype, int format);
- void emit_data(jint data, RelocationHolder const& rspec, int format);
+ void emit_data(jint data, relocInfo::relocType rtype, int format = 0);
+ void emit_data(jint data, RelocationHolder const& rspec, int format = 0);
void emit_data64(jlong data, relocInfo::relocType rtype, int format = 0);
void emit_data64(jlong data, RelocationHolder const& rspec, int format = 0);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index ff0726840d30a..c279e3073af87 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,7 +72,6 @@ static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jl
NEEDS_CLEANUP // remove this definitions ?
-const Register IC_Klass = rax; // where the IC klass is cached
const Register SYNC_header = rax; // synchronization header
const Register SHIFT_count = rcx; // where count for shift operations must be
@@ -336,23 +335,7 @@ void LIR_Assembler::osr_entry() {
// inline cache check; done before the frame is built.
int LIR_Assembler::check_icache() {
- Register receiver = FrameMap::receiver_opr->as_register();
- Register ic_klass = IC_Klass;
- const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
- const bool do_post_padding = VerifyOops || UseCompressedClassPointers;
- if (!do_post_padding) {
- // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
- __ align(CodeEntryAlignment, __ offset() + ic_cmp_size);
- }
- int offset = __ offset();
- __ inline_cache_check(receiver, IC_Klass);
- assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
- if (do_post_padding) {
- // force alignment after the cache check.
- // It's been verified to be aligned if !VerifyOops
- __ align(CodeEntryAlignment);
- }
- return offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -1635,7 +1618,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
index b6a27abf0f37e..7088cf33cf646 100644
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1207,9 +1207,10 @@ void LIRGenerator::do_vectorizedMismatch(Intrinsic* x) {
__ move(result_reg, result);
}
+#ifndef _LP64
// _i2l, _i2f, _i2d, _l2i, _l2f, _l2d, _f2i, _f2l, _f2d, _d2i, _d2l, _d2f
// _i2b, _i2c, _i2s
-LIR_Opr fixed_register_for(BasicType type) {
+static LIR_Opr fixed_register_for(BasicType type) {
switch (type) {
case T_FLOAT: return FrameMap::fpu0_float_opr;
case T_DOUBLE: return FrameMap::fpu0_double_opr;
@@ -1218,6 +1219,7 @@ LIR_Opr fixed_register_for(BasicType type) {
default: ShouldNotReachHere(); return LIR_OprFact::illegalOpr;
}
}
+#endif
void LIRGenerator::do_Convert(Convert* x) {
#ifdef _LP64
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
index 78361a305aeeb..caca3a1528261 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "c1/c1_Runtime1.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/compilerDefinitions.inline.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -34,10 +35,12 @@
#include "oops/arrayOop.hpp"
#include "oops/markWord.hpp"
#include "runtime/basicLock.hpp"
+#include "runtime/globals.hpp"
#include "runtime/os.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/checkedCast.hpp"
+#include "utilities/globalDefinitions.hpp"
int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
@@ -60,9 +63,6 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
jcc(Assembler::notZero, slow_case);
}
- // Load object header
- movptr(hdr, Address(obj, hdr_offset));
-
if (LockingMode == LM_LIGHTWEIGHT) {
#ifdef _LP64
const Register thread = r15_thread;
@@ -73,6 +73,8 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
lightweight_lock(obj, hdr, thread, tmp, slow_case);
} else if (LockingMode == LM_LEGACY) {
Label done;
+ // Load object header
+ movptr(hdr, Address(obj, hdr_offset));
// and mark it as unlocked
orptr(hdr, markWord::unlocked_value);
// save unlocked object header into the displaced header location on the stack
@@ -134,9 +136,14 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
verify_oop(obj);
if (LockingMode == LM_LIGHTWEIGHT) {
- movptr(disp_hdr, Address(obj, hdr_offset));
- andptr(disp_hdr, ~(int32_t)markWord::lock_mask_in_place);
- lightweight_unlock(obj, disp_hdr, hdr, slow_case);
+#ifdef _LP64
+ lightweight_unlock(obj, disp_hdr, r15_thread, hdr, slow_case);
+#else
+ // This relies on the implementation of lightweight_unlock being able to handle
+ // that the reg_rax and thread Register parameters may alias each other.
+ get_thread(disp_hdr);
+ lightweight_unlock(obj, disp_hdr, disp_hdr, hdr, slow_case);
+#endif
} else if (LockingMode == LM_LEGACY) {
// test if object header is pointing to the displaced header, and if so, restore
// the displaced header in the object - if the object header is not pointing to
@@ -179,6 +186,15 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
+#ifdef _LP64
+ int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
+ if (!is_aligned(base_offset, BytesPerWord)) {
+ assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+ // Clear gap/first 4 bytes following the length field.
+ xorl(t1, t1);
+ movl(Address(obj, base_offset), t1);
+ }
+#endif
}
#ifdef _LP64
else if (UseCompressedClassPointers) {
@@ -262,7 +278,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case) {
assert(obj == rax, "obj must be in rax, for cmpxchg");
assert_different_registers(obj, len, t1, t2, klass);
@@ -275,7 +291,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
const Register arr_size = t2; // okay to be the same
// align object end
- movptr(arr_size, header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ movptr(arr_size, base_offset_in_bytes + MinObjAlignmentInBytesMask);
lea(arr_size, Address(arr_size, len, f));
andptr(arr_size, ~MinObjAlignmentInBytesMask);
@@ -285,7 +301,10 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
// clear rest of allocated space
const Register len_zero = len;
- initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
+ // Align-up to word boundary, because we clear the 4 bytes potentially
+ // following the length field in initialize_header().
+ int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
+ initialize_body(obj, arr_size, base_offset, len_zero);
if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == rax, "must be");
@@ -295,30 +314,6 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
verify_oop(obj);
}
-
-
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- verify_oop(receiver);
- // explicit null check not needed since load from [klass_offset] causes a trap
- // check against inline cache
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
- int start_offset = offset();
-
- if (UseCompressedClassPointers) {
- load_klass(rscratch1, receiver, rscratch2);
- cmpptr(rscratch1, iCache);
- } else {
- cmpptr(iCache, Address(receiver, oopDesc::klass_offset_in_bytes()));
- }
- // if icache check fails, then jump to runtime routine
- // Note: RECEIVER must still contain the receiver!
- jump_cc(Assembler::notEqual,
- RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
- assert(UseCompressedClassPointers || offset() - start_offset == ic_cmp_size, "check alignment in emit_method_entry");
-}
-
-
void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
// Make sure there is enough stack space for this method's activation.
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
index b3593feb05640..ae340e64fb737 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -89,7 +89,7 @@
// header_size: size of object header in words
// f : element scale factor
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_array(Register obj, Register len, Register t, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case);
+ void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case);
int rsp_offset() const { return _rsp_offset; }
void set_rsp_offset(int n) { _rsp_offset = n; }
diff --git a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
index 8b56f464f2739..2c24c0c2cfb17 100644
--- a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
@@ -38,7 +38,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_x86.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_x86.hpp"
diff --git a/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp b/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp
index b9b4e8af02c5f..6dc8d14064ad2 100644
--- a/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -73,26 +73,74 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
__ jmp(continuation(), false /* maybe_short */);
}
-#ifdef _LP64
-int C2HandleAnonOMOwnerStub::max_size() const {
- // Max size of stub has been determined by testing with 0, in which case
- // C2CodeStubList::emit() will throw an assertion and report the actual size that
- // is needed.
- return DEBUG_ONLY(36) NOT_DEBUG(21);
+int C2FastUnlockLightweightStub::max_size() const {
+ return 128;
}
-void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
- __ bind(entry());
- Register mon = monitor();
- Register t = tmp();
- __ movptr(Address(mon, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), r15_thread);
- __ subl(Address(r15_thread, JavaThread::lock_stack_top_offset()), oopSize);
+void C2FastUnlockLightweightStub::emit(C2_MacroAssembler& masm) {
+ assert(_t == rax, "must be");
+
+ Label restore_held_monitor_count_and_slow_path;
+
+ { // Restore lock-stack and handle the unlock in runtime.
+
+ __ bind(_push_and_slow_path);
#ifdef ASSERT
- __ movl(t, Address(r15_thread, JavaThread::lock_stack_top_offset()));
- __ movptr(Address(r15_thread, t), 0);
+ // The obj was only cleared in debug.
+ __ movl(_t, Address(_thread, JavaThread::lock_stack_top_offset()));
+ __ movptr(Address(_thread, _t), _obj);
#endif
- __ jmp(continuation());
-}
+ __ addl(Address(_thread, JavaThread::lock_stack_top_offset()), oopSize);
+ }
+
+ { // Restore held monitor count and slow path.
+
+ __ bind(restore_held_monitor_count_and_slow_path);
+ // Restore held monitor count.
+ __ increment(Address(_thread, JavaThread::held_monitor_count_offset()));
+ // increment will always result in ZF = 0 (no overflows).
+ __ jmp(slow_path_continuation());
+ }
+
+ { // Handle monitor medium path.
+
+ __ bind(_check_successor);
+
+ Label fix_zf_and_unlocked;
+ const Register monitor = _mark;
+
+#ifndef _LP64
+ __ jmpb(restore_held_monitor_count_and_slow_path);
+#else // _LP64
+ // successor null check.
+ __ cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), NULL_WORD);
+ __ jccb(Assembler::equal, restore_held_monitor_count_and_slow_path);
+
+ // Release lock.
+ __ movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
+
+ // Fence.
+ // Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack.
+ __ lock(); __ addl(Address(rsp, 0), 0);
+
+ // Recheck successor.
+ __ cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), NULL_WORD);
+ // Observed a successor after the release -> fence we have handed off the monitor
+ __ jccb(Assembler::notEqual, fix_zf_and_unlocked);
+
+ // Try to relock, if it fails the monitor has been handed over
+ // TODO: Caveat, this may fail due to deflation, which does
+ // not handle the monitor handoff. Currently only works
+ // due to the responsible thread.
+ __ xorptr(rax, rax);
+ __ lock(); __ cmpxchgptr(_thread, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+ __ jccb (Assembler::equal, restore_held_monitor_count_and_slow_path);
#endif
+ __ bind(fix_zf_and_unlocked);
+ __ xorl(rax, rax);
+ __ jmp(unlocked_continuation());
+ }
+}
+
#undef __
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 3817c38f4ba3b..b6ecde62af655 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -33,9 +33,13 @@
#include "opto/output.hpp"
#include "opto/opcodes.hpp"
#include "opto/subnode.hpp"
+#include "runtime/globals.hpp"
#include "runtime/objectMonitor.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/checkedCast.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+#include "utilities/sizes.hpp"
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -554,6 +558,7 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
RTMLockingCounters* stack_rtm_counters,
Metadata* method_data,
bool use_rtm, bool profile_rtm) {
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
// Ensure the register assignments are disjoint
assert(tmpReg == rax, "");
@@ -605,7 +610,8 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
if (LockingMode == LM_MONITOR) {
// Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
testptr(objReg, objReg);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Attempt stack-locking ...
orptr (tmpReg, markWord::unlocked_value);
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
@@ -620,10 +626,6 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
// Next instruction set ZFlag == 1 (Success) if difference is less then one page.
andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - (int)os::vm_page_size())) );
movptr(Address(boxReg, 0), tmpReg);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "");
- lightweight_lock(objReg, tmpReg, thread, scrReg, NO_COUNT);
- jmp(COUNT);
}
jmp(DONE_LABEL);
@@ -754,6 +756,7 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
// Xcheck:jni is enabled.
void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
assert(boxReg == rax, "");
assert_different_registers(objReg, boxReg, tmpReg);
@@ -784,23 +787,6 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
}
// It's inflated.
- if (LockingMode == LM_LIGHTWEIGHT) {
- // If the owner is ANONYMOUS, we need to fix it - in an outline stub.
- testb(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t) ObjectMonitor::ANONYMOUS_OWNER);
-#ifdef _LP64
- if (!Compile::current()->output()->in_scratch_emit_size()) {
- C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmpReg, boxReg);
- Compile::current()->output()->add_stub(stub);
- jcc(Assembler::notEqual, stub->entry());
- bind(stub->continuation());
- } else
-#endif
- {
- // We can't easily implement this optimization on 32 bit because we don't have a thread register.
- // Call the slow-path instead.
- jcc(Assembler::notEqual, NO_COUNT);
- }
- }
#if INCLUDE_RTM_OPT
if (use_rtm) {
@@ -922,19 +908,14 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
jmpb (DONE_LABEL);
#endif
- if (LockingMode != LM_MONITOR) {
+ if (LockingMode == LM_LEGACY) {
bind (Stacked);
- if (LockingMode == LM_LIGHTWEIGHT) {
- mov(boxReg, tmpReg);
- lightweight_unlock(objReg, boxReg, tmpReg, NO_COUNT);
- jmp(COUNT);
- } else if (LockingMode == LM_LEGACY) {
- movptr(tmpReg, Address (boxReg, 0)); // re-fetch
- lock();
- cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
- }
+ movptr(tmpReg, Address (boxReg, 0)); // re-fetch
+ lock();
+ cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
// Intentional fall-thru into DONE_LABEL
}
+
bind(DONE_LABEL);
// ZFlag == 1 count in fast path
@@ -955,6 +936,247 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
bind(NO_COUNT);
}
+void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register rax_reg,
+ Register t, Register thread) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert(rax_reg == rax, "Used for CAS");
+ assert_different_registers(obj, box, rax_reg, t, thread);
+
+ // Handle inflated monitor.
+ Label inflated;
+ // Finish fast lock successfully. ZF value is irrelevant.
+ Label locked;
+ // Finish fast lock unsuccessfully. MUST jump with ZF == 0
+ Label slow_path;
+
+ if (DiagnoseSyncOnValueBasedClasses != 0) {
+ load_klass(rax_reg, obj, t);
+ movl(rax_reg, Address(rax_reg, Klass::access_flags_offset()));
+ testl(rax_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
+ jcc(Assembler::notZero, slow_path);
+ }
+
+ const Register mark = t;
+
+ { // Lightweight Lock
+
+ Label push;
+
+ const Register top = box;
+
+ // Load the mark.
+ movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Prefetch top.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+
+ // Check for monitor (0b10).
+ testptr(mark, markWord::monitor_value);
+ jcc(Assembler::notZero, inflated);
+
+ // Check if lock-stack is full.
+ cmpl(top, LockStack::end_offset() - 1);
+ jcc(Assembler::greater, slow_path);
+
+ // Check if recursive.
+ cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
+ jccb(Assembler::equal, push);
+
+ // Try to lock. Transition lock bits 0b01 => 0b00
+ movptr(rax_reg, mark);
+ orptr(rax_reg, markWord::unlocked_value);
+ andptr(mark, ~(int32_t)markWord::unlocked_value);
+ lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ jcc(Assembler::notEqual, slow_path);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ movptr(Address(thread, top), obj);
+ addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+ jmpb(locked);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated);
+
+ const Register tagged_monitor = mark;
+
+ // CAS owner (null => current thread).
+ xorptr(rax_reg, rax_reg);
+ lock(); cmpxchgptr(thread, Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+ jccb(Assembler::equal, locked);
+
+ // Check if recursive.
+ cmpptr(thread, rax_reg);
+ jccb(Assembler::notEqual, slow_path);
+
+ // Recursive.
+ increment(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+ }
+
+ bind(locked);
+ increment(Address(thread, JavaThread::held_monitor_count_offset()));
+ // Set ZF = 1
+ xorl(rax_reg, rax_reg);
+
+#ifdef ASSERT
+ // Check that locked label is reached with ZF set.
+ Label zf_correct;
+ jccb(Assembler::zero, zf_correct);
+ stop("Fast Lock ZF != 1");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with ZF not set.
+ jccb(Assembler::notZero, zf_correct);
+ stop("Fast Lock ZF != 0");
+ bind(zf_correct);
+#endif
+ // C2 uses the value of ZF to determine the continuation.
+}
+
+void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert(reg_rax == rax, "Used for CAS");
+ assert_different_registers(obj, reg_rax, t);
+
+ // Handle inflated monitor.
+ Label inflated, inflated_check_lock_stack;
+ // Finish fast unlock successfully. MUST jump with ZF == 1
+ Label unlocked;
+
+ // Assume success.
+ decrement(Address(thread, JavaThread::held_monitor_count_offset()));
+
+ const Register mark = t;
+ const Register top = reg_rax;
+
+ Label dummy;
+ C2FastUnlockLightweightStub* stub = nullptr;
+
+ if (!Compile::current()->output()->in_scratch_emit_size()) {
+ stub = new (Compile::current()->comp_arena()) C2FastUnlockLightweightStub(obj, mark, reg_rax, thread);
+ Compile::current()->output()->add_stub(stub);
+ }
+
+ Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
+ Label& check_successor = stub == nullptr ? dummy : stub->check_successor();
+
+ { // Lightweight Unlock
+
+ // Load top.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+
+ // Prefetch mark.
+ movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check if obj is top of lock-stack.
+ cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
+ // Top of lock stack was not obj. Must be monitor.
+ jcc(Assembler::notEqual, inflated_check_lock_stack);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
+ subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+
+ // Check if recursive.
+ cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
+ jcc(Assembler::equal, unlocked);
+
+ // We elide the monitor check, let the CAS fail instead.
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ movptr(reg_rax, mark);
+ andptr(reg_rax, ~(int32_t)markWord::lock_mask);
+ orptr(mark, markWord::unlocked_value);
+ lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ jcc(Assembler::notEqual, push_and_slow_path);
+ jmp(unlocked);
+ }
+
+
+ { // Handle inflated monitor.
+ bind(inflated_check_lock_stack);
+#ifdef ASSERT
+ Label check_done;
+ subl(top, oopSize);
+ cmpl(top, in_bytes(JavaThread::lock_stack_base_offset()));
+ jcc(Assembler::below, check_done);
+ cmpptr(obj, Address(thread, top));
+ jccb(Assembler::notEqual, inflated_check_lock_stack);
+ stop("Fast Unlock lock on stack");
+ bind(check_done);
+ testptr(mark, markWord::monitor_value);
+ jccb(Assembler::notZero, inflated);
+ stop("Fast Unlock not monitor");
+#endif
+
+ bind(inflated);
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register monitor = mark;
+
+#ifndef _LP64
+ // Check if recursive.
+ xorptr(reg_rax, reg_rax);
+ orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+ jcc(Assembler::notZero, check_successor);
+
+ // Check if the entry lists are empty.
+ movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
+ orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
+ jcc(Assembler::notZero, check_successor);
+
+ // Release lock.
+ movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
+#else // _LP64
+ Label recursive;
+
+ // Check if recursive.
+ cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 0);
+ jccb(Assembler::notEqual, recursive);
+
+ // Check if the entry lists are empty.
+ movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
+ orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
+ jcc(Assembler::notZero, check_successor);
+
+ // Release lock.
+ movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
+ jmpb(unlocked);
+
+ // Recursive unlock.
+ bind(recursive);
+ decrement(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+ xorl(t, t);
+#endif
+ }
+
+ bind(unlocked);
+ if (stub != nullptr) {
+ bind(stub->unlocked_continuation());
+ }
+
+#ifdef ASSERT
+ // Check that unlocked label is reached with ZF set.
+ Label zf_correct;
+ jccb(Assembler::zero, zf_correct);
+ stop("Fast Unlock ZF != 1");
+#endif
+
+ if (stub != nullptr) {
+ bind(stub->slow_path_continuation());
+ }
+#ifdef ASSERT
+ // Check that stub->continuation() label is reached with ZF not set.
+ jccb(Assembler::notZero, zf_correct);
+ stop("Fast Unlock ZF != 0");
+ bind(zf_correct);
+#endif
+ // C2 uses the value of ZF to determine the continuation.
+}
+
//-------------------------------------------------------------------------------------------
// Generic instructions support for use in .ad files C2 code generation
@@ -5282,6 +5504,42 @@ void C2_MacroAssembler::vector_mask_compress(KRegister dst, KRegister src, Regis
kmov(dst, rtmp2);
}
+#ifdef _LP64
+void C2_MacroAssembler::vector_compress_expand_avx2(int opcode, XMMRegister dst, XMMRegister src,
+ XMMRegister mask, Register rtmp, Register rscratch,
+ XMMRegister permv, XMMRegister xtmp, BasicType bt,
+ int vec_enc) {
+ assert(type2aelembytes(bt) >= 4, "");
+ assert(opcode == Op_CompressV || opcode == Op_ExpandV, "");
+ address compress_perm_table = nullptr;
+ address expand_perm_table = nullptr;
+ if (type2aelembytes(bt) == 8) {
+ compress_perm_table = StubRoutines::x86::compress_perm_table64();
+ expand_perm_table = StubRoutines::x86::expand_perm_table64();
+ vmovmskpd(rtmp, mask, vec_enc);
+ } else {
+ compress_perm_table = StubRoutines::x86::compress_perm_table32();
+ expand_perm_table = StubRoutines::x86::expand_perm_table32();
+ vmovmskps(rtmp, mask, vec_enc);
+ }
+ shlq(rtmp, 5); // for 32 byte permute row.
+ if (opcode == Op_CompressV) {
+ lea(rscratch, ExternalAddress(compress_perm_table));
+ } else {
+ lea(rscratch, ExternalAddress(expand_perm_table));
+ }
+ addptr(rtmp, rscratch);
+ vmovdqu(permv, Address(rtmp));
+ vpermps(dst, permv, src, Assembler::AVX_256bit);
+ vpxor(xtmp, xtmp, xtmp, vec_enc);
+ // Blend the result with zero vector using permute mask, each column entry
+ // in a permute table row contains either a valid permute index or a -1 (default)
+ // value, this can potentially be used as a blending mask after
+ // compressing/expanding the source vector lanes.
+ vblendvps(dst, dst, xtmp, permv, vec_enc, false, permv);
+}
+#endif
+
void C2_MacroAssembler::vector_compress_expand(int opcode, XMMRegister dst, XMMRegister src, KRegister mask,
bool merge, BasicType bt, int vec_enc) {
if (opcode == Op_CompressV) {
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
index e9e1412957bd2..26f7fb44aa939 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -43,6 +43,10 @@
bool use_rtm, bool profile_rtm);
void fast_unlock(Register obj, Register box, Register tmp, bool use_rtm);
+ void fast_lock_lightweight(Register obj, Register box, Register rax_reg,
+ Register t, Register thread);
+ void fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread);
+
#if INCLUDE_RTM_OPT
void rtm_counters_update(Register abort_status, Register rtm_counters);
void branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel);
@@ -390,6 +394,10 @@
void vector_round_float_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4);
+
+ void vector_compress_expand_avx2(int opcode, XMMRegister dst, XMMRegister src, XMMRegister mask,
+ Register rtmp, Register rscratch, XMMRegister permv, XMMRegister xtmp,
+ BasicType bt, int vec_enc);
#endif // _LP64
void udivI(Register rax, Register divisor, Register rdx);
diff --git a/src/hotspot/cpu/x86/compiledIC_x86.cpp b/src/hotspot/cpu/x86/compiledIC_x86.cpp
index 8fc001039fbd3..95b41f62b6aab 100644
--- a/src/hotspot/cpu/x86/compiledIC_x86.cpp
+++ b/src/hotspot/cpu/x86/compiledIC_x86.cpp
@@ -26,7 +26,6 @@
#include "asm/macroAssembler.inline.hpp"
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
@@ -36,7 +35,7 @@
// ----------------------------------------------------------------------------
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
// Stub is fixed up when the corresponding call is converted from
// calling compiled code to calling interpreted code.
// movq rbx, 0
@@ -66,32 +65,25 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
}
#undef __
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return NOT_LP64(10) // movl; jmp
LP64_ONLY(15); // movq (1+1+8); jmp (1+4)
}
-int CompiledStaticCall::to_trampoline_stub_size() {
+int CompiledDirectCall::to_trampoline_stub_size() {
// x86 doesn't use trampolines.
return 0;
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 4; // 3 in emit_to_interp_stub + 1 in emit_call
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder = nativeMovConstReg_at(stub);
NativeJump* jump = nativeJump_at(method_holder->next_instruction_address());
@@ -105,7 +97,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
assert(CompiledICLocker::is_safe(static_stub->addr()), "mt unsafe call");
// Reset stub.
address stub = static_stub->addr();
@@ -122,7 +114,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
index f609846f00d6c..f9f77c23f14ca 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
@@ -28,8 +28,8 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/cpu/x86/icBuffer_x86.cpp b/src/hotspot/cpu/x86/icBuffer_x86.cpp
deleted file mode 100644
index af374b5741659..0000000000000
--- a/src/hotspot/cpu/x86/icBuffer_x86.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_x86.hpp"
-#include "oops/oop.inline.hpp"
-
-int InlineCacheBuffer::ic_stub_code_size() {
- // Worst case, if destination is not a near call:
- // lea rax, lit1
- // lea scratch, lit2
- // jmp scratch
-
- // Best case
- // lea rax, lit1
- // jmp lit2
-
- int best = NativeMovConstReg::instruction_size + NativeJump::instruction_size;
- int worst = 2 * NativeMovConstReg::instruction_size + 3;
- return MAX2(best, worst);
-}
-
-
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler* masm = new MacroAssembler(&code);
- // note: even though the code contains an embedded value, we do not need reloc info
- // because
- // (1) the value is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear
- // assert(cached_value == nullptr || cached_oop->is_perm(), "must be perm oop");
- masm->lea(rax, AddressLiteral((address) cached_value, relocInfo::metadata_type));
- masm->jump(ExternalAddress(entry_point));
-}
-
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- address jmp = move->next_instruction_address();
- NativeInstruction* ni = nativeInstruction_at(jmp);
- if (ni->is_jump()) {
- NativeJump* jump = nativeJump_at(jmp);
- return jump->jump_destination();
- } else {
- assert(ni->is_far_jump(), "unexpected instruction");
- NativeFarJump* jump = nativeFarJump_at(jmp);
- return jump->jump_destination();
- }
-}
-
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- // creation also verifies the object
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin);
- // Verifies the jump
- address jmp = move->next_instruction_address();
- NativeInstruction* ni = nativeInstruction_at(jmp);
- if (ni->is_jump()) {
- NativeJump* jump = nativeJump_at(jmp);
- } else {
- assert(ni->is_far_jump(), "unexpected instruction");
- NativeFarJump* jump = nativeFarJump_at(jmp);
- }
- void* o = (void*)move->data();
- return o;
-}
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp
index f5f83ae21f475..33570f3155b15 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1192,8 +1192,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
const Register thread = lock_reg;
get_thread(thread);
#endif
- // Load object header, prepare for CAS from unlocked to locked.
- movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
lightweight_lock(obj_reg, swap_reg, thread, tmp_reg, slow_case);
} else if (LockingMode == LM_LEGACY) {
// Load immediate 1 into swap_reg %rax
@@ -1311,20 +1309,13 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
if (LockingMode == LM_LIGHTWEIGHT) {
#ifdef _LP64
- const Register thread = r15_thread;
+ lightweight_unlock(obj_reg, swap_reg, r15_thread, header_reg, slow_case);
#else
- const Register thread = header_reg;
- get_thread(thread);
+ // This relies on the implementation of lightweight_unlock being able to handle
+ // that the reg_rax and thread Register parameters may alias each other.
+ get_thread(swap_reg);
+ lightweight_unlock(obj_reg, swap_reg, swap_reg, header_reg, slow_case);
#endif
- // Handle unstructured locking.
- Register tmp = swap_reg;
- movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
- cmpptr(obj_reg, Address(thread, tmp, Address::times_1, -oopSize));
- jcc(Assembler::notEqual, slow_case);
- // Try to swing header from locked to unlocked.
- movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
- lightweight_unlock(obj_reg, swap_reg, header_reg, slow_case);
} else if (LockingMode == LM_LEGACY) {
// Load the old header from BasicLock structure
movptr(header_reg, Address(swap_reg,
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 882966564859e..f0e7a08dd5f2a 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/compiler_globals.hpp"
#include "compiler/disassembler.hpp"
#include "crc32c.h"
@@ -1341,13 +1342,45 @@ void MacroAssembler::ic_call(address entry, jint method_index) {
RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
#ifdef _LP64
// Needs full 64-bit immediate for later patching.
- mov64(rax, (intptr_t)Universe::non_oop_word());
+ mov64(rax, (int64_t)Universe::non_oop_word());
#else
movptr(rax, (intptr_t)Universe::non_oop_word());
#endif
call(AddressLiteral(entry, rh));
}
+int MacroAssembler::ic_check_size() {
+ return LP64_ONLY(14) NOT_LP64(12);
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
+ Register data = rax;
+ Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx);
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, offset() + ic_check_size());
+
+ int uep_offset = offset();
+
+ if (UseCompressedClassPointers) {
+ movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
+ } else {
+ movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
+ }
+
+ // if inline cache check fails, then jump to runtime routine
+ jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
+
+ return uep_offset;
+}
+
void MacroAssembler::emit_static_call_stub() {
// Static stub relocation also tags the Method* in the code-stream.
mov_metadata(rbx, (Metadata*) nullptr); // Method is zapped till fixup time.
@@ -2568,7 +2601,9 @@ void MacroAssembler::movptr(Register dst, Address src) {
// src should NEVER be a real pointer. Use AddressLiteral for true pointers
void MacroAssembler::movptr(Register dst, intptr_t src) {
#ifdef _LP64
- if (is_simm32(src)) {
+ if (is_uimm32(src)) {
+ movl(dst, checked_cast(src));
+ } else if (is_simm32(src)) {
movq(dst, checked_cast(src));
} else {
mov64(dst, src);
@@ -4085,8 +4120,9 @@ static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister r
}
}
-int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, bool save_fpu,
- int& gp_area_size, int& fp_area_size, int& xmm_area_size) {
+static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
+ bool save_fpu, int& gp_area_size,
+ int& fp_area_size, int& xmm_area_size) {
gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
StackAlignmentInBytes);
@@ -4352,7 +4388,7 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
}
// Look up the method for a megamorphic invokeinterface call in a single pass over itable:
-// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
// - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
// The target method is determined by .
// The receiver klass is in recv_klass.
@@ -9875,68 +9911,116 @@ void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigne
}
// Implements lightweight-locking.
-// Branches to slow upon failure to lock the object, with ZF cleared.
-// Falls through upon success with unspecified ZF.
//
// obj: the object to be locked
-// hdr: the (pre-loaded) header of the object, must be rax
+// reg_rax: rax
// thread: the thread which attempts to lock obj
// tmp: a temporary register
-void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register thread, Register tmp, Label& slow) {
- assert(hdr == rax, "header must be in rax for cmpxchg");
- assert_different_registers(obj, hdr, thread, tmp);
-
- // First we need to check if the lock-stack has room for pushing the object reference.
- // Note: we subtract 1 from the end-offset so that we can do a 'greater' comparison, instead
- // of 'greaterEqual' below, which readily clears the ZF. This makes C2 code a little simpler and
- // avoids one branch.
- cmpl(Address(thread, JavaThread::lock_stack_top_offset()), LockStack::end_offset() - 1);
- jcc(Assembler::greater, slow);
-
- // Now we attempt to take the fast-lock.
- // Clear lock_mask bits (locked state).
- andptr(hdr, ~(int32_t)markWord::lock_mask_in_place);
- movptr(tmp, hdr);
- // Set unlocked_value bit.
- orptr(hdr, markWord::unlocked_value);
- lock();
- cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
+void MacroAssembler::lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
+ assert(reg_rax == rax, "");
+ assert_different_registers(obj, reg_rax, thread, tmp);
+
+ Label push;
+ const Register top = tmp;
+
+ // Preload the markWord. It is important that this is the first
+ // instruction emitted as it is part of C1's null check semantics.
+ movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Load top.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+
+ // Check if the lock-stack is full.
+ cmpl(top, LockStack::end_offset());
+ jcc(Assembler::greaterEqual, slow);
+
+ // Check for recursion.
+ cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
+ jcc(Assembler::equal, push);
+
+ // Check header for monitor (0b10).
+ testptr(reg_rax, markWord::monitor_value);
+ jcc(Assembler::notZero, slow);
+
+ // Try to lock. Transition lock bits 0b01 => 0b00
+ movptr(tmp, reg_rax);
+ andptr(tmp, ~(int32_t)markWord::unlocked_value);
+ orptr(reg_rax, markWord::unlocked_value);
+ lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::notEqual, slow);
- // If successful, push object to lock-stack.
- movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
- movptr(Address(thread, tmp), obj);
- incrementl(tmp, oopSize);
- movl(Address(thread, JavaThread::lock_stack_top_offset()), tmp);
+ // Restore top, CAS clobbers register.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ movptr(Address(thread, top), obj);
+ incrementl(top, oopSize);
+ movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
}
// Implements lightweight-unlocking.
-// Branches to slow upon failure, with ZF cleared.
-// Falls through upon success, with unspecified ZF.
//
// obj: the object to be unlocked
-// hdr: the (pre-loaded) header of the object, must be rax
+// reg_rax: rax
+// thread: the thread
// tmp: a temporary register
-void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow) {
- assert(hdr == rax, "header must be in rax for cmpxchg");
- assert_different_registers(obj, hdr, tmp);
-
- // Mark-word must be lock_mask now, try to swing it back to unlocked_value.
- movptr(tmp, hdr); // The expected old value
- orptr(tmp, markWord::unlocked_value);
- lock();
- cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
+//
+// x86_32 Note: reg_rax and thread may alias each other due to limited register
+// availiability.
+void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
+ assert(reg_rax == rax, "");
+ assert_different_registers(obj, reg_rax, tmp);
+ LP64_ONLY(assert_different_registers(obj, reg_rax, thread, tmp);)
+
+ Label unlocked, push_and_slow;
+ const Register top = tmp;
+
+ // Check if obj is top of lock-stack.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+ cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
jcc(Assembler::notEqual, slow);
- // Pop the lock object from the lock-stack.
-#ifdef _LP64
- const Register thread = r15_thread;
-#else
- const Register thread = rax;
- get_thread(thread);
-#endif
+
+ // Pop lock-stack.
+ DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+
+ // Check if recursive.
+ cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
+ jcc(Assembler::equal, unlocked);
+
+ // Not recursive. Check header for monitor (0b10).
+ movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
+ testptr(reg_rax, markWord::monitor_value);
+ jcc(Assembler::notZero, push_and_slow);
+
+#ifdef ASSERT
+ // Check header not unlocked (0b01).
+ Label not_unlocked;
+ testptr(reg_rax, markWord::unlocked_value);
+ jcc(Assembler::zero, not_unlocked);
+ stop("lightweight_unlock already unlocked");
+ bind(not_unlocked);
+#endif
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ movptr(tmp, reg_rax);
+ orptr(tmp, markWord::unlocked_value);
+ lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
+ jcc(Assembler::equal, unlocked);
+
+ bind(push_and_slow);
+ // Restore lock-stack and handle the unlock in runtime.
+ if (thread == reg_rax) {
+ // On x86_32 we may lose the thread.
+ get_thread(thread);
+ }
#ifdef ASSERT
- movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
- movptr(Address(thread, tmp), 0);
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+ movptr(Address(thread, top), obj);
#endif
+ addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+ jmp(slow);
+
+ bind(unlocked);
}
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 4b30168452796..4789b63decc6c 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -896,6 +896,8 @@ class MacroAssembler: public Assembler {
// Emit the CompiledIC call idiom
void ic_call(address entry, jint method_index = 0);
+ static int ic_check_size();
+ int ic_check(int end_alignment);
void emit_static_call_stub();
@@ -2031,8 +2033,8 @@ class MacroAssembler: public Assembler {
void check_stack_alignment(Register sp, const char* msg, unsigned bias = 0, Register tmp = noreg);
- void lightweight_lock(Register obj, Register hdr, Register thread, Register tmp, Label& slow);
- void lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow);
+ void lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow);
+ void lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow);
};
/**
diff --git a/src/hotspot/cpu/x86/peephole_x86_64.cpp b/src/hotspot/cpu/x86/peephole_x86_64.cpp
index 8c956aeb05393..92a29490edaf8 100644
--- a/src/hotspot/cpu/x86/peephole_x86_64.cpp
+++ b/src/hotspot/cpu/x86/peephole_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,8 +33,8 @@
// lea d, [s1 + s2] and
// mov d, s1; shl d, s2 into
// lea d, [s1 << s2] with s2 = 1, 2, 3
-bool lea_coalesce_helper(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_,
- MachNode* (*new_root)(), uint inst0_rule, bool imm) {
+static bool lea_coalesce_helper(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_,
+ MachNode* (*new_root)(), uint inst0_rule, bool imm) {
MachNode* inst0 = block->get_node(block_index)->as_Mach();
assert(inst0->rule() == inst0_rule, "sanity");
@@ -136,7 +136,7 @@ bool lea_coalesce_helper(Block* block, int block_index, PhaseCFG* cfg_, PhaseReg
// This helper func takes a condition and returns the flags that need to be set for the condition
// It uses the same flags as the test instruction, so if the e.g. the overflow bit is required,
// this func returns clears_overflow, as that is what the test instruction does and what the downstream path expects
-juint map_condition_to_required_test_flags(Assembler::Condition condition) {
+static juint map_condition_to_required_test_flags(Assembler::Condition condition) {
switch (condition) {
case Assembler::Condition::zero: // Same value as equal
case Assembler::Condition::notZero: // Same value as notEqual
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86.cpp
index de759d2ff5761..3ecbb43f7f518 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86.cpp
@@ -26,6 +26,7 @@
#include "asm/macroAssembler.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/sharedRuntime.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "vmreg_x86.inline.hpp"
#ifdef COMPILER1
#include "c1/c1_Runtime1.hpp"
@@ -59,9 +60,16 @@ void SharedRuntime::inline_check_hashcode_from_object_header(MacroAssembler* mas
__ movptr(result, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- // check if locked
- __ testptr(result, markWord::unlocked_value);
- __ jcc(Assembler::zero, slowCase);
+
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ // check if monitor
+ __ testptr(result, markWord::monitor_value);
+ __ jcc(Assembler::notZero, slowCase);
+ } else {
+ // check if locked
+ __ testptr(result, markWord::unlocked_value);
+ __ jcc(Assembler::zero, slowCase);
+ }
// get hash
#ifdef _LP64
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
index 571160523cbe4..febc1b2c3b143 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,8 +25,8 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
@@ -36,7 +36,6 @@
#include "interpreter/interpreter.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/jniHandles.hpp"
@@ -944,25 +943,18 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
address c2i_unverified_entry = __ pc();
Label skip_fixup;
- Register holder = rax;
+ Register data = rax;
Register receiver = rcx;
Register temp = rbx;
{
-
- Label missed;
- __ movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
- __ cmpptr(temp, Address(holder, CompiledICHolder::holder_klass_offset()));
- __ movptr(rbx, Address(holder, CompiledICHolder::holder_metadata_offset()));
- __ jcc(Assembler::notEqual, missed);
+ __ ic_check(1 /* end_alignment */);
+ __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
// Method might have been compiled since the call site was patched to
// interpreted if that is the case treat it as a miss so we can get
// the call site corrected.
__ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
__ jcc(Assembler::equal, skip_fixup);
-
- __ bind(missed);
- __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
}
address c2i_entry = __ pc();
@@ -1449,23 +1441,12 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// as far as the interpreter and the compiler(s) are concerned.
- const Register ic_reg = rax;
const Register receiver = rcx;
- Label hit;
Label exception_pending;
__ verify_oop(receiver);
- __ cmpptr(ic_reg, Address(receiver, oopDesc::klass_offset_in_bytes()));
- __ jcc(Assembler::equal, hit);
-
- __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
// verified entry must be aligned for code patching.
- // and the first 5 bytes must be in the same cache line
- // if we align at 8 then we will be sure 5 bytes are in the same line
- __ align(8);
-
- __ bind(hit);
+ __ ic_check(8 /* end_alignment */);
int vep_offset = ((intptr_t)__ pc()) - start;
@@ -1713,8 +1694,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ jcc(Assembler::notEqual, slow_path_lock);
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- // Load object header
- __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ lightweight_lock(obj_reg, swap_reg, thread, lock_reg, slow_path_lock);
}
__ bind(count_mon);
@@ -1872,9 +1851,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ dec_held_monitor_count();
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- __ andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
- __ lightweight_unlock(obj_reg, swap_reg, lock_reg, slow_path_unlock);
+ __ lightweight_unlock(obj_reg, swap_reg, thread, lock_reg, slow_path_unlock);
__ dec_held_monitor_count();
}
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
index faa423bcf8e76..c666f982d0f52 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,6 @@
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
@@ -42,7 +41,6 @@
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "oops/method.inline.hpp"
#include "prims/methodHandles.hpp"
@@ -1000,20 +998,14 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
address c2i_unverified_entry = __ pc();
Label skip_fixup;
- Label ok;
- Register holder = rax;
+ Register data = rax;
Register receiver = j_rarg0;
Register temp = rbx;
{
- __ load_klass(temp, receiver, rscratch1);
- __ cmpptr(temp, Address(holder, CompiledICHolder::holder_klass_offset()));
- __ movptr(rbx, Address(holder, CompiledICHolder::holder_metadata_offset()));
- __ jcc(Assembler::equal, ok);
- __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- __ bind(ok);
+ __ ic_check(1 /* end_alignment */);
+ __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
// Method might have been compiled since the call site was patched to
// interpreted if that is the case treat it as a miss so we can get
// the call site corrected.
@@ -1450,7 +1442,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ align(BytesPerWord, __ offset() + NativeCall::displacement_offset);
// Emit stub for static call
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, __ pc());
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, __ pc());
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1487,7 +1479,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
// Emit stub for static call
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, __ pc());
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, __ pc());
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1744,6 +1736,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
in_ByteSize(-1),
oop_maps,
exception_offset);
+ if (nm == nullptr) return nm;
if (method->is_continuation_enter_intrinsic()) {
ContinuationEntry::set_enter_code(nm, interpreted_entry_offset);
} else if (method->is_continuation_yield_intrinsic()) {
@@ -1882,25 +1875,13 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// restoring them except rbp. rbp is the only callee save register
// as far as the interpreter and the compiler(s) are concerned.
-
- const Register ic_reg = rax;
const Register receiver = j_rarg0;
- Label hit;
Label exception_pending;
- assert_different_registers(ic_reg, receiver, rscratch1, rscratch2);
+ assert_different_registers(receiver, rscratch1, rscratch2);
__ verify_oop(receiver);
- __ load_klass(rscratch1, receiver, rscratch2);
- __ cmpq(ic_reg, rscratch1);
- __ jcc(Assembler::equal, hit);
-
- __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- // Verified entry point must be aligned
- __ align(8);
-
- __ bind(hit);
+ __ ic_check(8 /* end_alignment */);
int vep_offset = ((intptr_t)__ pc()) - start;
@@ -2189,8 +2170,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ jcc(Assembler::notEqual, slow_path_lock);
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- // Load object header
- __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ lightweight_lock(obj_reg, swap_reg, r15_thread, rscratch1, slow_path_lock);
}
__ bind(count_mon);
@@ -2333,9 +2312,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ dec_held_monitor_count();
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- __ andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
- __ lightweight_unlock(obj_reg, swap_reg, lock_reg, slow_path_unlock);
+ __ lightweight_unlock(obj_reg, swap_reg, r15_thread, lock_reg, slow_path_unlock);
__ dec_held_monitor_count();
}
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
index cad9e6475c610..71aafdc1cd3e2 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
@@ -951,6 +951,92 @@ address StubGenerator::generate_fp_mask(const char *stub_name, int64_t mask) {
return start;
}
+address StubGenerator::generate_compress_perm_table(const char *stub_name, int32_t esize) {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", stub_name);
+ address start = __ pc();
+ if (esize == 32) {
+ // Loop to generate 256 x 8 int compression permute index table. A row is
+ // accessed using 8 bit index computed using vector mask. An entry in
+ // a row holds either a valid permute index corresponding to set bit position
+ // or a -1 (default) value.
+ for (int mask = 0; mask < 256; mask++) {
+ int ctr = 0;
+ for (int j = 0; j < 8; j++) {
+ if (mask & (1 << j)) {
+ __ emit_data(j, relocInfo::none);
+ ctr++;
+ }
+ }
+ for (; ctr < 8; ctr++) {
+ __ emit_data(-1, relocInfo::none);
+ }
+ }
+ } else {
+ assert(esize == 64, "");
+ // Loop to generate 16 x 4 long compression permute index table. A row is
+ // accessed using 4 bit index computed using vector mask. An entry in
+ // a row holds either a valid permute index pair for a quadword corresponding
+ // to set bit position or a -1 (default) value.
+ for (int mask = 0; mask < 16; mask++) {
+ int ctr = 0;
+ for (int j = 0; j < 4; j++) {
+ if (mask & (1 << j)) {
+ __ emit_data(2 * j, relocInfo::none);
+ __ emit_data(2 * j + 1, relocInfo::none);
+ ctr++;
+ }
+ }
+ for (; ctr < 4; ctr++) {
+ __ emit_data64(-1L, relocInfo::none);
+ }
+ }
+ }
+ return start;
+}
+
+address StubGenerator::generate_expand_perm_table(const char *stub_name, int32_t esize) {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", stub_name);
+ address start = __ pc();
+ if (esize == 32) {
+ // Loop to generate 256 x 8 int expand permute index table. A row is accessed
+ // using 8 bit index computed using vector mask. An entry in a row holds either
+ // a valid permute index (starting from least significant lane) placed at poisition
+ // corresponding to set bit position or a -1 (default) value.
+ for (int mask = 0; mask < 256; mask++) {
+ int ctr = 0;
+ for (int j = 0; j < 8; j++) {
+ if (mask & (1 << j)) {
+ __ emit_data(ctr++, relocInfo::none);
+ } else {
+ __ emit_data(-1, relocInfo::none);
+ }
+ }
+ }
+ } else {
+ assert(esize == 64, "");
+ // Loop to generate 16 x 4 long expand permute index table. A row is accessed
+ // using 4 bit index computed using vector mask. An entry in a row holds either
+ // a valid doubleword permute index pair representing a quadword index (starting
+ // from least significant lane) placed at poisition corresponding to set bit
+ // position or a -1 (default) value.
+ for (int mask = 0; mask < 16; mask++) {
+ int ctr = 0;
+ for (int j = 0; j < 4; j++) {
+ if (mask & (1 << j)) {
+ __ emit_data(2 * ctr, relocInfo::none);
+ __ emit_data(2 * ctr + 1, relocInfo::none);
+ ctr++;
+ } else {
+ __ emit_data64(-1L, relocInfo::none);
+ }
+ }
+ }
+ }
+ return start;
+}
+
address StubGenerator::generate_vector_mask(const char *stub_name, int64_t mask) {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", stub_name);
@@ -4095,6 +4181,13 @@ void StubGenerator::generate_compiler_stubs() {
StubRoutines::x86::_vector_reverse_byte_perm_mask_int = generate_vector_reverse_byte_perm_mask_int("perm_mask_int");
StubRoutines::x86::_vector_reverse_byte_perm_mask_short = generate_vector_reverse_byte_perm_mask_short("perm_mask_short");
+ if (VM_Version::supports_avx2() && !VM_Version::supports_avx512vl()) {
+ StubRoutines::x86::_compress_perm_table32 = generate_compress_perm_table("compress_perm_table32", 32);
+ StubRoutines::x86::_compress_perm_table64 = generate_compress_perm_table("compress_perm_table64", 64);
+ StubRoutines::x86::_expand_perm_table32 = generate_expand_perm_table("expand_perm_table32", 32);
+ StubRoutines::x86::_expand_perm_table64 = generate_expand_perm_table("expand_perm_table64", 64);
+ }
+
if (VM_Version::supports_avx2() && !VM_Version::supports_avx512_vpopcntdq()) {
// lut implementation influenced by counting 1s algorithm from section 5-1 of Hackers' Delight.
StubRoutines::x86::_vector_popcount_lut = generate_popcount_avx_lut("popcount_lut");
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
index 6b7da7184988e..db43085d37f3a 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -99,6 +99,10 @@ class StubGenerator: public StubCodeGenerator {
address generate_fp_mask(const char *stub_name, int64_t mask);
+ address generate_compress_perm_table(const char *stub_name, int32_t esize);
+
+ address generate_expand_perm_table(const char *stub_name, int32_t esize);
+
address generate_vector_mask(const char *stub_name, int64_t mask);
address generate_vector_byte_perm_mask(const char *stub_name);
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
index cebf661ae75ed..bc1cbdbba26b5 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -82,6 +82,10 @@ address StubRoutines::x86::_join_0_1_base64 = nullptr;
address StubRoutines::x86::_join_1_2_base64 = nullptr;
address StubRoutines::x86::_join_2_3_base64 = nullptr;
address StubRoutines::x86::_decoding_table_base64 = nullptr;
+address StubRoutines::x86::_compress_perm_table32 = nullptr;
+address StubRoutines::x86::_compress_perm_table64 = nullptr;
+address StubRoutines::x86::_expand_perm_table32 = nullptr;
+address StubRoutines::x86::_expand_perm_table64 = nullptr;
#endif
address StubRoutines::x86::_pshuffle_byte_flip_mask_addr = nullptr;
@@ -275,7 +279,7 @@ uint32_t _crc32c_pow_2k_table[TILL_CYCLE]; // because _crc32c_pow_2k_table[TILL_
// A. Kadatch and B. Jenkins / Everything we know about CRC but afraid to forget September 3, 2010 8
// Listing 1: Multiplication of normalized polynomials
// "a" and "b" occupy D least significant bits.
-uint32_t crc32c_multiply(uint32_t a, uint32_t b) {
+static uint32_t crc32c_multiply(uint32_t a, uint32_t b) {
uint32_t product = 0;
uint32_t b_pow_x_table[D + 1]; // b_pow_x_table[k] = (b * x**k) mod P
b_pow_x_table[0] = b;
@@ -299,7 +303,7 @@ uint32_t crc32c_multiply(uint32_t a, uint32_t b) {
#undef P
// A. Kadatch and B. Jenkins / Everything we know about CRC but afraid to forget September 3, 2010 9
-void crc32c_init_pow_2k(void) {
+static void crc32c_init_pow_2k(void) {
// _crc32c_pow_2k_table(0) =
// x^(2^k) mod P(x) = x mod P(x) = x
// Since we are operating on a reflected values
@@ -314,7 +318,7 @@ void crc32c_init_pow_2k(void) {
}
// x^N mod P(x)
-uint32_t crc32c_f_pow_n(uint32_t n) {
+static uint32_t crc32c_f_pow_n(uint32_t n) {
// result = 1 (polynomial)
uint32_t one, result = 0x80000000, i = 0;
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
index 6c602324f3ef2..cfb91c5c08368 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,7 +37,7 @@ enum platform_dependent_constants {
_continuation_stubs_code_size = 1000 LP64_ONLY(+1000),
// AVX512 intrinsics add more code in 64-bit VM,
// Windows have more code to save/restore registers
- _compiler_stubs_code_size = 20000 LP64_ONLY(+32000) WINDOWS_ONLY(+2000),
+ _compiler_stubs_code_size = 20000 LP64_ONLY(+39000) WINDOWS_ONLY(+2000),
_final_stubs_code_size = 10000 LP64_ONLY(+20000) WINDOWS_ONLY(+2000) ZGC_ONLY(+20000)
};
@@ -58,6 +58,10 @@ class x86 {
static address _float_sign_flip;
static address _double_sign_mask;
static address _double_sign_flip;
+ static address _compress_perm_table32;
+ static address _compress_perm_table64;
+ static address _expand_perm_table32;
+ static address _expand_perm_table64;
public:
@@ -338,6 +342,10 @@ class x86 {
static address base64_decoding_table_addr() { return _decoding_table_base64; }
static address base64_AVX2_decode_tables_addr() { return _avx2_decode_tables_base64; }
static address base64_AVX2_decode_LUT_tables_addr() { return _avx2_decode_lut_tables_base64; }
+ static address compress_perm_table32() { return _compress_perm_table32; }
+ static address compress_perm_table64() { return _compress_perm_table64; }
+ static address expand_perm_table32() { return _expand_perm_table32; }
+ static address expand_perm_table64() { return _expand_perm_table64; }
#endif
static address pshuffle_byte_flip_mask_addr() { return _pshuffle_byte_flip_mask_addr; }
static address arrays_hashcode_powers_of_31() { return (address)_arrays_hashcode_powers_of_31; }
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86_64.cpp b/src/hotspot/cpu/x86/stubRoutines_x86_64.cpp
index 417b32eb4a64c..eb6c11d7167da 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86_64.cpp
@@ -44,4 +44,3 @@ address StubRoutines::x86::_float_sign_mask = nullptr;
address StubRoutines::x86::_float_sign_flip = nullptr;
address StubRoutines::x86::_double_sign_mask = nullptr;
address StubRoutines::x86::_double_sign_flip = nullptr;
-
diff --git a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
index 89e7a466264b9..7b9d49dd46140 100644
--- a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
+++ b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
@@ -300,8 +300,12 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ mov_metadata(rbx, entry);
__ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); // just in case callee is deoptimized
+ __ push_cont_fastpath();
+
__ call(Address(rbx, Method::from_compiled_offset()));
+ __ pop_cont_fastpath();
+
// return value shuffle
if (!needs_return_buffer) {
#ifdef ASSERT
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index df1ea6edd300f..f8213a2539f04 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -809,7 +809,8 @@ void VM_Version::get_processor_features() {
_stepping = cpu_stepping();
if (cpu_family() > 4) { // it supports CPUID
- _features = feature_flags();
+ _features = _cpuid_info.feature_flags(); // These can be changed by VM settings
+ _cpu_features = _features; // Preserve features
// Logical processors are only available on P4s and above,
// and only if hyperthreading is available.
_logical_processors_per_package = logical_processor_count();
@@ -2890,13 +2891,13 @@ int64_t VM_Version::maximum_qualified_cpu_frequency(void) {
return _max_qualified_cpu_frequency;
}
-uint64_t VM_Version::feature_flags() {
+uint64_t VM_Version::CpuidInfo::feature_flags() const {
uint64_t result = 0;
- if (_cpuid_info.std_cpuid1_edx.bits.cmpxchg8 != 0)
+ if (std_cpuid1_edx.bits.cmpxchg8 != 0)
result |= CPU_CX8;
- if (_cpuid_info.std_cpuid1_edx.bits.cmov != 0)
+ if (std_cpuid1_edx.bits.cmov != 0)
result |= CPU_CMOV;
- if (_cpuid_info.std_cpuid1_edx.bits.clflush != 0)
+ if (std_cpuid1_edx.bits.clflush != 0)
result |= CPU_FLUSH;
#ifdef _LP64
// clflush should always be available on x86_64
@@ -2904,158 +2905,158 @@ uint64_t VM_Version::feature_flags() {
// to flush the code cache.
assert ((result & CPU_FLUSH) != 0, "clflush should be available");
#endif
- if (_cpuid_info.std_cpuid1_edx.bits.fxsr != 0 || (is_amd_family() &&
- _cpuid_info.ext_cpuid1_edx.bits.fxsr != 0))
+ if (std_cpuid1_edx.bits.fxsr != 0 || (is_amd_family() &&
+ ext_cpuid1_edx.bits.fxsr != 0))
result |= CPU_FXSR;
// HT flag is set for multi-core processors also.
if (threads_per_core() > 1)
result |= CPU_HT;
- if (_cpuid_info.std_cpuid1_edx.bits.mmx != 0 || (is_amd_family() &&
- _cpuid_info.ext_cpuid1_edx.bits.mmx != 0))
+ if (std_cpuid1_edx.bits.mmx != 0 || (is_amd_family() &&
+ ext_cpuid1_edx.bits.mmx != 0))
result |= CPU_MMX;
- if (_cpuid_info.std_cpuid1_edx.bits.sse != 0)
+ if (std_cpuid1_edx.bits.sse != 0)
result |= CPU_SSE;
- if (_cpuid_info.std_cpuid1_edx.bits.sse2 != 0)
+ if (std_cpuid1_edx.bits.sse2 != 0)
result |= CPU_SSE2;
- if (_cpuid_info.std_cpuid1_ecx.bits.sse3 != 0)
+ if (std_cpuid1_ecx.bits.sse3 != 0)
result |= CPU_SSE3;
- if (_cpuid_info.std_cpuid1_ecx.bits.ssse3 != 0)
+ if (std_cpuid1_ecx.bits.ssse3 != 0)
result |= CPU_SSSE3;
- if (_cpuid_info.std_cpuid1_ecx.bits.sse4_1 != 0)
+ if (std_cpuid1_ecx.bits.sse4_1 != 0)
result |= CPU_SSE4_1;
- if (_cpuid_info.std_cpuid1_ecx.bits.sse4_2 != 0)
+ if (std_cpuid1_ecx.bits.sse4_2 != 0)
result |= CPU_SSE4_2;
- if (_cpuid_info.std_cpuid1_ecx.bits.popcnt != 0)
+ if (std_cpuid1_ecx.bits.popcnt != 0)
result |= CPU_POPCNT;
- if (_cpuid_info.std_cpuid1_ecx.bits.avx != 0 &&
- _cpuid_info.std_cpuid1_ecx.bits.osxsave != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.sse != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.ymm != 0) {
+ if (std_cpuid1_ecx.bits.avx != 0 &&
+ std_cpuid1_ecx.bits.osxsave != 0 &&
+ xem_xcr0_eax.bits.sse != 0 &&
+ xem_xcr0_eax.bits.ymm != 0) {
result |= CPU_AVX;
result |= CPU_VZEROUPPER;
- if (_cpuid_info.std_cpuid1_ecx.bits.f16c != 0)
+ if (std_cpuid1_ecx.bits.f16c != 0)
result |= CPU_F16C;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx2 != 0)
+ if (sef_cpuid7_ebx.bits.avx2 != 0)
result |= CPU_AVX2;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512f != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.opmask != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.zmm512 != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.zmm32 != 0) {
+ if (sef_cpuid7_ebx.bits.avx512f != 0 &&
+ xem_xcr0_eax.bits.opmask != 0 &&
+ xem_xcr0_eax.bits.zmm512 != 0 &&
+ xem_xcr0_eax.bits.zmm32 != 0) {
result |= CPU_AVX512F;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512cd != 0)
+ if (sef_cpuid7_ebx.bits.avx512cd != 0)
result |= CPU_AVX512CD;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512dq != 0)
+ if (sef_cpuid7_ebx.bits.avx512dq != 0)
result |= CPU_AVX512DQ;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512ifma != 0)
+ if (sef_cpuid7_ebx.bits.avx512ifma != 0)
result |= CPU_AVX512_IFMA;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512pf != 0)
+ if (sef_cpuid7_ebx.bits.avx512pf != 0)
result |= CPU_AVX512PF;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512er != 0)
+ if (sef_cpuid7_ebx.bits.avx512er != 0)
result |= CPU_AVX512ER;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512bw != 0)
+ if (sef_cpuid7_ebx.bits.avx512bw != 0)
result |= CPU_AVX512BW;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512vl != 0)
+ if (sef_cpuid7_ebx.bits.avx512vl != 0)
result |= CPU_AVX512VL;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpopcntdq != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vpopcntdq != 0)
result |= CPU_AVX512_VPOPCNTDQ;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpclmulqdq != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vpclmulqdq != 0)
result |= CPU_AVX512_VPCLMULQDQ;
- if (_cpuid_info.sef_cpuid7_ecx.bits.vaes != 0)
+ if (sef_cpuid7_ecx.bits.vaes != 0)
result |= CPU_AVX512_VAES;
- if (_cpuid_info.sef_cpuid7_ecx.bits.gfni != 0)
+ if (sef_cpuid7_ecx.bits.gfni != 0)
result |= CPU_GFNI;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vnni != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vnni != 0)
result |= CPU_AVX512_VNNI;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_bitalg != 0)
+ if (sef_cpuid7_ecx.bits.avx512_bitalg != 0)
result |= CPU_AVX512_BITALG;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vbmi != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vbmi != 0)
result |= CPU_AVX512_VBMI;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vbmi2 != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vbmi2 != 0)
result |= CPU_AVX512_VBMI2;
}
}
- if (_cpuid_info.std_cpuid1_ecx.bits.hv != 0)
+ if (std_cpuid1_ecx.bits.hv != 0)
result |= CPU_HV;
- if (_cpuid_info.sef_cpuid7_ebx.bits.bmi1 != 0)
+ if (sef_cpuid7_ebx.bits.bmi1 != 0)
result |= CPU_BMI1;
- if (_cpuid_info.std_cpuid1_edx.bits.tsc != 0)
+ if (std_cpuid1_edx.bits.tsc != 0)
result |= CPU_TSC;
- if (_cpuid_info.ext_cpuid7_edx.bits.tsc_invariance != 0)
+ if (ext_cpuid7_edx.bits.tsc_invariance != 0)
result |= CPU_TSCINV_BIT;
- if (_cpuid_info.std_cpuid1_ecx.bits.aes != 0)
+ if (std_cpuid1_ecx.bits.aes != 0)
result |= CPU_AES;
- if (_cpuid_info.sef_cpuid7_ebx.bits.erms != 0)
+ if (sef_cpuid7_ebx.bits.erms != 0)
result |= CPU_ERMS;
- if (_cpuid_info.sef_cpuid7_edx.bits.fast_short_rep_mov != 0)
+ if (sef_cpuid7_edx.bits.fast_short_rep_mov != 0)
result |= CPU_FSRM;
- if (_cpuid_info.std_cpuid1_ecx.bits.clmul != 0)
+ if (std_cpuid1_ecx.bits.clmul != 0)
result |= CPU_CLMUL;
- if (_cpuid_info.sef_cpuid7_ebx.bits.rtm != 0)
+ if (sef_cpuid7_ebx.bits.rtm != 0)
result |= CPU_RTM;
- if (_cpuid_info.sef_cpuid7_ebx.bits.adx != 0)
+ if (sef_cpuid7_ebx.bits.adx != 0)
result |= CPU_ADX;
- if (_cpuid_info.sef_cpuid7_ebx.bits.bmi2 != 0)
+ if (sef_cpuid7_ebx.bits.bmi2 != 0)
result |= CPU_BMI2;
- if (_cpuid_info.sef_cpuid7_ebx.bits.sha != 0)
+ if (sef_cpuid7_ebx.bits.sha != 0)
result |= CPU_SHA;
- if (_cpuid_info.std_cpuid1_ecx.bits.fma != 0)
+ if (std_cpuid1_ecx.bits.fma != 0)
result |= CPU_FMA;
- if (_cpuid_info.sef_cpuid7_ebx.bits.clflushopt != 0)
+ if (sef_cpuid7_ebx.bits.clflushopt != 0)
result |= CPU_FLUSHOPT;
- if (_cpuid_info.ext_cpuid1_edx.bits.rdtscp != 0)
+ if (ext_cpuid1_edx.bits.rdtscp != 0)
result |= CPU_RDTSCP;
- if (_cpuid_info.sef_cpuid7_ecx.bits.rdpid != 0)
+ if (sef_cpuid7_ecx.bits.rdpid != 0)
result |= CPU_RDPID;
// AMD|Hygon features.
if (is_amd_family()) {
- if ((_cpuid_info.ext_cpuid1_edx.bits.tdnow != 0) ||
- (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0))
+ if ((ext_cpuid1_edx.bits.tdnow != 0) ||
+ (ext_cpuid1_ecx.bits.prefetchw != 0))
result |= CPU_3DNOW_PREFETCH;
- if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0)
+ if (ext_cpuid1_ecx.bits.lzcnt != 0)
result |= CPU_LZCNT;
- if (_cpuid_info.ext_cpuid1_ecx.bits.sse4a != 0)
+ if (ext_cpuid1_ecx.bits.sse4a != 0)
result |= CPU_SSE4A;
}
// Intel features.
if (is_intel()) {
- if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0) {
+ if (ext_cpuid1_ecx.bits.lzcnt != 0) {
result |= CPU_LZCNT;
}
- if (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0) {
+ if (ext_cpuid1_ecx.bits.prefetchw != 0) {
result |= CPU_3DNOW_PREFETCH;
}
- if (_cpuid_info.sef_cpuid7_ebx.bits.clwb != 0) {
+ if (sef_cpuid7_ebx.bits.clwb != 0) {
result |= CPU_CLWB;
}
- if (_cpuid_info.sef_cpuid7_edx.bits.serialize != 0)
+ if (sef_cpuid7_edx.bits.serialize != 0)
result |= CPU_SERIALIZE;
}
// ZX features.
if (is_zx()) {
- if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0) {
+ if (ext_cpuid1_ecx.bits.lzcnt != 0) {
result |= CPU_LZCNT;
}
- if (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0) {
+ if (ext_cpuid1_ecx.bits.prefetchw != 0) {
result |= CPU_3DNOW_PREFETCH;
}
}
// Protection key features.
- if (_cpuid_info.sef_cpuid7_ecx.bits.pku != 0) {
+ if (sef_cpuid7_ecx.bits.pku != 0) {
result |= CPU_PKU;
}
- if (_cpuid_info.sef_cpuid7_ecx.bits.ospke != 0) {
+ if (sef_cpuid7_ecx.bits.ospke != 0) {
result |= CPU_OSPKE;
}
// Control flow enforcement (CET) features.
- if (_cpuid_info.sef_cpuid7_ecx.bits.cet_ss != 0) {
+ if (sef_cpuid7_ecx.bits.cet_ss != 0) {
result |= CPU_CET_SS;
}
- if (_cpuid_info.sef_cpuid7_edx.bits.cet_ibt != 0) {
+ if (sef_cpuid7_edx.bits.cet_ibt != 0) {
result |= CPU_CET_IBT;
}
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index e521a6ee3bc0e..03596a6e4468c 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -428,7 +428,8 @@ class VM_Version : public Abstract_VM_Version {
//
// The info block is laid out in subblocks of 4 dwords corresponding to
// eax, ebx, ecx and edx, whether or not they contain anything useful.
- struct CpuidInfo {
+ class CpuidInfo {
+ public:
// cpuid function 0
uint32_t std_max_function;
uint32_t std_vendor_name_0;
@@ -522,6 +523,31 @@ class VM_Version : public Abstract_VM_Version {
// Space to save zmm registers after signal handle
int zmm_save[16*4]; // Save zmm0, zmm7, zmm8, zmm31
+
+ uint64_t feature_flags() const;
+
+ // Asserts
+ void assert_is_initialized() const {
+ assert(std_cpuid1_eax.bits.family != 0, "VM_Version not initialized");
+ }
+
+ // Extractors
+ uint32_t extended_cpu_family() const {
+ uint32_t result = std_cpuid1_eax.bits.family;
+ result += std_cpuid1_eax.bits.ext_family;
+ return result;
+ }
+
+ uint32_t extended_cpu_model() const {
+ uint32_t result = std_cpuid1_eax.bits.model;
+ result |= std_cpuid1_eax.bits.ext_model << 4;
+ return result;
+ }
+
+ uint32_t cpu_stepping() const {
+ uint32_t result = std_cpuid1_eax.bits.stepping;
+ return result;
+ }
};
private:
@@ -529,23 +555,6 @@ class VM_Version : public Abstract_VM_Version {
static CpuidInfo _cpuid_info;
// Extractors and predicates
- static uint32_t extended_cpu_family() {
- uint32_t result = _cpuid_info.std_cpuid1_eax.bits.family;
- result += _cpuid_info.std_cpuid1_eax.bits.ext_family;
- return result;
- }
-
- static uint32_t extended_cpu_model() {
- uint32_t result = _cpuid_info.std_cpuid1_eax.bits.model;
- result |= _cpuid_info.std_cpuid1_eax.bits.ext_model << 4;
- return result;
- }
-
- static uint32_t cpu_stepping() {
- uint32_t result = _cpuid_info.std_cpuid1_eax.bits.stepping;
- return result;
- }
-
static uint logical_processor_count() {
uint result = threads_per_core();
return result;
@@ -553,7 +562,6 @@ class VM_Version : public Abstract_VM_Version {
static bool compute_has_intel_jcc_erratum();
- static uint64_t feature_flags();
static bool os_supports_avx_vectors();
static void get_processor_features();
@@ -594,11 +602,6 @@ class VM_Version : public Abstract_VM_Version {
// Override Abstract_VM_Version implementation
static void print_platform_virtualization_info(outputStream*);
- // Asserts
- static void assert_is_initialized() {
- assert(_cpuid_info.std_cpuid1_eax.bits.family != 0, "VM_Version not initialized");
- }
-
//
// Processor family:
// 3 - 386
@@ -614,6 +617,10 @@ class VM_Version : public Abstract_VM_Version {
// processors. Use the feature test functions below to
// determine whether a particular instruction is supported.
//
+ static void assert_is_initialized() { _cpuid_info.assert_is_initialized(); }
+ static uint32_t extended_cpu_family() { return _cpuid_info.extended_cpu_family(); }
+ static uint32_t extended_cpu_model() { return _cpuid_info.extended_cpu_model(); }
+ static uint32_t cpu_stepping() { return _cpuid_info.cpu_stepping(); }
static int cpu_family() { return _cpu;}
static bool is_P6() { return cpu_family() >= 6; }
static bool is_amd() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x68747541; } // 'htuA'
@@ -640,7 +647,7 @@ class VM_Version : public Abstract_VM_Version {
}
//
- // Feature identification
+ // Feature identification which can be affected by VM settings
//
static bool supports_cpuid() { return _features != 0; }
static bool supports_cmov() { return (_features & CPU_CMOV) != 0; }
@@ -703,6 +710,11 @@ class VM_Version : public Abstract_VM_Version {
static bool supports_cet_ss() { return (_features & CPU_CET_SS) != 0; }
static bool supports_cet_ibt() { return (_features & CPU_CET_IBT) != 0; }
+ //
+ // Feature identification not affected by VM flags
+ //
+ static bool cpu_supports_evex() { return (_cpu_features & CPU_AVX512F) != 0; }
+
// Intel features
static bool is_intel_family_core() { return is_intel() &&
extended_cpu_family() == CPU_FAMILY_INTEL_CORE; }
@@ -765,6 +777,10 @@ class VM_Version : public Abstract_VM_Version {
return true;
}
+ constexpr static bool supports_recursive_lightweight_locking() {
+ return true;
+ }
+
// For AVX CPUs only. f16c support is disabled if UseAVX == 0.
static bool supports_float16() {
return supports_f16c() || supports_avx512vl();
diff --git a/src/hotspot/cpu/x86/vtableStubs_x86_32.cpp b/src/hotspot/cpu/x86/vtableStubs_x86_32.cpp
index 0e78e0274d7f2..398f2e37eb5cc 100644
--- a/src/hotspot/cpu/x86/vtableStubs_x86_32.cpp
+++ b/src/hotspot/cpu/x86/vtableStubs_x86_32.cpp
@@ -24,10 +24,10 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_x86.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -176,21 +176,21 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
#endif /* PRODUCT */
// Entry arguments:
- // rax: CompiledICHolder
+ // rax: CompiledICData
// rcx: Receiver
// Most registers are in use; we'll use rax, rbx, rcx, rdx, rsi, rdi
// (If we need to make rsi, rdi callee-save, do a push/pop here.)
const Register recv_klass_reg = rsi;
- const Register holder_klass_reg = rax; // declaring interface klass (DECC)
+ const Register holder_klass_reg = rax; // declaring interface klass (DEFC)
const Register resolved_klass_reg = rdi; // resolved interface klass (REFC)
const Register temp_reg = rdx;
const Register method = rbx;
- const Register icholder_reg = rax;
+ const Register icdata_reg = rax;
const Register receiver = rcx;
- __ movptr(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
- __ movptr(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
+ __ movptr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+ __ movptr(holder_klass_reg, Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
Label L_no_such_interface;
diff --git a/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp b/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
index f162a651183f9..158d6f9c6922b 100644
--- a/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
+++ b/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
@@ -24,10 +24,10 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_x86.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -168,21 +168,21 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
#endif // PRODUCT
// Entry arguments:
- // rax: CompiledICHolder
+ // rax: CompiledICData
// j_rarg0: Receiver
// Most registers are in use; we'll use rax, rbx, r10, r11
// (various calling sequences use r[cd]x, r[sd]i, r[89]; stay away from them)
const Register recv_klass_reg = r10;
- const Register holder_klass_reg = rax; // declaring interface klass (DECC)
+ const Register holder_klass_reg = rax; // declaring interface klass (DEFC)
const Register resolved_klass_reg = r14; // resolved interface klass (REFC)
const Register temp_reg = r11;
const Register temp_reg2 = r13;
const Register method = rbx;
- const Register icholder_reg = rax;
+ const Register icdata_reg = rax;
- __ movptr(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
- __ movptr(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
+ __ movptr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+ __ movptr(holder_klass_reg, Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
Label L_no_such_interface;
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index caa82aab99c2d..6df02d280bcef 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1312,7 +1312,7 @@ int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
// That's why we must use the macroassembler to generate a handler.
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_exception_handler());
- if (base == NULL) {
+ if (base == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -1330,7 +1330,7 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
// That's why we must use the macroassembler to generate a handler.
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_deopt_handler());
- if (base == NULL) {
+ if (base == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -1358,7 +1358,7 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
return offset;
}
-Assembler::Width widthForType(BasicType bt) {
+static Assembler::Width widthForType(BasicType bt) {
if (bt == T_BYTE) {
return Assembler::B;
} else if (bt == T_SHORT) {
@@ -1425,6 +1425,8 @@ bool Matcher::match_rule_supported(int opcode) {
return false;
}
break;
+ case Op_CompressV:
+ case Op_ExpandV:
case Op_PopCountVL:
if (UseAVX < 2) {
return false;
@@ -1659,12 +1661,6 @@ bool Matcher::match_rule_supported(int opcode) {
return false;
}
break;
- case Op_CompressV:
- case Op_ExpandV:
- if (!VM_Version::supports_avx512vl()) {
- return false;
- }
- break;
case Op_SqrtF:
if (UseSSE < 1) {
return false;
@@ -1703,7 +1699,7 @@ static inline bool is_pop_count_instr_target(BasicType bt) {
(is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
}
-bool Matcher::match_rule_supported_superword(int opcode, int vlen, BasicType bt) {
+bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
return match_rule_supported_vector(opcode, vlen, bt);
}
@@ -1952,13 +1948,12 @@ bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
if (is_subword_type(bt) && !VM_Version::supports_avx512_vbmi2()) {
return false;
}
- if (size_in_bits < 128 ) {
+ if (!is_LP64 && !VM_Version::supports_avx512vl() && size_in_bits < 512) {
return false;
}
- if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
+ if (size_in_bits < 128 ) {
return false;
}
- break;
case Op_VectorLongToMask:
if (UseAVX < 1 || !is_LP64) {
return false;
@@ -2186,7 +2181,7 @@ MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd,
}
}
ShouldNotReachHere();
- return NULL;
+ return nullptr;
}
bool Matcher::is_reg2reg_move(MachNode* m) {
@@ -2285,7 +2280,7 @@ int Matcher::min_vector_size(const BasicType bt) {
return MIN2(size,max_size);
}
-int Matcher::superword_max_vector_size(const BasicType bt) {
+int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
// Limit the max vector size for auto vectorization to 256 bits (32 bytes)
// by default on Cascade Lake
if (VM_Version::is_default_intel_cascade_lake()) {
@@ -2355,7 +2350,7 @@ class FusedPatternMatcher {
int _con_op;
static int match_next(Node* n, int next_op, int next_op_idx) {
- if (n->in(1) == NULL || n->in(2) == NULL) {
+ if (n->in(1) == nullptr || n->in(2) == nullptr) {
return -1;
}
@@ -2422,7 +2417,7 @@ class FusedPatternMatcher {
static bool is_bmi_pattern(Node* n, Node* m) {
assert(UseBMI1Instructions, "sanity");
- if (n != NULL && m != NULL) {
+ if (n != nullptr && m != nullptr) {
if (m->Opcode() == Op_LoadI) {
FusedPatternMatcher bmii(n, m, Op_ConI);
return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) ||
@@ -7446,7 +7441,7 @@ instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg
// --------------------------------- VectorMaskCmp --------------------------------------
instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
- predicate(n->bottom_type()->isa_vectmask() == NULL &&
+ predicate(n->bottom_type()->isa_vectmask() == nullptr &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
@@ -7466,7 +7461,7 @@ instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
- n->bottom_type()->isa_vectmask() == NULL &&
+ n->bottom_type()->isa_vectmask() == nullptr &&
is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
effect(TEMP ktmp);
@@ -7506,7 +7501,7 @@ instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
%}
instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
- predicate(n->bottom_type()->isa_vectmask() == NULL &&
+ predicate(n->bottom_type()->isa_vectmask() == nullptr &&
!Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
@@ -7526,7 +7521,7 @@ instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
%}
instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
- predicate(n->bottom_type()->isa_vectmask() == NULL &&
+ predicate(n->bottom_type()->isa_vectmask() == nullptr &&
!Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
@@ -7547,7 +7542,7 @@ instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xt
%}
instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
- predicate(n->bottom_type()->isa_vectmask() == NULL &&
+ predicate(n->bottom_type()->isa_vectmask() == nullptr &&
Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
@@ -7574,7 +7569,7 @@ instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
%}
instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
- predicate((n->bottom_type()->isa_vectmask() == NULL &&
+ predicate((n->bottom_type()->isa_vectmask() == nullptr &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
@@ -7790,7 +7785,7 @@ instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
- n->in(2)->bottom_type()->isa_vectmask() == NULL &&
+ n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
Matcher::vector_length_in_bytes(n) <= 32 &&
is_integral_type(Matcher::vector_element_basic_type(n)));
match(Set dst (VectorBlend (Binary src1 src2) mask));
@@ -7804,7 +7799,7 @@ instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
- n->in(2)->bottom_type()->isa_vectmask() == NULL &&
+ n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
Matcher::vector_length_in_bytes(n) <= 32 &&
!is_integral_type(Matcher::vector_element_basic_type(n)));
match(Set dst (VectorBlend (Binary src1 src2) mask));
@@ -7818,7 +7813,7 @@ instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp) %{
predicate(UseAVX > 0 && EnableX86ECoreOpts &&
- n->in(2)->bottom_type()->isa_vectmask() == NULL &&
+ n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
Matcher::vector_length_in_bytes(n) <= 32);
match(Set dst (VectorBlend (Binary src1 src2) mask));
format %{ "vector_blend $dst,$src1,$src2,$mask\t! using $vtmp as TEMP" %}
@@ -7834,7 +7829,7 @@ instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp
instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
predicate(Matcher::vector_length_in_bytes(n) == 64 &&
- n->in(2)->bottom_type()->isa_vectmask() == NULL);
+ n->in(2)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorBlend (Binary src1 src2) mask));
format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
effect(TEMP ktmp);
@@ -8051,7 +8046,7 @@ instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
//------------------------------------- LoadMask --------------------------------------------
instruct loadMask(legVec dst, legVec src) %{
- predicate(n->bottom_type()->isa_vectmask() == NULL && !VM_Version::supports_avx512vlbw());
+ predicate(n->bottom_type()->isa_vectmask() == nullptr && !VM_Version::supports_avx512vlbw());
match(Set dst (VectorLoadMask src));
effect(TEMP dst);
format %{ "vector_loadmask_byte $dst, $src\n\t" %}
@@ -8091,7 +8086,7 @@ instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
//------------------------------------- StoreMask --------------------------------------------
instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
- predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
ins_encode %{
@@ -8109,7 +8104,7 @@ instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
%}
instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
- predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
effect(TEMP_DEF dst, TEMP xtmp);
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
@@ -8132,7 +8127,7 @@ instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
%}
instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
- predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
effect(TEMP_DEF dst, TEMP xtmp);
@@ -8192,7 +8187,7 @@ instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
%}
instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
- predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
ins_encode %{
@@ -8208,7 +8203,7 @@ instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
%}
instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
- predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
ins_encode %{
@@ -9045,7 +9040,7 @@ instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
%}
instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorMaskToLong mask));
format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
@@ -9061,7 +9056,7 @@ instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
%}
instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
@@ -9094,7 +9089,7 @@ instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
%}
instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorMaskTrueCount mask));
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
@@ -9110,7 +9105,7 @@ instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsRe
%}
instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
@@ -9144,7 +9139,7 @@ instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsRe
%}
instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorMaskFirstTrue mask));
match(Set dst (VectorMaskLastTrue mask));
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
@@ -9161,7 +9156,7 @@ instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp,
%}
instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == NULL);
+ predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
@@ -9178,8 +9173,26 @@ instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp,
%}
// --------------------------------- Compress/Expand Operations ---------------------------
+#ifdef _LP64
+instruct vcompress_reg_avx(vec dst, vec src, vec mask, rRegI rtmp, rRegL rscratch, vec perm, vec xtmp, rFlagsReg cr) %{
+ predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
+ match(Set dst (CompressV src mask));
+ match(Set dst (ExpandV src mask));
+ effect(TEMP_DEF dst, TEMP perm, TEMP xtmp, TEMP rtmp, TEMP rscratch, KILL cr);
+ format %{ "vector_compress $dst, $src, $mask \t!using $xtmp, $rtmp, $rscratch and $perm as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ int vlen_enc = vector_length_encoding(this);
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vector_compress_expand_avx2(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$XMMRegister, $rtmp$$Register,
+ $rscratch$$Register, $perm$$XMMRegister, $xtmp$$XMMRegister, bt, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+#endif
instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
+ predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
match(Set dst (CompressV src mask));
match(Set dst (ExpandV src mask));
format %{ "vector_compress_expand $dst, $src, $mask" %}
@@ -9998,7 +10011,7 @@ instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
%}
instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp) %{
- predicate(n->bottom_type()->isa_vectmask() == NULL && Matcher::vector_length(n) <= 8);
+ predicate(n->bottom_type()->isa_vectmask() == nullptr && Matcher::vector_length(n) <= 8);
match(Set dst (VectorLongToMask src));
effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp);
format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp as TEMP" %}
@@ -10013,7 +10026,7 @@ instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec x
instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
- predicate(n->bottom_type()->isa_vectmask() == NULL && Matcher::vector_length(n) > 8);
+ predicate(n->bottom_type()->isa_vectmask() == nullptr && Matcher::vector_length(n) > 8);
match(Set dst (VectorLongToMask src));
effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad
index f2e9c042b244f..2fe655a576778 100644
--- a/src/hotspot/cpu/x86/x86_32.ad
+++ b/src/hotspot/cpu/x86/x86_32.ad
@@ -504,7 +504,7 @@ void emit_cmpfp_fixup(MacroAssembler& _masm) {
__ bind(exit);
}
-void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
+static void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
Label done;
__ movl(dst, -1);
__ jcc(Assembler::parity, done);
@@ -614,7 +614,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
int framesize = C->output()->frame_size_in_bytes();
int bangsize = C->output()->bang_size_in_bytes();
- __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, C->in_24_bit_fp_mode(), C->stub_function() != NULL);
+ __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, C->in_24_bit_fp_mode(), C->stub_function() != nullptr);
C->output()->set_frame_complete(cbuf.insts_size());
@@ -1052,7 +1052,7 @@ uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bo
if( src_first == dst_first && src_second == dst_second )
return size; // Self copy, no move
- if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
+ if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
uint ireg = ideal_reg();
assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
assert((src_first_rc != rc_float && dst_first_rc != rc_float), "sanity");
@@ -1320,12 +1320,12 @@ uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bo
#ifndef PRODUCT
void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
- implementation( NULL, ra_, false, st );
+ implementation( nullptr, ra_, false, st );
}
#endif
void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- implementation( &cbuf, ra_, false, NULL );
+ implementation( &cbuf, ra_, false, nullptr );
}
uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
@@ -1383,24 +1383,12 @@ void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
MacroAssembler masm(&cbuf);
-#ifdef ASSERT
- uint insts_size = cbuf.insts_size();
-#endif
- masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));
- masm.jump_cc(Assembler::notEqual,
- RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- /* WARNING these NOPs are critical so that verified entry point is properly
- aligned for patching by NativeJump::patch_verified_entry() */
- int nops_cnt = 2;
- if( !OptoBreakpoint ) // Leave space for int3
- nops_cnt += 1;
- masm.nop(nops_cnt);
-
- assert(cbuf.insts_size() - insts_size == size(ra_), "checking code size of inline cache node");
+ masm.ic_check(CodeEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
- return OptoBreakpoint ? 11 : 12;
+ return MachNode::size(ra_); // too many variables; just compute it
+ // the hard way
}
@@ -1725,7 +1713,7 @@ encode %{
MacroAssembler _masm(&cbuf);
__ check_klass_subtype_slow_path(Resi, Reax, Recx, Redi,
- NULL, &miss,
+ nullptr, &miss,
/*set_cond_codes:*/ true);
if ($primary) {
__ xorptr(Redi, Redi);
@@ -1842,8 +1830,8 @@ encode %{
cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
} else {
// Emit stubs for static call.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
- if (stub == NULL) {
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, mark);
+ if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -3396,7 +3384,7 @@ operand immP() %{
interface(CONST_INTER);
%}
-// NULL Pointer Immediate
+// Null Pointer Immediate
operand immP0() %{
predicate( n->get_ptr() == 0 );
match(ConP);
@@ -13776,7 +13764,7 @@ instruct cmpFastLockRTM(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eD
%}
instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP scr, eRegP thread) %{
- predicate(!Compile::current()->use_rtm());
+ predicate(LockingMode != LM_LIGHTWEIGHT && !Compile::current()->use_rtm());
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP scr, USE_KILL box, TEMP thread);
ins_cost(300);
@@ -13790,6 +13778,7 @@ instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP
%}
instruct cmpFastUnlock(eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastUnlock object box));
effect(TEMP tmp, USE_KILL box);
ins_cost(300);
@@ -13800,6 +13789,32 @@ instruct cmpFastUnlock(eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
ins_pipe(pipe_slow);
%}
+instruct cmpFastLockLightweight(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI eax_reg, eRegP tmp, eRegP thread) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastLock object box));
+ effect(TEMP eax_reg, TEMP tmp, USE_KILL box, TEMP thread);
+ ins_cost(300);
+ format %{ "FASTLOCK $object,$box\t! kills $box,$eax_reg,$tmp" %}
+ ins_encode %{
+ __ get_thread($thread$$Register);
+ __ fast_lock_lightweight($object$$Register, $box$$Register, $eax_reg$$Register, $tmp$$Register, $thread$$Register);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct cmpFastUnlockLightweight(eFlagsReg cr, eRegP object, eAXRegP eax_reg, eRegP tmp, eRegP thread) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastUnlock object eax_reg));
+ effect(TEMP tmp, USE_KILL eax_reg, TEMP thread);
+ ins_cost(300);
+ format %{ "FASTUNLOCK $object,$eax_reg\t! kills $eax_reg,$tmp" %}
+ ins_encode %{
+ __ get_thread($thread$$Register);
+ __ fast_unlock_lightweight($object$$Register, $eax_reg$$Register, $tmp$$Register, $thread$$Register);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct mask_all_evexL_LT32(kReg dst, eRegL src) %{
predicate(Matcher::vector_length(n) <= 32);
match(Set dst (MaskAll src));
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index 80f281a1bf92d..d43929efd3ec5 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -519,7 +519,7 @@ int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
}
// This could be in MacroAssembler but it's fairly C2 specific
-void emit_cmpfp_fixup(MacroAssembler& _masm) {
+static void emit_cmpfp_fixup(MacroAssembler& _masm) {
Label exit;
__ jccb(Assembler::noParity, exit);
__ pushf();
@@ -539,7 +539,7 @@ void emit_cmpfp_fixup(MacroAssembler& _masm) {
__ bind(exit);
}
-void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
+static void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
Label done;
__ movl(dst, -1);
__ jcc(Assembler::parity, done);
@@ -558,10 +558,10 @@ void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
// je #
// |-jz -> a | b # a & b
// | -> a #
-void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
- XMMRegister a, XMMRegister b,
- XMMRegister xmmt, Register rt,
- bool min, bool single) {
+static void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
+ XMMRegister a, XMMRegister b,
+ XMMRegister xmmt, Register rt,
+ bool min, bool single) {
Label nan, zero, below, above, done;
@@ -706,7 +706,7 @@ void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
st->print("# stack alignment check");
#endif
}
- if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
+ if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
st->print("\n\t");
st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
st->print("\n\t");
@@ -741,7 +741,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
__ bind(L_skip_barrier);
}
- __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
+ __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
C->output()->set_frame_complete(cbuf.insts_size());
@@ -970,7 +970,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
PhaseRegAlloc* ra_,
bool do_size,
outputStream* st) const {
- assert(cbuf != NULL || st != NULL, "sanity");
+ assert(cbuf != nullptr || st != nullptr, "sanity");
// Get registers to move
OptoReg::Name src_second = ra_->get_reg_second(in(1));
OptoReg::Name src_first = ra_->get_reg_first(in(1));
@@ -989,7 +989,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
// Self copy, no move
return 0;
}
- if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
+ if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
uint ireg = ideal_reg();
assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
@@ -1428,12 +1428,12 @@ uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
#ifndef PRODUCT
void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
- implementation(NULL, ra_, false, st);
+ implementation(nullptr, ra_, false, st);
}
#endif
void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
- implementation(&cbuf, ra_, false, NULL);
+ implementation(&cbuf, ra_, false, nullptr);
}
uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
@@ -1472,40 +1472,19 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
if (UseCompressedClassPointers) {
st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
- st->print_cr("\tcmpq rax, rscratch1\t # Inline cache check");
+ st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
} else {
- st->print_cr("\tcmpq rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
- "# Inline cache check");
+ st->print_cr("movq rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tcmpq rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
}
st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
- st->print_cr("\tnop\t# nops to align entry point");
}
#endif
void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
{
MacroAssembler masm(&cbuf);
- uint insts_size = cbuf.insts_size();
- if (UseCompressedClassPointers) {
- masm.load_klass(rscratch1, j_rarg0, rscratch2);
- masm.cmpptr(rax, rscratch1);
- } else {
- masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
- }
-
- masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- /* WARNING these NOPs are critical so that verified entry point is properly
- 4 bytes aligned for patching by NativeJump::patch_verified_entry() */
- int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
- if (OptoBreakpoint) {
- // Leave space for int3
- nops_cnt -= 1;
- }
- nops_cnt &= 0x3; // Do not add nops if code is aligned.
- if (nops_cnt > 0)
- masm.nop(nops_cnt);
+ masm.ic_check(InteriorEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc* ra_) const
@@ -1784,7 +1763,7 @@ encode %{
MacroAssembler _masm(&cbuf);
__ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
- NULL, &miss,
+ nullptr, &miss,
/*set_cond_codes:*/ true);
if ($primary) {
__ xorptr(Rrdi, Rrdi);
@@ -1840,8 +1819,8 @@ encode %{
cbuf.shared_stub_to_interp_for(_method, call_offset);
} else {
// Emit stubs for static call.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
- if (stub == NULL) {
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, mark);
+ if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
}
@@ -2179,7 +2158,7 @@ operand immP()
interface(CONST_INTER);
%}
-// NULL Pointer Immediate
+// Null Pointer Immediate
operand immP0()
%{
predicate(n->get_ptr() == 0);
@@ -2207,7 +2186,7 @@ operand immNKlass() %{
interface(CONST_INTER);
%}
-// NULL Pointer Immediate
+// Null Pointer Immediate
operand immN0() %{
predicate(n->get_narrowcon() == 0);
match(ConN);
@@ -3121,7 +3100,7 @@ operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
// Indirect Narrow Oop Plus Offset Operand
// Note: x86 architecture doesn't support "scale * index + offset" without a base
-// we can't free r12 even with CompressedOops::base() == NULL.
+// we can't free r12 even with CompressedOops::base() == nullptr.
operand indCompressedOopOffset(rRegN reg, immL32 off) %{
predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
constraint(ALLOC_IN_RC(ptr_reg));
@@ -4480,7 +4459,7 @@ instruct loadD(regD dst, memory mem)
// max = java.lang.Math.max(float a, float b)
instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
- predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
@@ -4491,7 +4470,7 @@ instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp,
%}
instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
- predicate(UseAVX > 0 && SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
@@ -4505,7 +4484,7 @@ instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRe
// max = java.lang.Math.max(double a, double b)
instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MaxD a b));
effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
@@ -4516,7 +4495,7 @@ instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp,
%}
instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
- predicate(UseAVX > 0 && SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MaxD a b));
effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
@@ -4530,7 +4509,7 @@ instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRe
// min = java.lang.Math.min(float a, float b)
instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
- predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
@@ -4541,7 +4520,7 @@ instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp,
%}
instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
- predicate(UseAVX > 0 && SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
@@ -4555,7 +4534,7 @@ instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRe
// min = java.lang.Math.min(double a, double b)
instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MinD a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
@@ -4566,7 +4545,7 @@ instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp,
%}
instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
- predicate(UseAVX > 0 && SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MinD a b));
effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
@@ -4902,7 +4881,7 @@ instruct loadConF(regF dst, immF con) %{
instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
match(Set dst src);
effect(KILL cr);
- format %{ "xorq $dst, $src\t# compressed NULL ptr" %}
+ format %{ "xorq $dst, $src\t# compressed null pointer" %}
ins_encode %{
__ xorq($dst$$Register, $dst$$Register);
%}
@@ -4916,7 +4895,7 @@ instruct loadConN(rRegN dst, immN src) %{
format %{ "movl $dst, $src\t# compressed ptr" %}
ins_encode %{
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
ShouldNotReachHere();
} else {
__ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
@@ -4932,7 +4911,7 @@ instruct loadConNKlass(rRegN dst, immNKlass src) %{
format %{ "movl $dst, $src\t# compressed klass ptr" %}
ins_encode %{
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
ShouldNotReachHere();
} else {
__ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
@@ -5158,7 +5137,7 @@ instruct storeP(memory mem, any_RegP src)
instruct storeImmP0(memory mem, immP0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL) && n->as_Store()->barrier_data() == 0);
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
match(Set mem (StoreP mem zero));
ins_cost(125); // XXX
@@ -5169,7 +5148,7 @@ instruct storeImmP0(memory mem, immP0 zero)
ins_pipe(ialu_mem_reg);
%}
-// Store NULL Pointer, mark word, or other simple pointer constant.
+// Store Null Pointer, mark word, or other simple pointer constant.
instruct storeImmP(memory mem, immP31 src)
%{
predicate(n->as_Store()->barrier_data() == 0);
@@ -5210,7 +5189,7 @@ instruct storeNKlass(memory mem, rRegN src)
instruct storeImmN0(memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() == NULL);
+ predicate(CompressedOops::base() == nullptr);
match(Set mem (StoreN mem zero));
ins_cost(125); // XXX
@@ -5229,7 +5208,7 @@ instruct storeImmN(memory mem, immN src)
format %{ "movl $mem, $src\t# compressed ptr" %}
ins_encode %{
address con = (address)$src$$constant;
- if (con == NULL) {
+ if (con == nullptr) {
__ movl($mem$$Address, 0);
} else {
__ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
@@ -5253,7 +5232,7 @@ instruct storeImmNKlass(memory mem, immNKlass src)
// Store Integer Immediate
instruct storeImmI0(memory mem, immI_0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreI mem zero));
ins_cost(125); // XXX
@@ -5279,7 +5258,7 @@ instruct storeImmI(memory mem, immI src)
// Store Long Immediate
instruct storeImmL0(memory mem, immL0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreL mem zero));
ins_cost(125); // XXX
@@ -5305,7 +5284,7 @@ instruct storeImmL(memory mem, immL32 src)
// Store Short/Char Immediate
instruct storeImmC0(memory mem, immI_0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreC mem zero));
ins_cost(125); // XXX
@@ -5332,7 +5311,7 @@ instruct storeImmI16(memory mem, immI16 src)
// Store Byte Immediate
instruct storeImmB0(memory mem, immI_0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreB mem zero));
ins_cost(125); // XXX
@@ -5358,7 +5337,7 @@ instruct storeImmB(memory mem, immI8 src)
// Store CMS card-mark Immediate
instruct storeImmCM0_reg(memory mem, immI_0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreCM mem zero));
ins_cost(125); // XXX
@@ -5397,7 +5376,7 @@ instruct storeF(memory mem, regF src)
// Store immediate Float value (it is faster than store from XMM register)
instruct storeF0(memory mem, immF0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreF mem zero));
ins_cost(25); // XXX
@@ -5436,7 +5415,7 @@ instruct storeD(memory mem, regD src)
// Store immediate double 0.0 (it is faster than store from XMM register)
instruct storeD0_imm(memory mem, immD0 src)
%{
- predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
+ predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
match(Set mem (StoreD mem src));
ins_cost(50);
@@ -5449,7 +5428,7 @@ instruct storeD0_imm(memory mem, immD0 src)
instruct storeD0(memory mem, immD0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL));
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
match(Set mem (StoreD mem zero));
ins_cost(25); // XXX
@@ -11680,7 +11659,7 @@ instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
// any compare to a zero should be eq/neq.
instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
%{
- predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
+ predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
n->in(1)->as_Load()->barrier_data() == 0);
match(Set cr (CmpP (LoadP op) zero));
@@ -11694,7 +11673,7 @@ instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
%{
- predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
+ predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
n->in(1)->as_Load()->barrier_data() == 0);
match(Set cr (CmpP (LoadP mem) zero));
@@ -11777,7 +11756,7 @@ instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() != NULL);
+ predicate(CompressedOops::base() != nullptr);
match(Set cr (CmpN (LoadN mem) zero));
ins_cost(500); // XXX
@@ -11790,7 +11769,7 @@ instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() == NULL);
+ predicate(CompressedOops::base() == nullptr);
match(Set cr (CmpN (LoadN mem) zero));
format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
@@ -12404,7 +12383,7 @@ instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp,
%}
instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
- predicate(!Compile::current()->use_rtm());
+ predicate(LockingMode != LM_LIGHTWEIGHT && !Compile::current()->use_rtm());
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP scr, USE_KILL box);
ins_cost(300);
@@ -12417,6 +12396,7 @@ instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRe
%}
instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastUnlock object box));
effect(TEMP tmp, USE_KILL box);
ins_cost(300);
@@ -12427,6 +12407,30 @@ instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
ins_pipe(pipe_slow);
%}
+instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastLock object box));
+ effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
+ ins_cost(300);
+ format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
+ ins_encode %{
+ __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastUnlock object rax_reg));
+ effect(TEMP tmp, USE_KILL rax_reg);
+ ins_cost(300);
+ format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
+ ins_encode %{
+ __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// ============================================================================
// Safepoint Instructions
diff --git a/src/hotspot/cpu/zero/compiledIC_zero.cpp b/src/hotspot/cpu/zero/compiledIC_zero.cpp
index b0564643af080..24153aeacc5e1 100644
--- a/src/hotspot/cpu/zero/compiledIC_zero.cpp
+++ b/src/hotspot/cpu/zero/compiledIC_zero.cpp
@@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
@@ -43,27 +42,27 @@
// ----------------------------------------------------------------------------
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
ShouldNotReachHere(); // Only needed for COMPILER2.
return nullptr;
}
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
ShouldNotReachHere(); // Only needed for COMPILER2.
return 0;
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
ShouldNotReachHere(); // Only needed for COMPILER2.
return 0;
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
ShouldNotReachHere(); // Only needed for COMPILER2.
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
ShouldNotReachHere(); // Only needed for COMPILER2.
}
@@ -71,7 +70,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code.
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
ShouldNotReachHere(); // Only needed for COMPILER2.
}
diff --git a/src/hotspot/cpu/zero/icBuffer_zero.cpp b/src/hotspot/cpu/zero/icBuffer_zero.cpp
deleted file mode 100644
index adde916a4c4ad..0000000000000
--- a/src/hotspot/cpu/zero/icBuffer_zero.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2007 Red Hat, Inc.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/assembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_zero.hpp"
-#include "oops/oop.inline.hpp"
-
-int InlineCacheBuffer::ic_stub_code_size() {
- // NB set this once the functions below are implemented
- return 4;
-}
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin,
- void* cached_oop,
- address entry_point) {
- // NB ic_stub_code_size() must return the size of the code we generate
- ShouldNotCallThis();
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- // NB ic_stub_code_size() must return the size of the code we generate
- ShouldNotCallThis();
- return nullptr;
-}
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- ShouldNotCallThis();
- return nullptr;
-}
diff --git a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
index 4244b5817db98..986cee685123b 100644
--- a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
+++ b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
@@ -26,10 +26,8 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
-#include "oops/compiledICHolder.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/vframeArray.hpp"
diff --git a/src/hotspot/os/aix/attachListener_aix.cpp b/src/hotspot/os/aix/attachListener_aix.cpp
index 2e079c9b3a18b..51e2b2b3c0ac3 100644
--- a/src/hotspot/os/aix/attachListener_aix.cpp
+++ b/src/hotspot/os/aix/attachListener_aix.cpp
@@ -478,14 +478,14 @@ AttachOperation* AttachListener::dequeue() {
void AttachListener::vm_start() {
char fn[UNIX_PATH_MAX];
- struct stat64 st;
+ struct stat st;
int ret;
int n = snprintf(fn, UNIX_PATH_MAX, "%s/.java_pid%d",
os::get_temp_directory(), os::current_process_id());
assert(n < (int)UNIX_PATH_MAX, "java_pid file name buffer overflow");
- RESTARTABLE(::stat64(fn, &st), ret);
+ RESTARTABLE(::stat(fn, &st), ret);
if (ret == 0) {
ret = ::unlink(fn);
if (ret == -1) {
@@ -505,8 +505,8 @@ int AttachListener::pd_init() {
bool AttachListener::check_socket_file() {
int ret;
- struct stat64 st;
- ret = stat64(AixAttachListener::path(), &st);
+ struct stat st;
+ ret = stat(AixAttachListener::path(), &st);
if (ret == -1) { // need to restart attach listener.
log_debug(attach)("Socket file %s does not exist - Restart Attach Listener",
AixAttachListener::path());
@@ -545,14 +545,14 @@ bool AttachListener::is_init_trigger() {
}
char fn[PATH_MAX + 1];
int ret;
- struct stat64 st;
+ struct stat st;
os::snprintf_checked(fn, sizeof(fn), ".attach_pid%d", os::current_process_id());
- RESTARTABLE(::stat64(fn, &st), ret);
+ RESTARTABLE(::stat(fn, &st), ret);
if (ret == -1) {
log_trace(attach)("Failed to find attach file: %s, trying alternate", fn);
snprintf(fn, sizeof(fn), "%s/.attach_pid%d",
os::get_temp_directory(), os::current_process_id());
- RESTARTABLE(::stat64(fn, &st), ret);
+ RESTARTABLE(::stat(fn, &st), ret);
if (ret == -1) {
log_debug(attach)("Failed to find attach file: %s", fn);
}
diff --git a/src/hotspot/os/aix/globals_aix.hpp b/src/hotspot/os/aix/globals_aix.hpp
index a047e79b695fa..fb353348a5364 100644
--- a/src/hotspot/os/aix/globals_aix.hpp
+++ b/src/hotspot/os/aix/globals_aix.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -49,11 +49,12 @@
"Allow VM to run with EXTSHM=ON.") \
\
/* Maximum expected size of the data segment. That correlates with the */ \
- /* to the maximum C Heap consumption we expect. */ \
- /* We need to know this because we need to leave "breathing space" for the */ \
- /* data segment when placing the java heap. If that space is too small, we */ \
- /* reduce our chance of getting a low heap address (needed for compressed */ \
- /* Oops). */ \
+ /* maximum C Heap consumption we expect. */ \
+ /* We need to leave "breathing space" for the data segment when */ \
+ /* placing the java heap. If the MaxExpectedDataSegmentSize setting */ \
+ /* is too small, we might run into resource issues creating many native */ \
+ /* threads, if it is too large, we reduce our chance of getting a low heap */ \
+ /* address (needed for compressed Oops). */ \
product(uintx, MaxExpectedDataSegmentSize, 8*G, \
"Maximum expected Data Segment Size.") \
\
diff --git a/src/hotspot/os/aix/libperfstat_aix.cpp b/src/hotspot/os/aix/libperfstat_aix.cpp
index f547b4c78e77c..0185d7d041c85 100644
--- a/src/hotspot/os/aix/libperfstat_aix.cpp
+++ b/src/hotspot/os/aix/libperfstat_aix.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
* Copyright (c) 2022, IBM Corp.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -213,12 +213,8 @@ bool libperfstat::get_cpuinfo(cpuinfo_t* pci) {
if (-1 == libperfstat::perfstat_cpu_total(nullptr, &psct, sizeof(PERFSTAT_CPU_TOTAL_T_LATEST), 1)) {
if (-1 == libperfstat::perfstat_cpu_total(nullptr, &psct, sizeof(perfstat_cpu_total_t_71), 1)) {
- if (-1 == libperfstat::perfstat_cpu_total(nullptr, &psct, sizeof(perfstat_cpu_total_t_61), 1)) {
- if (-1 == libperfstat::perfstat_cpu_total(nullptr, &psct, sizeof(perfstat_cpu_total_t_53), 1)) {
- trcVerbose("perfstat_cpu_total() failed (errno=%d)", errno);
- return false;
- }
- }
+ trcVerbose("perfstat_cpu_total() failed (errno=%d)", errno);
+ return false;
}
}
@@ -252,14 +248,8 @@ bool libperfstat::get_partitioninfo(partitioninfo_t* ppi) {
if (-1 == libperfstat::perfstat_partition_total(nullptr, &pspt, sizeof(PERFSTAT_PARTITON_TOTAL_T_LATEST), 1)) {
if (-1 == libperfstat::perfstat_partition_total(nullptr, &pspt, sizeof(perfstat_partition_total_t_71), 1)) {
ame_details = false;
- if (-1 == libperfstat::perfstat_partition_total(nullptr, &pspt, sizeof(perfstat_partition_total_t_61), 1)) {
- if (-1 == libperfstat::perfstat_partition_total(nullptr, &pspt, sizeof(perfstat_partition_total_t_53), 1)) {
- if (-1 == libperfstat::perfstat_partition_total(nullptr, &pspt, sizeof(perfstat_partition_total_t_53_5), 1)) {
- trcVerbose("perfstat_partition_total() failed (errno=%d)", errno);
- return false;
- }
- }
- }
+ trcVerbose("perfstat_partition_total() failed (errno=%d)", errno);
+ return false;
}
}
@@ -324,10 +314,8 @@ bool libperfstat::get_wparinfo(wparinfo_t* pwi) {
memset (&pswt, '\0', sizeof(pswt));
if (-1 == libperfstat::perfstat_wpar_total(nullptr, &pswt, sizeof(PERFSTAT_WPAR_TOTAL_T_LATEST), 1)) {
- if (-1 == libperfstat::perfstat_wpar_total(nullptr, &pswt, sizeof(perfstat_wpar_total_t_61), 1)) {
- trcVerbose("perfstat_wpar_total() failed (errno=%d)", errno);
- return false;
- }
+ trcVerbose("perfstat_wpar_total() failed (errno=%d)", errno);
+ return false;
}
// WPAR type info.
diff --git a/src/hotspot/os/aix/libperfstat_aix.hpp b/src/hotspot/os/aix/libperfstat_aix.hpp
index 704c5b41a31b9..67cae0d08b65c 100644
--- a/src/hotspot/os/aix/libperfstat_aix.hpp
+++ b/src/hotspot/os/aix/libperfstat_aix.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
* Copyright (c) 2022, IBM Corp.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -38,16 +38,8 @@
#include
///////////////////////////////////////////////////////////////////////////////////////////////
-// These are excerpts from the AIX 5.3, 6.1, 7.1 libperfstat.h -
+// These are excerpts from the AIX 7.1 libperfstat.h -
// this is all we need from libperfstat.h and I want to avoid having to include
-//
-// Note: I define all structures as if I were to include libperfstat.h on an AIX 5.2
-// build machine.
-//
-// The ratio behind that is that if I would build on an AIX 5.2 build machine,
-// include libperfstat.h and hard-link against libperfstat.a, the program should
-// work without recompilation on all newer AIX versions.
-//
#define IDENTIFIER_LENGTH 64 /* length of strings included in the structures */
#define FIRST_CPU "" /* pseudo-name for fist CPU */
@@ -94,169 +86,6 @@ typedef struct { /* Virtual memory utilization */
} perfstat_memory_total_t;
-typedef struct { /* global cpu information AIX 5.3 < TL10 */
- int ncpus; /* number of active logical processors */
- int ncpus_cfg; /* number of configured processors */
- char description[IDENTIFIER_LENGTH]; /* processor description (type/official name) */
- u_longlong_t processorHZ; /* processor speed in Hz */
- u_longlong_t user; /* raw total number of clock ticks spent in user mode */
- u_longlong_t sys; /* raw total number of clock ticks spent in system mode */
- u_longlong_t idle; /* raw total number of clock ticks spent idle */
- u_longlong_t wait; /* raw total number of clock ticks spent waiting for I/O */
- u_longlong_t pswitch; /* number of process switches (change in currently running process) */
- u_longlong_t syscall; /* number of system calls executed */
- u_longlong_t sysread; /* number of read system calls executed */
- u_longlong_t syswrite; /* number of write system calls executed */
- u_longlong_t sysfork; /* number of forks system calls executed */
- u_longlong_t sysexec; /* number of execs system calls executed */
- u_longlong_t readch; /* number of characters transferred with read system call */
- u_longlong_t writech; /* number of characters transferred with write system call */
- u_longlong_t devintrs; /* number of device interrupts */
- u_longlong_t softintrs; /* number of software interrupts */
- time_t lbolt; /* number of ticks since last reboot */
- u_longlong_t loadavg[3]; /* (1<. */
- u_longlong_t runque; /* length of the run queue (processes ready) */
- u_longlong_t swpque; /* ength of the swap queue (processes waiting to be paged in) */
- u_longlong_t bread; /* number of blocks read */
- u_longlong_t bwrite; /* number of blocks written */
- u_longlong_t lread; /* number of logical read requests */
- u_longlong_t lwrite; /* number of logical write requests */
- u_longlong_t phread; /* number of physical reads (reads on raw devices) */
- u_longlong_t phwrite; /* number of physical writes (writes on raw devices) */
- u_longlong_t runocc; /* updated whenever runque is updated, i.e. the runqueue is occupied.
- * This can be used to compute the simple average of ready processes */
- u_longlong_t swpocc; /* updated whenever swpque is updated. i.e. the swpqueue is occupied.
- * This can be used to compute the simple average processes waiting to be paged in */
- u_longlong_t iget; /* number of inode lookups */
- u_longlong_t namei; /* number of vnode lookup from a path name */
- u_longlong_t dirblk; /* number of 512-byte block reads by the directory search routine to locate an entry for a file */
- u_longlong_t msg; /* number of IPC message operations */
- u_longlong_t sema; /* number of IPC semaphore operations */
- u_longlong_t rcvint; /* number of tty receive interrupts */
- u_longlong_t xmtint; /* number of tyy transmit interrupts */
- u_longlong_t mdmint; /* number of modem interrupts */
- u_longlong_t tty_rawinch; /* number of raw input characters */
- u_longlong_t tty_caninch; /* number of canonical input characters (always zero) */
- u_longlong_t tty_rawoutch; /* number of raw output characters */
- u_longlong_t ksched; /* number of kernel processes created */
- u_longlong_t koverf; /* kernel process creation attempts where:
- * -the user has forked to their maximum limit
- * -the configuration limit of processes has been reached */
- u_longlong_t kexit; /* number of kernel processes that became zombies */
- u_longlong_t rbread; /* number of remote read requests */
- u_longlong_t rcread; /* number of cached remote reads */
- u_longlong_t rbwrt; /* number of remote writes */
- u_longlong_t rcwrt; /* number of cached remote writes */
- u_longlong_t traps; /* number of traps */
- int ncpus_high; /* index of highest processor online */
- u_longlong_t puser; /* raw number of physical processor tics in user mode */
- u_longlong_t psys; /* raw number of physical processor tics in system mode */
- u_longlong_t pidle; /* raw number of physical processor tics idle */
- u_longlong_t pwait; /* raw number of physical processor tics waiting for I/O */
- u_longlong_t decrintrs; /* number of decrementer tics interrupts */
- u_longlong_t mpcrintrs; /* number of mpc's received interrupts */
- u_longlong_t mpcsintrs; /* number of mpc's sent interrupts */
- u_longlong_t phantintrs; /* number of phantom interrupts */
- u_longlong_t idle_donated_purr; /* number of idle cycles donated by a dedicated partition enabled for donation */
- u_longlong_t idle_donated_spurr; /* number of idle spurr cycles donated by a dedicated partition enabled for donation */
- u_longlong_t busy_donated_purr; /* number of busy cycles donated by a dedicated partition enabled for donation */
- u_longlong_t busy_donated_spurr; /* number of busy spurr cycles donated by a dedicated partition enabled for donation */
- u_longlong_t idle_stolen_purr; /* number of idle cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t idle_stolen_spurr; /* number of idle spurr cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t busy_stolen_purr; /* number of busy cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t busy_stolen_spurr; /* number of busy spurr cycles stolen by the hypervisor from a dedicated partition */
- short iowait; /* number of processes that are asleep waiting for buffered I/O */
- short physio; /* number of processes waiting for raw I/O */
- longlong_t twait; /* number of threads that are waiting for filesystem direct(cio) */
- u_longlong_t hpi; /* number of hypervisor page-ins */
- u_longlong_t hpit; /* Time spent in hypervisor page-ins (in nanoseconds) */
-} perfstat_cpu_total_t_53;
-
-typedef struct { /* global cpu information AIX 6.1|5.3 > TL09 */
- int ncpus; /* number of active logical processors */
- int ncpus_cfg; /* number of configured processors */
- char description[IDENTIFIER_LENGTH]; /* processor description (type/official name) */
- u_longlong_t processorHZ; /* processor speed in Hz */
- u_longlong_t user; /* raw total number of clock ticks spent in user mode */
- u_longlong_t sys; /* raw total number of clock ticks spent in system mode */
- u_longlong_t idle; /* raw total number of clock ticks spent idle */
- u_longlong_t wait; /* raw total number of clock ticks spent waiting for I/O */
- u_longlong_t pswitch; /* number of process switches (change in currently running process) */
- u_longlong_t syscall; /* number of system calls executed */
- u_longlong_t sysread; /* number of read system calls executed */
- u_longlong_t syswrite; /* number of write system calls executed */
- u_longlong_t sysfork; /* number of forks system calls executed */
- u_longlong_t sysexec; /* number of execs system calls executed */
- u_longlong_t readch; /* number of characters transferred with read system call */
- u_longlong_t writech; /* number of characters transferred with write system call */
- u_longlong_t devintrs; /* number of device interrupts */
- u_longlong_t softintrs; /* number of software interrupts */
- time_t lbolt; /* number of ticks since last reboot */
- u_longlong_t loadavg[3]; /* (1<. */
- u_longlong_t runque; /* length of the run queue (processes ready) */
- u_longlong_t swpque; /* length of the swap queue (processes waiting to be paged in) */
- u_longlong_t bread; /* number of blocks read */
- u_longlong_t bwrite; /* number of blocks written */
- u_longlong_t lread; /* number of logical read requests */
- u_longlong_t lwrite; /* number of logical write requests */
- u_longlong_t phread; /* number of physical reads (reads on raw devices) */
- u_longlong_t phwrite; /* number of physical writes (writes on raw devices) */
- u_longlong_t runocc; /* updated whenever runque is updated, i.e. the runqueue is occupied.
- * This can be used to compute the simple average of ready processes */
- u_longlong_t swpocc; /* updated whenever swpque is updated. i.e. the swpqueue is occupied.
- * This can be used to compute the simple average processes waiting to be paged in */
- u_longlong_t iget; /* number of inode lookups */
- u_longlong_t namei; /* number of vnode lookup from a path name */
- u_longlong_t dirblk; /* number of 512-byte block reads by the directory search routine to locate an entry for a file */
- u_longlong_t msg; /* number of IPC message operations */
- u_longlong_t sema; /* number of IPC semaphore operations */
- u_longlong_t rcvint; /* number of tty receive interrupts */
- u_longlong_t xmtint; /* number of tyy transmit interrupts */
- u_longlong_t mdmint; /* number of modem interrupts */
- u_longlong_t tty_rawinch; /* number of raw input characters */
- u_longlong_t tty_caninch; /* number of canonical input characters (always zero) */
- u_longlong_t tty_rawoutch; /* number of raw output characters */
- u_longlong_t ksched; /* number of kernel processes created */
- u_longlong_t koverf; /* kernel process creation attempts where:
- * -the user has forked to their maximum limit
- * -the configuration limit of processes has been reached */
- u_longlong_t kexit; /* number of kernel processes that became zombies */
- u_longlong_t rbread; /* number of remote read requests */
- u_longlong_t rcread; /* number of cached remote reads */
- u_longlong_t rbwrt; /* number of remote writes */
- u_longlong_t rcwrt; /* number of cached remote writes */
- u_longlong_t traps; /* number of traps */
- int ncpus_high; /* index of highest processor online */
- u_longlong_t puser; /* raw number of physical processor tics in user mode */
- u_longlong_t psys; /* raw number of physical processor tics in system mode */
- u_longlong_t pidle; /* raw number of physical processor tics idle */
- u_longlong_t pwait; /* raw number of physical processor tics waiting for I/O */
- u_longlong_t decrintrs; /* number of decrementer tics interrupts */
- u_longlong_t mpcrintrs; /* number of mpc's received interrupts */
- u_longlong_t mpcsintrs; /* number of mpc's sent interrupts */
- u_longlong_t phantintrs; /* number of phantom interrupts */
- u_longlong_t idle_donated_purr; /* number of idle cycles donated by a dedicated partition enabled for donation */
- u_longlong_t idle_donated_spurr; /* number of idle spurr cycles donated by a dedicated partition enabled for donation */
- u_longlong_t busy_donated_purr; /* number of busy cycles donated by a dedicated partition enabled for donation */
- u_longlong_t busy_donated_spurr; /* number of busy spurr cycles donated by a dedicated partition enabled for donation */
- u_longlong_t idle_stolen_purr; /* number of idle cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t idle_stolen_spurr; /* number of idle spurr cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t busy_stolen_purr; /* number of busy cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t busy_stolen_spurr; /* number of busy spurr cycles stolen by the hypervisor from a dedicated partition */
- short iowait; /* number of processes that are asleep waiting for buffered I/O */
- short physio; /* number of processes waiting for raw I/O */
- longlong_t twait; /* number of threads that are waiting for filesystem direct(cio) */
- u_longlong_t hpi; /* number of hypervisor page-ins */
- u_longlong_t hpit; /* Time spent in hypervisor page-ins (in nanoseconds) */
- u_longlong_t puser_spurr; /* number of spurr cycles spent in user mode */
- u_longlong_t psys_spurr; /* number of spurr cycles spent in kernel mode */
- u_longlong_t pidle_spurr; /* number of spurr cycles spent in idle mode */
- u_longlong_t pwait_spurr; /* number of spurr cycles spent in wait mode */
- int spurrflag; /* set if running in spurr mode */
-} perfstat_cpu_total_t_61;
-
typedef struct { /* global cpu information AIX 7.1 */
int ncpus; /* number of active logical processors */
int ncpus_cfg; /* number of configured processors */
@@ -564,166 +393,6 @@ typedef union {
} b;
} perfstat_partition_type_t;
-typedef struct { /* partition total information AIX 5.3 < TL6 */
- char name[IDENTIFIER_LENGTH]; /* name of the logical partition */
- perfstat_partition_type_t type; /* set of bits describing the partition */
- int lpar_id; /* logical partition identifier */
- int group_id; /* identifier of the LPAR group this partition is a member of */
- int pool_id; /* identifier of the shared pool of physical processors this partition is a member of */
- int online_cpus; /* number of virtual CPUs currently online on the partition */
- int max_cpus; /* maximum number of virtual CPUs this partition can ever have */
- int min_cpus; /* minimum number of virtual CPUs this partition must have */
- u_longlong_t online_memory; /* amount of memory currently online */
- u_longlong_t max_memory; /* maximum amount of memory this partition can ever have */
- u_longlong_t min_memory; /* minimum amount of memory this partition must have */
- int entitled_proc_capacity; /* number of processor units this partition is entitled to receive */
- int max_proc_capacity; /* maximum number of processor units this partition can ever have */
- int min_proc_capacity; /* minimum number of processor units this partition must have */
- int proc_capacity_increment; /* increment value to the entitled capacity */
- int unalloc_proc_capacity; /* number of processor units currently unallocated in the shared processor pool this partition belongs to */
- int var_proc_capacity_weight; /* partition priority weight to receive extra capacity */
- int unalloc_var_proc_capacity_weight; /* number of variable processor capacity weight units currently unallocated in the shared processor pool this partition belongs to */
- int online_phys_cpus_sys; /* number of physical CPUs currently active in the system containing this partition */
- int max_phys_cpus_sys; /* maximum possible number of physical CPUs in the system containing this partition */
- int phys_cpus_pool; /* number of the physical CPUs currently in the shared processor pool this partition belong to */
- u_longlong_t puser; /* raw number of physical processor tics in user mode */
- u_longlong_t psys; /* raw number of physical processor tics in system mode */
- u_longlong_t pidle; /* raw number of physical processor tics idle */
- u_longlong_t pwait; /* raw number of physical processor tics waiting for I/O */
- u_longlong_t pool_idle_time; /* number of clock tics a processor in the shared pool was idle */
- u_longlong_t phantintrs; /* number of phantom interrupts received by the partition */
- u_longlong_t invol_virt_cswitch; /* number involuntary virtual CPU context switches */
- u_longlong_t vol_virt_cswitch; /* number voluntary virtual CPU context switches */
- u_longlong_t timebase_last; /* most recently cpu time base */
- u_longlong_t reserved_pages; /* Currently number of 16GB pages. Cannot participate in DR operations */
- u_longlong_t reserved_pagesize; /* Currently 16GB pagesize Cannot participate in DR operations */
-} perfstat_partition_total_t_53_5;
-
-typedef struct { /* partition total information AIX 5.3 < TL10 */
- char name[IDENTIFIER_LENGTH]; /* name of the logical partition */
- perfstat_partition_type_t type; /* set of bits describing the partition */
- int lpar_id; /* logical partition identifier */
- int group_id; /* identifier of the LPAR group this partition is a member of */
- int pool_id; /* identifier of the shared pool of physical processors this partition is a member of */
- int online_cpus; /* number of virtual CPUs currently online on the partition */
- int max_cpus; /* maximum number of virtual CPUs this partition can ever have */
- int min_cpus; /* minimum number of virtual CPUs this partition must have */
- u_longlong_t online_memory; /* amount of memory currently online */
- u_longlong_t max_memory; /* maximum amount of memory this partition can ever have */
- u_longlong_t min_memory; /* minimum amount of memory this partition must have */
- int entitled_proc_capacity; /* number of processor units this partition is entitled to receive */
- int max_proc_capacity; /* maximum number of processor units this partition can ever have */
- int min_proc_capacity; /* minimum number of processor units this partition must have */
- int proc_capacity_increment; /* increment value to the entitled capacity */
- int unalloc_proc_capacity; /* number of processor units currently unallocated in the shared processor pool this partition belongs to */
- int var_proc_capacity_weight; /* partition priority weight to receive extra capacity */
- int unalloc_var_proc_capacity_weight; /* number of variable processor capacity weight units currently unallocated in the shared processor pool this partition belongs to */
- int online_phys_cpus_sys; /* number of physical CPUs currently active in the system containing this partition */
- int max_phys_cpus_sys; /* maximum possible number of physical CPUs in the system containing this partition */
- int phys_cpus_pool; /* number of the physical CPUs currently in the shared processor pool this partition belong to */
- u_longlong_t puser; /* raw number of physical processor tics in user mode */
- u_longlong_t psys; /* raw number of physical processor tics in system mode */
- u_longlong_t pidle; /* raw number of physical processor tics idle */
- u_longlong_t pwait; /* raw number of physical processor tics waiting for I/O */
- u_longlong_t pool_idle_time; /* number of clock tics a processor in the shared pool was idle */
- u_longlong_t phantintrs; /* number of phantom interrupts received by the partition */
- u_longlong_t invol_virt_cswitch; /* number involuntary virtual CPU context switches */
- u_longlong_t vol_virt_cswitch; /* number voluntary virtual CPU context switches */
- u_longlong_t timebase_last; /* most recently cpu time base */
- u_longlong_t reserved_pages; /* Currently number of 16GB pages. Cannot participate in DR operations */
- u_longlong_t reserved_pagesize; /* Currently 16GB pagesize Cannot participate in DR operations */
- u_longlong_t idle_donated_purr; /* number of idle cycles donated by a dedicated partition enabled for donation */
- u_longlong_t idle_donated_spurr; /* number of idle spurr cycles donated by a dedicated partition enabled for donation */
- u_longlong_t busy_donated_purr; /* number of busy cycles donated by a dedicated partition enabled for donation */
- u_longlong_t busy_donated_spurr; /* number of busy spurr cycles donated by a dedicated partition enabled for donation */
- u_longlong_t idle_stolen_purr; /* number of idle cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t idle_stolen_spurr; /* number of idle spurr cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t busy_stolen_purr; /* number of busy cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t busy_stolen_spurr; /* number of busy spurr cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t shcpus_in_sys; /* Number of physical processors allocated for shared processor use */
- u_longlong_t max_pool_capacity; /* Maximum processor capacity of partitions pool */
- u_longlong_t entitled_pool_capacity; /* Entitled processor capacity of partitions pool */
- u_longlong_t pool_max_time; /* Summation of maximum time that could be consumed by the pool (nano seconds) */
- u_longlong_t pool_busy_time; /* Summation of busy (non-idle) time accumulated across all partitions in the pool (nano seconds) */
- u_longlong_t pool_scaled_busy_time; /* Scaled summation of busy (non-idle) time accumulated across all partitions in the pool (nano seconds) */
- u_longlong_t shcpu_tot_time; /* Summation of total time across all physical processors allocated for shared processor use (nano seconds) */
- u_longlong_t shcpu_busy_time; /* Summation of busy (non-idle) time accumulated across all shared processor partitions (nano seconds) */
- u_longlong_t shcpu_scaled_busy_time; /* Scaled summation of busy time accumulated across all shared processor partitions (nano seconds) */
- int ams_pool_id; /* AMS pool id of the pool the LPAR belongs to */
- int var_mem_weight; /* variable memory capacity weight */
- u_longlong_t iome; /* I/O memory entitlement of the partition in bytes*/
- u_longlong_t pmem; /* Physical memory currently backing the partition's logical memory in bytes*/
- u_longlong_t hpi; /* number of hypervisor page-ins */
- u_longlong_t hpit; /* Time spent in hypervisor page-ins (in nanoseconds)*/
- u_longlong_t hypv_pagesize; /* Hypervisor page size in KB*/
-} perfstat_partition_total_t_53;
-
-typedef struct { /* partition total information AIX 6.1|5.3 > TL09 */
- char name[IDENTIFIER_LENGTH]; /* name of the logical partition */
- perfstat_partition_type_t type; /* set of bits describing the partition */
- int lpar_id; /* logical partition identifier */
- int group_id; /* identifier of the LPAR group this partition is a member of */
- int pool_id; /* identifier of the shared pool of physical processors this partition is a member of */
- int online_cpus; /* number of virtual CPUs currently online on the partition */
- int max_cpus; /* maximum number of virtual CPUs this partition can ever have */
- int min_cpus; /* minimum number of virtual CPUs this partition must have */
- u_longlong_t online_memory; /* amount of memory currently online */
- u_longlong_t max_memory; /* maximum amount of memory this partition can ever have */
- u_longlong_t min_memory; /* minimum amount of memory this partition must have */
- int entitled_proc_capacity; /* number of processor units this partition is entitled to receive */
- int max_proc_capacity; /* maximum number of processor units this partition can ever have */
- int min_proc_capacity; /* minimum number of processor units this partition must have */
- int proc_capacity_increment; /* increment value to the entitled capacity */
- int unalloc_proc_capacity; /* number of processor units currently unallocated in the shared processor pool this partition belongs to */
- int var_proc_capacity_weight; /* partition priority weight to receive extra capacity */
- int unalloc_var_proc_capacity_weight; /* number of variable processor capacity weight units currently unallocated in the shared processor pool this partition belongs to */
- int online_phys_cpus_sys; /* number of physical CPUs currently active in the system containing this partition */
- int max_phys_cpus_sys; /* maximum possible number of physical CPUs in the system containing this partition */
- int phys_cpus_pool; /* number of the physical CPUs currently in the shared processor pool this partition belong to */
- u_longlong_t puser; /* raw number of physical processor tics in user mode */
- u_longlong_t psys; /* raw number of physical processor tics in system mode */
- u_longlong_t pidle; /* raw number of physical processor tics idle */
- u_longlong_t pwait; /* raw number of physical processor tics waiting for I/O */
- u_longlong_t pool_idle_time; /* number of clock tics a processor in the shared pool was idle */
- u_longlong_t phantintrs; /* number of phantom interrupts received by the partition */
- u_longlong_t invol_virt_cswitch; /* number involuntary virtual CPU context switches */
- u_longlong_t vol_virt_cswitch; /* number voluntary virtual CPU context switches */
- u_longlong_t timebase_last; /* most recently cpu time base */
- u_longlong_t reserved_pages; /* Currently number of 16GB pages. Cannot participate in DR operations */
- u_longlong_t reserved_pagesize; /* Currently 16GB pagesize Cannot participate in DR operations */
- u_longlong_t idle_donated_purr; /* number of idle cycles donated by a dedicated partition enabled for donation */
- u_longlong_t idle_donated_spurr; /* number of idle spurr cycles donated by a dedicated partition enabled for donation */
- u_longlong_t busy_donated_purr; /* number of busy cycles donated by a dedicated partition enabled for donation */
- u_longlong_t busy_donated_spurr; /* number of busy spurr cycles donated by a dedicated partition enabled for donation */
- u_longlong_t idle_stolen_purr; /* number of idle cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t idle_stolen_spurr; /* number of idle spurr cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t busy_stolen_purr; /* number of busy cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t busy_stolen_spurr; /* number of busy spurr cycles stolen by the hypervisor from a dedicated partition */
- u_longlong_t shcpus_in_sys; /* Number of physical processors allocated for shared processor use */
- u_longlong_t max_pool_capacity; /* Maximum processor capacity of partitions pool */
- u_longlong_t entitled_pool_capacity; /* Entitled processor capacity of partitions pool */
- u_longlong_t pool_max_time; /* Summation of maximum time that could be consumed by the pool (nano seconds) */
- u_longlong_t pool_busy_time; /* Summation of busy (non-idle) time accumulated across all partitions in the pool (nano seconds) */
- u_longlong_t pool_scaled_busy_time; /* Scaled summation of busy (non-idle) time accumulated across all partitions in the pool (nano seconds) */
- u_longlong_t shcpu_tot_time; /* Summation of total time across all physical processors allocated for shared processor use (nano seconds) */
- u_longlong_t shcpu_busy_time; /* Summation of busy (non-idle) time accumulated across all shared processor partitions (nano seconds) */
- u_longlong_t shcpu_scaled_busy_time; /* Scaled summation of busy time accumulated across all shared processor partitions (nano seconds) */
- int ams_pool_id; /* AMS pool id of the pool the LPAR belongs to */
- int var_mem_weight; /* variable memory capacity weight */
- u_longlong_t iome; /* I/O memory entitlement of the partition in bytes*/
- u_longlong_t pmem; /* Physical memory currently backing the partition's logical memory in bytes*/
- u_longlong_t hpi; /* number of hypervisor page-ins */
- u_longlong_t hpit; /* Time spent in hypervisor page-ins (in nanoseconds)*/
- u_longlong_t hypv_pagesize; /* Hypervisor page size in KB*/
- uint online_lcpus; /* number of online logical cpus */
- uint smt_thrds; /* number of hardware threads that are running */
- u_longlong_t puser_spurr; /* number of spurr cycles spent in user mode */
- u_longlong_t psys_spurr; /* number of spurr cycles spent in kernel mode */
- u_longlong_t pidle_spurr; /* number of spurr cycles spent in idle mode */
- u_longlong_t pwait_spurr; /* number of spurr cycles spent in wait mode */
- int spurrflag; /* set if running in spurr mode */
-} perfstat_partition_total_t_61;
-
typedef struct { /* partition total information AIX 7.1 */
char name[IDENTIFIER_LENGTH]; /* name of the logical partition */
perfstat_partition_type_t type; /* set of bits describing the partition */
@@ -941,17 +610,6 @@ typedef union { /* WPAR Type & Flags */
} b;
} perfstat_wpar_type_t;
-typedef struct { /* Workload partition Information AIX 5.3 & 6.1*/
- char name[MAXCORRALNAMELEN+1]; /* name of the Workload Partition */
- perfstat_wpar_type_t type; /* set of bits describing the wpar */
- cid_t wpar_id; /* workload partition identifier */
- uint online_cpus; /* Number of Virtual CPUs in partition rset or number of virtual CPUs currently online on the Global partition*/
- int cpu_limit; /* CPU limit in 100ths of % - 1..10000 */
- int mem_limit; /* Memory limit in 100ths of % - 1..10000 */
- u_longlong_t online_memory; /* amount of memory currently online in Global Partition */
- int entitled_proc_capacity; /* number of processor units this partition is entitled to receive */
-} perfstat_wpar_total_t_61;
-
typedef struct { /* Workload partition Information AIX 7.1*/
char name[MAXCORRALNAMELEN+1]; /* name of the Workload Partition */
perfstat_wpar_type_t type; /* set of bits describing the wpar */
@@ -983,7 +641,7 @@ typedef struct { /* WPAR identifier */
-// end: libperfstat.h (AIX 5.2, 5.3, 6.1, 7.1)
+// end: libperfstat.h (AIX 7.1)
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
#define PERFSTAT_PARTITON_TOTAL_T_LATEST perfstat_partition_total_t_71_1 /* latest perfstat_partition_total_t structure */
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index adaab8914a7fc..f6312f2f88206 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -29,7 +29,6 @@
// no precompiled headers
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
#include "interpreter/interpreter.hpp"
@@ -117,6 +116,10 @@
#include
#include
+#ifndef _LARGE_FILES
+#error Hotspot on AIX must be compiled with -D_LARGE_FILES
+#endif
+
// Missing prototypes for various system APIs.
extern "C"
int mread_real_time(timebasestruct_t *t, size_t size_of_timebasestruct_t);
@@ -273,6 +276,22 @@ julong os::Aix::available_memory() {
}
}
+jlong os::total_swap_space() {
+ perfstat_memory_total_t memory_info;
+ if (libperfstat::perfstat_memory_total(nullptr, &memory_info, sizeof(perfstat_memory_total_t), 1) == -1) {
+ return -1;
+ }
+ return (jlong)(memory_info.pgsp_total * 4 * K);
+}
+
+jlong os::free_swap_space() {
+ perfstat_memory_total_t memory_info;
+ if (libperfstat::perfstat_memory_total(nullptr, &memory_info, sizeof(perfstat_memory_total_t), 1) == -1) {
+ return -1;
+ }
+ return (jlong)(memory_info.pgsp_free * 4 * K);
+}
+
julong os::physical_memory() {
return Aix::physical_memory();
}
@@ -1108,10 +1127,9 @@ bool os::dll_address_to_library_name(address addr, char* buf,
return true;
}
-void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
+static void* dll_load_library(const char *filename, char *ebuf, int ebuflen) {
log_info(os)("attempting shared library load of %s", filename);
-
if (ebuf && ebuflen > 0) {
ebuf[0] = '\0';
ebuf[ebuflen - 1] = '\0';
@@ -1159,6 +1177,26 @@ void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
}
return nullptr;
}
+// Load library named
+// If filename matches .so, and loading fails, repeat with .a.
+void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
+ void* result = nullptr;
+ char* const file_path = strdup(filename);
+ char* const pointer_to_dot = strrchr(file_path, '.');
+ const char old_extension[] = ".so";
+ const char new_extension[] = ".a";
+ STATIC_ASSERT(sizeof(old_extension) >= sizeof(new_extension));
+ // First try to load the existing file.
+ result = dll_load_library(filename, ebuf, ebuflen);
+ // If the load fails,we try to reload by changing the extension to .a for .so files only.
+ // Shared object in .so format dont have braces, hence they get removed for archives with members.
+ if (result == nullptr && pointer_to_dot != nullptr && strcmp(pointer_to_dot, old_extension) == 0) {
+ snprintf(pointer_to_dot, sizeof(old_extension), "%s", new_extension);
+ result = dll_load_library(file_path, ebuf, ebuflen);
+ }
+ FREE_C_HEAP_ARRAY(char, file_path);
+ return result;
+}
void os::print_dll_info(outputStream *st) {
st->print_cr("Dynamic libraries:");
@@ -1584,7 +1622,7 @@ static char* reserve_shmated_memory (size_t bytes, char* requested_addr) {
// Now attach the shared segment.
// Note that we deliberately *don't* pass SHM_RND. The contract of os::attempt_reserve_memory_at() -
- // which invokes this function with a request address != NULL - is to map at the specified address
+ // which invokes this function with a request address != nullptr - is to map at the specified address
// excactly, or to fail. If the caller passed us an address that is not usable (aka not a valid segment
// boundary), shmat should not round down the address, or think up a completely new one.
// (In places where this matters, e.g. when reserving the heap, we take care of passing segment-aligned
@@ -1886,6 +1924,10 @@ void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) {
void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
}
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+ return page_size;
+}
+
void os::numa_make_global(char *addr, size_t bytes) {
}
@@ -2506,10 +2548,10 @@ int os::open(const char *path, int oflag, int mode) {
// IV90804: OPENING A FILE IN AFS WITH O_CLOEXEC FAILS WITH AN EINVAL ERROR APPLIES TO AIX 7100-04 17/04/14 PTF PECHANGE
int oflag_with_o_cloexec = oflag | O_CLOEXEC;
- int fd = ::open64(path, oflag_with_o_cloexec, mode);
+ int fd = ::open(path, oflag_with_o_cloexec, mode);
if (fd == -1) {
// we might fail in the open call when O_CLOEXEC is set, so try again without (see IV90804)
- fd = ::open64(path, oflag, mode);
+ fd = ::open(path, oflag, mode);
if (fd == -1) {
return -1;
}
@@ -2517,8 +2559,8 @@ int os::open(const char *path, int oflag, int mode) {
// If the open succeeded, the file might still be a directory.
{
- struct stat64 buf64;
- int ret = ::fstat64(fd, &buf64);
+ struct stat buf64;
+ int ret = ::fstat(fd, &buf64);
int st_mode = buf64.st_mode;
if (ret != -1) {
@@ -2572,67 +2614,17 @@ int os::open(const char *path, int oflag, int mode) {
int os::create_binary_file(const char* path, bool rewrite_existing) {
int oflags = O_WRONLY | O_CREAT;
oflags |= rewrite_existing ? O_TRUNC : O_EXCL;
- return ::open64(path, oflags, S_IREAD | S_IWRITE);
+ return ::open(path, oflags, S_IREAD | S_IWRITE);
}
// return current position of file pointer
jlong os::current_file_offset(int fd) {
- return (jlong)::lseek64(fd, (off64_t)0, SEEK_CUR);
+ return (jlong)::lseek(fd, (off_t)0, SEEK_CUR);
}
// move file pointer to the specified offset
jlong os::seek_to_file_offset(int fd, jlong offset) {
- return (jlong)::lseek64(fd, (off64_t)offset, SEEK_SET);
-}
-
-// Map a block of memory.
-char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- int prot;
- int flags = MAP_PRIVATE;
-
- if (read_only) {
- prot = PROT_READ;
- flags = MAP_SHARED;
- } else {
- prot = PROT_READ | PROT_WRITE;
- flags = MAP_PRIVATE;
- }
-
- if (allow_exec) {
- prot |= PROT_EXEC;
- }
-
- if (addr != nullptr) {
- flags |= MAP_FIXED;
- }
-
- // Allow anonymous mappings if 'fd' is -1.
- if (fd == -1) {
- flags |= MAP_ANONYMOUS;
- }
-
- char* mapped_address = (char*)::mmap(addr, (size_t)bytes, prot, flags,
- fd, file_offset);
- if (mapped_address == MAP_FAILED) {
- return nullptr;
- }
- return mapped_address;
-}
-
-// Remap a block of memory.
-char* os::pd_remap_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- // same as map_memory() on this OS
- return os::map_memory(fd, file_name, file_offset, addr, bytes, read_only,
- allow_exec);
-}
-
-// Unmap a block of memory.
-bool os::pd_unmap_memory(char* addr, size_t bytes) {
- return munmap(addr, bytes) == 0;
+ return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
}
// current_thread_cpu_time(bool) and thread_cpu_time(Thread*, bool)
@@ -3027,4 +3019,3 @@ void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) {}
void os::jfr_report_memory_info() {}
#endif // INCLUDE_JFR
-
diff --git a/src/hotspot/os/aix/os_perf_aix.cpp b/src/hotspot/os/aix/os_perf_aix.cpp
index e1719df48c331..b5ae1a6a725a5 100644
--- a/src/hotspot/os/aix/os_perf_aix.cpp
+++ b/src/hotspot/os/aix/os_perf_aix.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2022, IBM Corp.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, IBM Corp.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -87,6 +87,7 @@ static bool read_psinfo(const u_longlong_t& pid, psinfo_t& psinfo) {
}
len = fread(&psinfo, 1, sizeof(psinfo_t), fp);
+ fclose(fp);
return len == sizeof(psinfo_t);
}
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 958372e20a5ce..86d81a59f0d58 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,7 +24,6 @@
// no precompiled headers
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
#include "compiler/disassembler.hpp"
@@ -181,6 +180,32 @@ void os::Bsd::print_uptime_info(outputStream* st) {
}
}
+jlong os::total_swap_space() {
+#if defined(__APPLE__)
+ struct xsw_usage vmusage;
+ size_t size = sizeof(vmusage);
+ if (sysctlbyname("vm.swapusage", &vmusage, &size, nullptr, 0) != 0) {
+ return -1;
+ }
+ return (jlong)vmusage.xsu_total;
+#else
+ return -1;
+#endif
+}
+
+jlong os::free_swap_space() {
+#if defined(__APPLE__)
+ struct xsw_usage vmusage;
+ size_t size = sizeof(vmusage);
+ if (sysctlbyname("vm.swapusage", &vmusage, &size, nullptr, 0) != 0) {
+ return -1;
+ }
+ return (jlong)vmusage.xsu_avail;
+#else
+ return -1;
+#endif
+}
+
julong os::physical_memory() {
return Bsd::physical_memory();
}
@@ -985,6 +1010,13 @@ void *os::Bsd::dlopen_helper(const char *filename, int mode, char *ebuf, int ebu
if (!ieee_handling) {
Events::log_dll_message(nullptr, "IEEE subnormal handling check failed before loading %s", filename);
log_info(os)("IEEE subnormal handling check failed before loading %s", filename);
+ if (CheckJNICalls) {
+ tty->print_cr("WARNING: IEEE subnormal handling check failed before loading %s", filename);
+ Thread* current = Thread::current();
+ if (current->is_Java_thread()) {
+ JavaThread::cast(current)->print_jni_stack();
+ }
+ }
}
// Save and restore the floating-point environment around dlopen().
@@ -1038,6 +1070,13 @@ void *os::Bsd::dlopen_helper(const char *filename, int mode, char *ebuf, int ebu
} else {
Events::log_dll_message(nullptr, "IEEE subnormal handling could not be corrected after loading %s", filename);
log_info(os)("IEEE subnormal handling could not be corrected after loading %s", filename);
+ if (CheckJNICalls) {
+ tty->print_cr("WARNING: IEEE subnormal handling could not be corrected after loading %s", filename);
+ Thread* current = Thread::current();
+ if (current->is_Java_thread()) {
+ JavaThread::cast(current)->print_jni_stack();
+ }
+ }
assert(false, "fesetenv didn't work");
}
}
@@ -1229,7 +1268,8 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
}
#endif // !__APPLE__
-int _print_dll_info_cb(const char * name, address base_address, address top_address, void * param) {
+static int _print_dll_info_cb(const char * name, address base_address,
+ address top_address, void * param) {
outputStream * out = (outputStream *) param;
out->print_cr(INTPTR_FORMAT " \t%s", (intptr_t)base_address, name);
return 0;
@@ -1623,6 +1663,10 @@ void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
::madvise(addr, bytes, MADV_DONTNEED);
}
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+ return page_size;
+}
+
void os::numa_make_global(char *addr, size_t bytes) {
}
@@ -2301,53 +2345,6 @@ jlong os::seek_to_file_offset(int fd, jlong offset) {
return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
}
-// Map a block of memory.
-char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- int prot;
- int flags;
-
- if (read_only) {
- prot = PROT_READ;
- flags = MAP_SHARED;
- } else {
- prot = PROT_READ | PROT_WRITE;
- flags = MAP_PRIVATE;
- }
-
- if (allow_exec) {
- prot |= PROT_EXEC;
- }
-
- if (addr != nullptr) {
- flags |= MAP_FIXED;
- }
-
- char* mapped_address = (char*)mmap(addr, (size_t)bytes, prot, flags,
- fd, file_offset);
- if (mapped_address == MAP_FAILED) {
- return nullptr;
- }
- return mapped_address;
-}
-
-
-// Remap a block of memory.
-char* os::pd_remap_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- // same as map_memory() on this OS
- return os::map_memory(fd, file_name, file_offset, addr, bytes, read_only,
- allow_exec);
-}
-
-
-// Unmap a block of memory.
-bool os::pd_unmap_memory(char* addr, size_t bytes) {
- return munmap(addr, bytes) == 0;
-}
-
// current_thread_cpu_time(bool) and thread_cpu_time(Thread*, bool)
// are used by JVM M&M and JVMTI to get user+sys or user CPU time
// of a thread.
diff --git a/src/hotspot/os/linux/attachListener_linux.cpp b/src/hotspot/os/linux/attachListener_linux.cpp
index 41a24c450073f..0e98bca06078d 100644
--- a/src/hotspot/os/linux/attachListener_linux.cpp
+++ b/src/hotspot/os/linux/attachListener_linux.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -184,6 +184,8 @@ int LinuxAttachListener::init() {
char initial_path[UNIX_PATH_MAX]; // socket file during setup
int listener; // listener socket (file descriptor)
+ static_assert(sizeof(off_t) == 8, "Expected Large File Support in this file");
+
// register function to cleanup
if (!_atexit_registered) {
_atexit_registered = true;
@@ -446,14 +448,14 @@ AttachOperation* AttachListener::dequeue() {
void AttachListener::vm_start() {
char fn[UNIX_PATH_MAX];
- struct stat64 st;
+ struct stat st;
int ret;
int n = snprintf(fn, UNIX_PATH_MAX, "%s/.java_pid%d",
os::get_temp_directory(), os::current_process_id());
assert(n < (int)UNIX_PATH_MAX, "java_pid file name buffer overflow");
- RESTARTABLE(::stat64(fn, &st), ret);
+ RESTARTABLE(::stat(fn, &st), ret);
if (ret == 0) {
ret = ::unlink(fn);
if (ret == -1) {
@@ -473,8 +475,8 @@ int AttachListener::pd_init() {
bool AttachListener::check_socket_file() {
int ret;
- struct stat64 st;
- ret = stat64(LinuxAttachListener::path(), &st);
+ struct stat st;
+ ret = stat(LinuxAttachListener::path(), &st);
if (ret == -1) { // need to restart attach listener.
log_debug(attach)("Socket file %s does not exist - Restart Attach Listener",
LinuxAttachListener::path());
@@ -513,14 +515,14 @@ bool AttachListener::is_init_trigger() {
}
char fn[PATH_MAX + 1];
int ret;
- struct stat64 st;
+ struct stat st;
os::snprintf_checked(fn, sizeof(fn), ".attach_pid%d", os::current_process_id());
- RESTARTABLE(::stat64(fn, &st), ret);
+ RESTARTABLE(::stat(fn, &st), ret);
if (ret == -1) {
log_trace(attach)("Failed to find attach file: %s, trying alternate", fn);
snprintf(fn, sizeof(fn), "%s/.attach_pid%d",
os::get_temp_directory(), os::current_process_id());
- RESTARTABLE(::stat64(fn, &st), ret);
+ RESTARTABLE(::stat(fn, &st), ret);
if (ret == -1) {
log_debug(attach)("Failed to find attach file: %s", fn);
}
diff --git a/src/hotspot/os/linux/globals_linux.hpp b/src/hotspot/os/linux/globals_linux.hpp
index 577776e9d7e7b..8e4289c7ee3a2 100644
--- a/src/hotspot/os/linux/globals_linux.hpp
+++ b/src/hotspot/os/linux/globals_linux.hpp
@@ -92,7 +92,9 @@
develop(bool, DelayThreadStartALot, false, \
"Artificially delay thread starts randomly for testing.") \
\
-
+ product(bool, UseMadvPopulateWrite, true, DIAGNOSTIC, \
+ "Use MADV_POPULATE_WRITE in os::pd_pretouch_memory.") \
+ \
// end of RUNTIME_OS_FLAGS
diff --git a/src/hotspot/os/linux/hugepages.cpp b/src/hotspot/os/linux/hugepages.cpp
index 67645b91aa566..b71593487cf8a 100644
--- a/src/hotspot/os/linux/hugepages.cpp
+++ b/src/hotspot/os/linux/hugepages.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2023, Red Hat Inc. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,15 +36,15 @@
#include
-StaticHugePageSupport::StaticHugePageSupport() :
+ExplicitHugePageSupport::ExplicitHugePageSupport() :
_initialized(false), _pagesizes(), _default_hugepage_size(SIZE_MAX), _inconsistent(false) {}
-os::PageSizes StaticHugePageSupport::pagesizes() const {
+os::PageSizes ExplicitHugePageSupport::pagesizes() const {
assert(_initialized, "Not initialized");
return _pagesizes;
}
-size_t StaticHugePageSupport::default_hugepage_size() const {
+size_t ExplicitHugePageSupport::default_hugepage_size() const {
assert(_initialized, "Not initialized");
return _default_hugepage_size;
}
@@ -132,9 +132,9 @@ static os::PageSizes scan_hugepages() {
return pagesizes;
}
-void StaticHugePageSupport::print_on(outputStream* os) {
+void ExplicitHugePageSupport::print_on(outputStream* os) {
if (_initialized) {
- os->print_cr("Static hugepage support:");
+ os->print_cr("Explicit hugepage support:");
for (size_t s = _pagesizes.smallest(); s != 0; s = _pagesizes.next_larger(s)) {
os->print_cr(" hugepage size: " EXACTFMT, EXACTFMTARGS(s));
}
@@ -143,18 +143,18 @@ void StaticHugePageSupport::print_on(outputStream* os) {
os->print_cr(" unknown.");
}
if (_inconsistent) {
- os->print_cr(" Support inconsistent. JVM will not use static hugepages.");
+ os->print_cr(" Support inconsistent. JVM will not use explicit hugepages.");
}
}
-void StaticHugePageSupport::scan_os() {
+void ExplicitHugePageSupport::scan_os() {
_default_hugepage_size = scan_default_hugepagesize();
if (_default_hugepage_size > 0) {
_pagesizes = scan_hugepages();
// See https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt: /proc/meminfo should match
// /sys/kernel/mm/hugepages/hugepages-xxxx. However, we may run on a broken kernel (e.g. on WSL)
// that only exposes /proc/meminfo but not /sys/kernel/mm/hugepages. In that case, we are not
- // sure about the state of hugepage support by the kernel, so we won't use static hugepages.
+ // sure about the state of hugepage support by the kernel, so we won't use explicit hugepages.
if (!_pagesizes.contains(_default_hugepage_size)) {
log_info(pagesize)("Unexpected configuration: default pagesize (" SIZE_FORMAT ") "
"has no associated directory in /sys/kernel/mm/hugepages..", _default_hugepage_size);
@@ -307,18 +307,31 @@ void ShmemTHPSupport::print_on(outputStream* os) {
}
}
-StaticHugePageSupport HugePages::_static_hugepage_support;
+ExplicitHugePageSupport HugePages::_explicit_hugepage_support;
THPSupport HugePages::_thp_support;
ShmemTHPSupport HugePages::_shmem_thp_support;
+size_t HugePages::thp_pagesize_fallback() {
+ // Older kernels won't publish the THP page size. Fall back to default explicit huge page size,
+ // since that is likely to be the THP page size as well. Don't do it if the page size is considered
+ // too large to avoid large alignment waste. If explicit huge page size is unknown, use educated guess.
+ if (thp_pagesize() != 0) {
+ return thp_pagesize();
+ }
+ if (supports_explicit_hugepages()) {
+ return MIN2(default_explicit_hugepage_size(), 16 * M);
+ }
+ return 2 * M;
+}
+
void HugePages::initialize() {
- _static_hugepage_support.scan_os();
+ _explicit_hugepage_support.scan_os();
_thp_support.scan_os();
_shmem_thp_support.scan_os();
}
void HugePages::print_on(outputStream* os) {
- _static_hugepage_support.print_on(os);
+ _explicit_hugepage_support.print_on(os);
_thp_support.print_on(os);
_shmem_thp_support.print_on(os);
}
diff --git a/src/hotspot/os/linux/hugepages.hpp b/src/hotspot/os/linux/hugepages.hpp
index ce9ab36edccf8..efd27c55fd60f 100644
--- a/src/hotspot/os/linux/hugepages.hpp
+++ b/src/hotspot/os/linux/hugepages.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2023, Red Hat Inc. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,13 +34,13 @@ class outputStream;
// Header contains the interface that reads OS information about
// available hugepage support:
-// - class StaticHugePageSupport - about static (non-THP) hugepages
+// - class ExplicitHugePageSupport - about explicit (non-THP) hugepages
// - class THPSupport - about transparent huge pages
// and:
// - class HugePages - a static umbrella wrapper
// Information about static (non-thp) hugepages
-class StaticHugePageSupport {
+class ExplicitHugePageSupport {
bool _initialized;
// All supported hugepage sizes (sizes for which entries exist
@@ -56,7 +56,7 @@ class StaticHugePageSupport {
bool _inconsistent;
public:
- StaticHugePageSupport();
+ ExplicitHugePageSupport();
void scan_os();
@@ -122,22 +122,23 @@ class ShmemTHPSupport {
// Umbrella static interface
class HugePages : public AllStatic {
- static StaticHugePageSupport _static_hugepage_support;
+ static ExplicitHugePageSupport _explicit_hugepage_support;
static THPSupport _thp_support;
static ShmemTHPSupport _shmem_thp_support;
public:
- static const StaticHugePageSupport& static_info() { return _static_hugepage_support; }
+ static const ExplicitHugePageSupport& explicit_hugepage_info() { return _explicit_hugepage_support; }
static const THPSupport& thp_info() { return _thp_support; }
static const ShmemTHPSupport& shmem_thp_info() { return _shmem_thp_support; }
- static size_t default_static_hugepage_size() { return _static_hugepage_support.default_hugepage_size(); }
- static bool supports_static_hugepages() { return default_static_hugepage_size() > 0 && !_static_hugepage_support.inconsistent(); }
+ static size_t default_explicit_hugepage_size() { return _explicit_hugepage_support.default_hugepage_size(); }
+ static bool supports_explicit_hugepages() { return default_explicit_hugepage_size() > 0 && !_explicit_hugepage_support.inconsistent(); }
static bool supports_thp() { return thp_mode() == THPMode::madvise || thp_mode() == THPMode::always; }
static THPMode thp_mode() { return _thp_support.mode(); }
static size_t thp_pagesize() { return _thp_support.pagesize(); }
+ static size_t thp_pagesize_fallback();
static bool supports_shmem_thp() { return _shmem_thp_support.is_enabled(); }
static ShmemTHPMode shmem_thp_mode() { return _shmem_thp_support.mode(); }
diff --git a/src/hotspot/os/linux/osContainer_linux.cpp b/src/hotspot/os/linux/osContainer_linux.cpp
index 14bfa5a7678d4..cd723228a75e5 100644
--- a/src/hotspot/os/linux/osContainer_linux.cpp
+++ b/src/hotspot/os/linux/osContainer_linux.cpp
@@ -149,7 +149,7 @@ jlong OSContainer::pids_current() {
void OSContainer::print_container_helper(outputStream* st, jlong j, const char* metrics) {
st->print("%s: ", metrics);
- if (j > 0) {
+ if (j >= 0) {
if (j >= 1024) {
st->print_cr(UINT64_FORMAT " k", uint64_t(j) / K);
} else {
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 65574294b4099..119f98c687432 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -25,7 +25,6 @@
// no precompiled headers
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
#include "compiler/disassembler.hpp"
@@ -86,6 +85,8 @@
#endif
// put OS-includes here
+# include
+# include
# include
# include
# include
@@ -289,6 +290,34 @@ julong os::Linux::free_memory() {
return free_mem;
}
+jlong os::total_swap_space() {
+ if (OSContainer::is_containerized()) {
+ if (OSContainer::memory_limit_in_bytes() > 0) {
+ return (jlong)(OSContainer::memory_and_swap_limit_in_bytes() - OSContainer::memory_limit_in_bytes());
+ }
+ }
+ struct sysinfo si;
+ int ret = sysinfo(&si);
+ if (ret != 0) {
+ return -1;
+ }
+ return (jlong)(si.totalswap * si.mem_unit);
+}
+
+jlong os::free_swap_space() {
+ if (OSContainer::is_containerized()) {
+ // TODO add a good implementation
+ return -1;
+ } else {
+ struct sysinfo si;
+ int ret = sysinfo(&si);
+ if (ret != 0) {
+ return -1;
+ }
+ return (jlong)(si.freeswap * si.mem_unit);
+ }
+}
+
julong os::physical_memory() {
jlong phys_mem = 0;
if (OSContainer::is_containerized()) {
@@ -315,6 +344,29 @@ static void next_line(FILE *f) {
} while (c != '\n' && c != EOF);
}
+void os::Linux::kernel_version(long* major, long* minor) {
+ *major = -1;
+ *minor = -1;
+
+ struct utsname buffer;
+ int ret = uname(&buffer);
+ if (ret != 0) {
+ log_warning(os)("uname(2) failed to get kernel version: %s", os::errno_name(ret));
+ return;
+ }
+
+ char* walker = buffer.release;
+ long* set_v = major;
+ while (*minor == -1 && walker != nullptr) {
+ if (isdigit(walker[0])) {
+ *set_v = strtol(walker, &walker, 10);
+ set_v = minor;
+ } else {
+ ++walker;
+ }
+ }
+}
+
bool os::Linux::get_tick_information(CPUPerfTicks* pticks, int which_logical_cpu) {
FILE* fh;
uint64_t userTicks, niceTicks, systemTicks, idleTicks;
@@ -417,7 +469,7 @@ pid_t os::Linux::gettid() {
julong os::Linux::host_swap() {
struct sysinfo si;
sysinfo(&si);
- return (julong)si.totalswap;
+ return (julong)(si.totalswap * si.mem_unit);
}
// Most versions of linux have a bug where the number of processors are
@@ -1813,6 +1865,13 @@ void * os::Linux::dlopen_helper(const char *filename, char *ebuf, int ebuflen) {
if (!ieee_handling) {
Events::log_dll_message(nullptr, "IEEE subnormal handling check failed before loading %s", filename);
log_info(os)("IEEE subnormal handling check failed before loading %s", filename);
+ if (CheckJNICalls) {
+ tty->print_cr("WARNING: IEEE subnormal handling check failed before loading %s", filename);
+ Thread* current = Thread::current();
+ if (current->is_Java_thread()) {
+ JavaThread::cast(current)->print_jni_stack();
+ }
+ }
}
// Save and restore the floating-point environment around dlopen().
@@ -1867,6 +1926,13 @@ void * os::Linux::dlopen_helper(const char *filename, char *ebuf, int ebuflen) {
} else {
Events::log_dll_message(nullptr, "IEEE subnormal handling could not be corrected after loading %s", filename);
log_info(os)("IEEE subnormal handling could not be corrected after loading %s", filename);
+ if (CheckJNICalls) {
+ tty->print_cr("WARNING: IEEE subnormal handling could not be corrected after loading %s", filename);
+ Thread* current = Thread::current();
+ if (current->is_Java_thread()) {
+ JavaThread::cast(current)->print_jni_stack();
+ }
+ }
assert(false, "fesetenv didn't work");
}
}
@@ -2204,6 +2270,8 @@ void os::Linux::print_proc_sys_info(outputStream* st) {
"/proc/sys/kernel/threads-max", st);
_print_ascii_file_h("/proc/sys/vm/max_map_count (maximum number of memory map areas a process may have)",
"/proc/sys/vm/max_map_count", st);
+ _print_ascii_file_h("/proc/sys/vm/swappiness (control to define how aggressively the kernel swaps out anonymous memory)",
+ "/proc/sys/vm/swappiness", st);
_print_ascii_file_h("/proc/sys/kernel/pid_max (system-wide limit on number of process identifiers)",
"/proc/sys/kernel/pid_max", st);
}
@@ -2758,6 +2826,8 @@ void os::jvm_path(char *buf, jint buflen) {
void linux_wrap_code(char* base, size_t size) {
static volatile jint cnt = 0;
+ static_assert(sizeof(off_t) == 8, "Expected Large File Support in this file");
+
if (!UseOprofile) {
return;
}
@@ -2882,6 +2952,15 @@ void os::pd_commit_memory_or_exit(char* addr, size_t size, bool exec,
#define MADV_HUGEPAGE 14
#endif
+// Define MADV_POPULATE_WRITE here so we can build HotSpot on old systems.
+#define MADV_POPULATE_WRITE_value 23
+#ifndef MADV_POPULATE_WRITE
+ #define MADV_POPULATE_WRITE MADV_POPULATE_WRITE_value
+#else
+ // Sanity-check our assumed default value if we build with a new enough libc.
+ static_assert(MADV_POPULATE_WRITE == MADV_POPULATE_WRITE_value);
+#endif
+
// Note that the value for MAP_FIXED_NOREPLACE differs between architectures, but all architectures
// supported by OpenJDK share the same flag value.
#define MAP_FIXED_NOREPLACE_value 0x100000
@@ -2941,6 +3020,31 @@ void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
}
}
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+ const size_t len = pointer_delta(last, first, sizeof(char)) + page_size;
+ // Use madvise to pretouch on Linux when THP is used, and fallback to the
+ // common method if unsupported. THP can form right after madvise rather than
+ // being assembled later.
+ if (HugePages::thp_mode() == THPMode::always || UseTransparentHugePages) {
+ int err = 0;
+ if (UseMadvPopulateWrite &&
+ ::madvise(first, len, MADV_POPULATE_WRITE) == -1) {
+ err = errno;
+ }
+ if (!UseMadvPopulateWrite || err == EINVAL) { // Not to use or not supported
+ // When using THP we need to always pre-touch using small pages as the
+ // OS will initially always use small pages.
+ return os::vm_page_size();
+ } else if (err != 0) {
+ log_info(gc, os)("::madvise(" PTR_FORMAT ", " SIZE_FORMAT ", %d) failed; "
+ "error='%s' (errno=%d)", p2i(first), len,
+ MADV_POPULATE_WRITE, os::strerror(err), err);
+ }
+ return 0;
+ }
+ return page_size;
+}
+
void os::numa_make_global(char *addr, size_t bytes) {
Linux::numa_interleave_memory(addr, bytes);
}
@@ -3660,14 +3764,14 @@ bool os::unguard_memory(char* addr, size_t size) {
}
static int hugetlbfs_page_size_flag(size_t page_size) {
- if (page_size != HugePages::default_static_hugepage_size()) {
+ if (page_size != HugePages::default_explicit_hugepage_size()) {
return (exact_log2(page_size) << MAP_HUGE_SHIFT);
}
return 0;
}
static bool hugetlbfs_sanity_check(size_t page_size) {
- const os::PageSizes page_sizes = HugePages::static_info().pagesizes();
+ const os::PageSizes page_sizes = HugePages::explicit_hugepage_info().pagesizes();
assert(page_sizes.contains(page_size), "Invalid page sizes passed");
// Include the page size flag to ensure we sanity check the correct page size.
@@ -3847,8 +3951,8 @@ void os::Linux::large_page_init() {
return;
}
- // Check if the OS supports static hugepages.
- if (!UseTransparentHugePages && !HugePages::supports_static_hugepages()) {
+ // Check if the OS supports explicit hugepages.
+ if (!UseTransparentHugePages && !HugePages::supports_explicit_hugepages()) {
warn_no_large_pages_configured();
UseLargePages = false;
return;
@@ -3858,8 +3962,12 @@ void os::Linux::large_page_init() {
// In THP mode:
// - os::large_page_size() is the *THP page size*
// - os::pagesizes() has two members, the THP page size and the system page size
- assert(HugePages::thp_pagesize() > 0, "Missing OS info");
_large_page_size = HugePages::thp_pagesize();
+ if (_large_page_size == 0) {
+ log_info(pagesize) ("Cannot determine THP page size (kernel < 4.10 ?)");
+ _large_page_size = HugePages::thp_pagesize_fallback();
+ log_info(pagesize) ("Assuming THP page size to be: " EXACTFMT " (heuristics)", EXACTFMTARGS(_large_page_size));
+ }
_page_sizes.add(_large_page_size);
_page_sizes.add(os::vm_page_size());
// +UseTransparentHugePages implies +UseLargePages
@@ -3867,12 +3975,12 @@ void os::Linux::large_page_init() {
} else {
- // In static hugepage mode:
- // - os::large_page_size() is the default static hugepage size (/proc/meminfo "Hugepagesize")
+ // In explicit hugepage mode:
+ // - os::large_page_size() is the default explicit hugepage size (/proc/meminfo "Hugepagesize")
// - os::pagesizes() contains all hugepage sizes the kernel supports, regardless whether there
// are pages configured in the pool or not (from /sys/kernel/hugepages/hugepage-xxxx ...)
- os::PageSizes all_large_pages = HugePages::static_info().pagesizes();
- const size_t default_large_page_size = HugePages::default_static_hugepage_size();
+ os::PageSizes all_large_pages = HugePages::explicit_hugepage_info().pagesizes();
+ const size_t default_large_page_size = HugePages::default_explicit_hugepage_size();
// 3) Consistency check and post-processing
@@ -3956,7 +4064,7 @@ static bool commit_memory_special(size_t bytes,
char* req_addr,
bool exec) {
assert(UseLargePages, "Should only get here for huge pages");
- assert(!UseTransparentHugePages, "Should only get here for static hugepage mode");
+ assert(!UseTransparentHugePages, "Should only get here for explicit hugepage mode");
assert(is_aligned(bytes, page_size), "Unaligned size");
assert(is_aligned(req_addr, page_size), "Unaligned address");
assert(req_addr != nullptr, "Must have a requested address for special mappings");
@@ -3990,7 +4098,7 @@ static char* reserve_memory_special_huge_tlbfs(size_t bytes,
size_t page_size,
char* req_addr,
bool exec) {
- const os::PageSizes page_sizes = HugePages::static_info().pagesizes();
+ const os::PageSizes page_sizes = HugePages::explicit_hugepage_info().pagesizes();
assert(UseLargePages, "only for Huge TLBFS large pages");
assert(is_aligned(req_addr, alignment), "Must be");
assert(is_aligned(req_addr, page_size), "Must be");
@@ -4069,7 +4177,7 @@ size_t os::large_page_size() {
return _large_page_size;
}
-// static hugepages (hugetlbfs) allow application to commit large page memory
+// explicit hugepages (hugetlbfs) allow application to commit large page memory
// on demand.
// However, when committing memory with hugepages fails, the region
// that was supposed to be committed will lose the old reservation
@@ -4120,7 +4228,7 @@ char* os::pd_attempt_reserve_memory_at(char* requested_addr, size_t bytes, bool
size_t os::vm_min_address() {
// Determined by sysctl vm.mmap_min_addr. It exists as a safety zone to prevent
- // NULL pointer dereferences.
+ // null pointer dereferences.
// Most distros set this value to 64 KB. It *can* be zero, but rarely is. Here,
// we impose a minimum value if vm.mmap_min_addr is too low, for increased protection.
static size_t value = 0;
@@ -4248,7 +4356,7 @@ jlong os::Linux::fast_thread_cpu_time(clockid_t clockid) {
// the number of bytes written to out_fd is returned if transfer was successful
// otherwise, returns -1 that implies an error
jlong os::Linux::sendfile(int out_fd, int in_fd, jlong* offset, jlong count) {
- return ::sendfile64(out_fd, in_fd, (off64_t*)offset, (size_t)count);
+ return ::sendfile(out_fd, in_fd, (off_t*)offset, (size_t)count);
}
// Determine if the vmid is the parent pid for a child in a PID namespace.
@@ -4363,6 +4471,9 @@ void os::init(void) {
check_pax();
+ // Check the availability of MADV_POPULATE_WRITE.
+ FLAG_SET_DEFAULT(UseMadvPopulateWrite, (::madvise(0, 0, MADV_POPULATE_WRITE) == 0));
+
os::Posix::init();
}
@@ -4926,14 +5037,14 @@ int os::open(const char *path, int oflag, int mode) {
oflag |= O_CLOEXEC;
#endif
- int fd = ::open64(path, oflag, mode);
+ int fd = ::open(path, oflag, mode);
if (fd == -1) return -1;
//If the open succeeded, the file might still be a directory
{
- struct stat64 buf64;
- int ret = ::fstat64(fd, &buf64);
- int st_mode = buf64.st_mode;
+ struct stat buf;
+ int ret = ::fstat(fd, &buf);
+ int st_mode = buf.st_mode;
if (ret != -1) {
if ((st_mode & S_IFMT) == S_IFDIR) {
@@ -4970,62 +5081,17 @@ int os::open(const char *path, int oflag, int mode) {
int os::create_binary_file(const char* path, bool rewrite_existing) {
int oflags = O_WRONLY | O_CREAT;
oflags |= rewrite_existing ? O_TRUNC : O_EXCL;
- return ::open64(path, oflags, S_IREAD | S_IWRITE);
+ return ::open(path, oflags, S_IREAD | S_IWRITE);
}
// return current position of file pointer
jlong os::current_file_offset(int fd) {
- return (jlong)::lseek64(fd, (off64_t)0, SEEK_CUR);
+ return (jlong)::lseek(fd, (off_t)0, SEEK_CUR);
}
// move file pointer to the specified offset
jlong os::seek_to_file_offset(int fd, jlong offset) {
- return (jlong)::lseek64(fd, (off64_t)offset, SEEK_SET);
-}
-
-// Map a block of memory.
-char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- int prot;
- int flags = MAP_PRIVATE;
-
- if (read_only) {
- prot = PROT_READ;
- } else {
- prot = PROT_READ | PROT_WRITE;
- }
-
- if (allow_exec) {
- prot |= PROT_EXEC;
- }
-
- if (addr != nullptr) {
- flags |= MAP_FIXED;
- }
-
- char* mapped_address = (char*)mmap(addr, (size_t)bytes, prot, flags,
- fd, file_offset);
- if (mapped_address == MAP_FAILED) {
- return nullptr;
- }
- return mapped_address;
-}
-
-
-// Remap a block of memory.
-char* os::pd_remap_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- // same as map_memory() on this OS
- return os::map_memory(fd, file_name, file_offset, addr, bytes, read_only,
- allow_exec);
-}
-
-
-// Unmap a block of memory.
-bool os::pd_unmap_memory(char* addr, size_t bytes) {
- return munmap(addr, bytes) == 0;
+ return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
}
static jlong slow_thread_cpu_time(Thread *thread, bool user_sys_cpu_time);
diff --git a/src/hotspot/os/linux/os_linux.hpp b/src/hotspot/os/linux/os_linux.hpp
index 4b2ccf8e370db..6b902e8280244 100644
--- a/src/hotspot/os/linux/os_linux.hpp
+++ b/src/hotspot/os/linux/os_linux.hpp
@@ -93,6 +93,8 @@ class os::Linux {
bool has_steal_ticks;
};
+ static void kernel_version(long* major, long* minor);
+
// which_logical_cpu=-1 returns accumulated ticks for all cpus.
static bool get_tick_information(CPUPerfTicks* pticks, int which_logical_cpu);
static bool _stack_is_executable;
diff --git a/src/hotspot/os/linux/systemMemoryBarrier_linux.cpp b/src/hotspot/os/linux/systemMemoryBarrier_linux.cpp
index 446449a40e094..892d825b40cdc 100644
--- a/src/hotspot/os/linux/systemMemoryBarrier_linux.cpp
+++ b/src/hotspot/os/linux/systemMemoryBarrier_linux.cpp
@@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "logging/log.hpp"
-#include "runtime/os.hpp"
+#include "os_linux.hpp"
#include "utilities/debug.hpp"
#include "utilities/systemMemoryBarrier.hpp"
@@ -61,6 +61,18 @@ static long membarrier(int cmd, unsigned int flags, int cpu_id) {
}
bool LinuxSystemMemoryBarrier::initialize() {
+#if defined(RISCV)
+// RISCV port was introduced in kernel 4.4.
+// 4.4 also made membar private expedited mandatory.
+// But RISCV actually don't support it until 6.9.
+ long major, minor;
+ os::Linux::kernel_version(&major, &minor);
+ if (!(major > 6 || (major == 6 && minor >= 9))) {
+ log_info(os)("Linux kernel %ld.%ld does not support MEMBARRIER PRIVATE_EXPEDITED on RISC-V.",
+ major, minor);
+ return false;
+ }
+#endif
long ret = membarrier(MEMBARRIER_CMD_QUERY, 0, 0);
if (ret < 0) {
log_info(os)("MEMBARRIER_CMD_QUERY unsupported");
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index 1158392fa4964..39a6779b3fc04 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -265,7 +265,7 @@ bool os::dir_is_empty(const char* path) {
return result;
}
-static char* reserve_mmapped_memory(size_t bytes, char* requested_addr) {
+static char* reserve_mmapped_memory(size_t bytes, char* requested_addr, MEMFLAGS flag) {
char * addr;
int flags = MAP_PRIVATE NOT_AIX( | MAP_NORESERVE ) | MAP_ANONYMOUS;
if (requested_addr != nullptr) {
@@ -280,13 +280,14 @@ static char* reserve_mmapped_memory(size_t bytes, char* requested_addr) {
flags, -1, 0);
if (addr != MAP_FAILED) {
- MemTracker::record_virtual_memory_reserve((address)addr, bytes, CALLER_PC);
+ MemTracker::record_virtual_memory_reserve((address)addr, bytes, CALLER_PC, flag);
return addr;
}
return nullptr;
}
static int util_posix_fallocate(int fd, off_t offset, off_t len) {
+ static_assert(sizeof(off_t) == 8, "Expected Large File Support in this file");
#ifdef __APPLE__
fstore_t store = { F_ALLOCATECONTIG, F_PEOFPOSMODE, 0, len };
// First we try to get a continuous chunk of disk space
@@ -392,7 +393,7 @@ char* os::reserve_memory_aligned(size_t size, size_t alignment, bool exec) {
return chop_extra_memory(size, alignment, extra_base, extra_size);
}
-char* os::map_memory_to_file_aligned(size_t size, size_t alignment, int file_desc) {
+char* os::map_memory_to_file_aligned(size_t size, size_t alignment, int file_desc, MEMFLAGS flag) {
size_t extra_size = calculate_aligned_extra_size(size, alignment);
// For file mapping, we do not call os:map_memory_to_file(size,fd) since:
// - we later chop away parts of the mapping using os::release_memory and that could fail if the
@@ -400,7 +401,7 @@ char* os::map_memory_to_file_aligned(size_t size, size_t alignment, int file_des
// - The memory API os::reserve_memory uses is an implementation detail. It may (and usually is)
// mmap but it also may System V shared memory which cannot be uncommitted as a whole, so
// chopping off and unmapping excess bits back and front (see below) would not work.
- char* extra_base = reserve_mmapped_memory(extra_size, nullptr);
+ char* extra_base = reserve_mmapped_memory(extra_size, nullptr, flag);
if (extra_base == nullptr) {
return nullptr;
}
@@ -752,11 +753,11 @@ void os::dll_unload(void *lib) {
}
jlong os::lseek(int fd, jlong offset, int whence) {
- return (jlong) BSD_ONLY(::lseek) NOT_BSD(::lseek64)(fd, offset, whence);
+ return (jlong) ::lseek(fd, offset, whence);
}
int os::ftruncate(int fd, jlong length) {
- return BSD_ONLY(::ftruncate) NOT_BSD(::ftruncate64)(fd, length);
+ return ::ftruncate(fd, length);
}
const char* os::get_current_directory(char *buf, size_t buflen) {
@@ -2030,3 +2031,53 @@ void os::die() {
const char* os::file_separator() { return "/"; }
const char* os::line_separator() { return "\n"; }
const char* os::path_separator() { return ":"; }
+
+// Map file into memory; uses mmap().
+// Notes:
+// - if caller specifies addr, MAP_FIXED is used. That means existing
+// mappings will be replaced.
+// - The file descriptor must be valid (to create anonymous mappings, use
+// os::reserve_memory()).
+// Returns address to mapped memory, nullptr on error
+char* os::pd_map_memory(int fd, const char* unused,
+ size_t file_offset, char *addr, size_t bytes,
+ bool read_only, bool allow_exec) {
+
+ assert(fd != -1, "Specify a valid file descriptor");
+
+ int prot;
+ int flags = MAP_PRIVATE;
+
+ if (read_only) {
+ prot = PROT_READ;
+ } else {
+ prot = PROT_READ | PROT_WRITE;
+ }
+
+ if (allow_exec) {
+ prot |= PROT_EXEC;
+ }
+
+ if (addr != nullptr) {
+ flags |= MAP_FIXED;
+ }
+
+ char* mapped_address = (char*)mmap(addr, (size_t)bytes, prot, flags,
+ fd, file_offset);
+ if (mapped_address == MAP_FAILED) {
+ return nullptr;
+ }
+
+ // If we did specify an address, and the mapping succeeded, it should
+ // have returned that address since we specify MAP_FIXED
+ assert(addr == nullptr || addr == mapped_address,
+ "mmap+MAP_FIXED returned " PTR_FORMAT ", expected " PTR_FORMAT,
+ p2i(mapped_address), p2i(addr));
+
+ return mapped_address;
+}
+
+// Unmap a block of memory. Uses munmap.
+bool os::pd_unmap_memory(char* addr, size_t bytes) {
+ return munmap(addr, bytes) == 0;
+}
diff --git a/src/hotspot/os/posix/os_posix.hpp b/src/hotspot/os/posix/os_posix.hpp
index 8c71516f70b8f..d872a6dae899e 100644
--- a/src/hotspot/os/posix/os_posix.hpp
+++ b/src/hotspot/os/posix/os_posix.hpp
@@ -31,7 +31,7 @@
// Note: the Posix API aims to capture functionality available on all Posix
// compliant platforms, but in practice the implementations may depend on
-// non-Posix functionality. For example, the use of lseek64 and ftruncate64.
+// non-Posix functionality.
// This use of non-Posix API's is made possible by compiling/linking in a mode
// that is not restricted to being fully Posix complaint, such as by declaring
// -D_GNU_SOURCE. But be aware that in doing so we may enable non-Posix
diff --git a/src/hotspot/os/posix/perfMemory_posix.cpp b/src/hotspot/os/posix/perfMemory_posix.cpp
index 8736e05bc62ae..4339a21ae4e92 100644
--- a/src/hotspot/os/posix/perfMemory_posix.cpp
+++ b/src/hotspot/os/posix/perfMemory_posix.cpp
@@ -1086,11 +1086,10 @@ static char* mmap_create_shared(size_t size) {
static void unmap_shared(char* addr, size_t bytes) {
int res;
if (MemTracker::enabled()) {
- // Note: Tracker contains a ThreadCritical.
- Tracker tkr(Tracker::release);
+ ThreadCritical tc;
res = ::munmap(addr, bytes);
if (res == 0) {
- tkr.record((address)addr, bytes);
+ MemTracker::record_virtual_memory_release((address)addr, bytes);
}
} else {
res = ::munmap(addr, bytes);
diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp
index eaadb36731518..6a958f8903b8e 100644
--- a/src/hotspot/os/posix/signals_posix.cpp
+++ b/src/hotspot/os/posix/signals_posix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -340,7 +340,7 @@ static const struct {
////////////////////////////////////////////////////////////////////////////////
// sun.misc.Signal and BREAK_SIGNAL support
-void jdk_misc_signal_init() {
+static void jdk_misc_signal_init() {
// Initialize signal structures
::memset((void*)pending_signals, 0, sizeof(pending_signals));
@@ -380,7 +380,7 @@ int os::signal_wait() {
////////////////////////////////////////////////////////////////////////////////
// signal chaining support
-struct sigaction* get_chained_signal_action(int sig) {
+static struct sigaction* get_chained_signal_action(int sig) {
struct sigaction *actp = nullptr;
if (libjsig_is_loaded) {
@@ -1245,7 +1245,7 @@ int os::get_signal_number(const char* signal_name) {
return -1;
}
-void set_signal_handler(int sig) {
+static void set_signal_handler(int sig) {
// Check for overwrite.
struct sigaction oldAct;
sigaction(sig, (struct sigaction*)nullptr, &oldAct);
@@ -1292,7 +1292,7 @@ void set_signal_handler(int sig) {
// install signal handlers for signals that HotSpot needs to
// handle in order to support Java-level exception handling.
-void install_signal_handlers() {
+static void install_signal_handlers() {
// signal-chaining
typedef void (*signal_setting_t)();
signal_setting_t begin_signal_setting = nullptr;
@@ -1723,7 +1723,7 @@ static void SR_handler(int sig, siginfo_t* siginfo, void* context) {
errno = old_errno;
}
-int SR_initialize() {
+static int SR_initialize() {
struct sigaction act;
char *s;
// Get signal number to use for suspend/resume
diff --git a/src/hotspot/os/windows/attachListener_windows.cpp b/src/hotspot/os/windows/attachListener_windows.cpp
index da3b6c56739f0..7e455cf0a49f2 100644
--- a/src/hotspot/os/windows/attachListener_windows.cpp
+++ b/src/hotspot/os/windows/attachListener_windows.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -248,10 +248,13 @@ Win32AttachOperation* Win32AttachListener::dequeue() {
DWORD res = ::WaitForSingleObject(enqueued_ops_semaphore(), INFINITE);
// returning from WaitForSingleObject will have decreased
// the current count of the semaphore by 1.
- guarantee(res == WAIT_OBJECT_0, "wait failed");
+ guarantee(res != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ guarantee(res == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", res);
res = ::WaitForSingleObject(mutex(), INFINITE);
- guarantee(res == WAIT_OBJECT_0, "wait failed");
+ guarantee(res != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ guarantee(res == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", res);
+
Win32AttachOperation* op = head();
if (op != nullptr) {
@@ -338,6 +341,9 @@ void Win32AttachOperation::complete(jint result, bufferedStream* result_stream)
}
DWORD res = ::WaitForSingleObject(Win32AttachListener::mutex(), INFINITE);
+ assert(res != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ assert(res == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", res);
+
if (res == WAIT_OBJECT_0) {
// put the operation back on the available list
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 1b5c7850bf23f..2ddebc9bf6dee 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,6 @@
// no precompiled headers
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
@@ -840,6 +839,20 @@ julong os::win32::available_memory() {
return (julong)ms.ullAvailPhys;
}
+jlong os::total_swap_space() {
+ MEMORYSTATUSEX ms;
+ ms.dwLength = sizeof(ms);
+ GlobalMemoryStatusEx(&ms);
+ return (jlong) ms.ullTotalPageFile;
+}
+
+jlong os::free_swap_space() {
+ MEMORYSTATUSEX ms;
+ ms.dwLength = sizeof(ms);
+ GlobalMemoryStatusEx(&ms);
+ return (jlong) ms.ullAvailPageFile;
+}
+
julong os::physical_memory() {
return win32::physical_memory();
}
@@ -2469,9 +2482,9 @@ LONG Handle_IDiv_Exception(struct _EXCEPTION_POINTERS* exceptionInfo) {
#elif defined(_M_AMD64)
PCONTEXT ctx = exceptionInfo->ContextRecord;
address pc = (address)ctx->Rip;
- guarantee(pc[0] >= Assembler::REX && pc[0] <= Assembler::REX_WRXB && pc[1] == 0xF7 || pc[0] == 0xF7,
+ guarantee((pc[0] >= Assembler::REX && pc[0] <= Assembler::REX_WRXB && pc[1] == 0xF7) || pc[0] == 0xF7,
"not an idiv opcode, pc[0] = 0x%x and pc[1] = 0x%x", pc[0], pc[1]);
- guarantee(pc[0] >= Assembler::REX && pc[0] <= Assembler::REX_WRXB && (pc[2] & ~0x7) == 0xF8 || (pc[1] & ~0x7) == 0xF8,
+ guarantee((pc[0] >= Assembler::REX && pc[0] <= Assembler::REX_WRXB && (pc[2] & ~0x7) == 0xF8) || (pc[1] & ~0x7) == 0xF8,
"cannot handle non-register operands, pc[0] = 0x%x, pc[1] = 0x%x and pc[2] = 0x%x", pc[0], pc[1], pc[2]);
if (pc[0] == 0xF7) {
// set correct result values and continue after idiv instruction
@@ -3330,7 +3343,7 @@ char* os::replace_existing_mapping_with_file_mapping(char* base, size_t size, in
// Multiple threads can race in this code but it's not possible to unmap small sections of
// virtual space to get requested alignment, like posix-like os's.
// Windows prevents multiple thread from remapping over each other so this loop is thread-safe.
-static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int file_desc) {
+static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int file_desc, MEMFLAGS flag = mtNone) {
assert(is_aligned(alignment, os::vm_allocation_granularity()),
"Alignment must be a multiple of allocation granularity (page size)");
assert(is_aligned(size, os::vm_allocation_granularity()),
@@ -3343,8 +3356,8 @@ static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fi
static const int max_attempts = 20;
for (int attempt = 0; attempt < max_attempts && aligned_base == nullptr; attempt ++) {
- char* extra_base = file_desc != -1 ? os::map_memory_to_file(extra_size, file_desc) :
- os::reserve_memory(extra_size);
+ char* extra_base = file_desc != -1 ? os::map_memory_to_file(extra_size, file_desc, flag) :
+ os::reserve_memory(extra_size, false, flag);
if (extra_base == nullptr) {
return nullptr;
}
@@ -3360,8 +3373,8 @@ static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fi
// Attempt to map, into the just vacated space, the slightly smaller aligned area.
// Which may fail, hence the loop.
- aligned_base = file_desc != -1 ? os::attempt_map_memory_to_file_at(aligned_base, size, file_desc) :
- os::attempt_reserve_memory_at(aligned_base, size);
+ aligned_base = file_desc != -1 ? os::attempt_map_memory_to_file_at(aligned_base, size, file_desc, flag) :
+ os::attempt_reserve_memory_at(aligned_base, size, false, flag);
}
assert(aligned_base != nullptr, "Did not manage to re-map after %d attempts?", max_attempts);
@@ -3374,8 +3387,8 @@ char* os::reserve_memory_aligned(size_t size, size_t alignment, bool exec) {
return map_or_reserve_memory_aligned(size, alignment, -1 /* file_desc */);
}
-char* os::map_memory_to_file_aligned(size_t size, size_t alignment, int fd) {
- return map_or_reserve_memory_aligned(size, alignment, fd);
+char* os::map_memory_to_file_aligned(size_t size, size_t alignment, int fd, MEMFLAGS flag) {
+ return map_or_reserve_memory_aligned(size, alignment, fd, flag);
}
char* os::pd_reserve_memory(size_t bytes, bool exec) {
@@ -3796,6 +3809,11 @@ bool os::unguard_memory(char* addr, size_t bytes) {
void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) { }
void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) { }
+
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+ return page_size;
+}
+
void os::numa_make_global(char *addr, size_t bytes) { }
void os::numa_make_local(char *addr, size_t bytes, int lgrp_hint) { }
bool os::numa_topology_changed() { return false; }
@@ -5149,22 +5167,6 @@ char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
return base;
}
-
-// Remap a block of memory.
-char* os::pd_remap_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- // This OS does not allow existing memory maps to be remapped so we
- // would have to unmap the memory before we remap it.
-
- // Because there is a small window between unmapping memory and mapping
- // it in again with different protections, CDS archives are mapped RW
- // on windows, so this function isn't called.
- ShouldNotReachHere();
- return nullptr;
-}
-
-
// Unmap a block of memory.
// Returns true=success, otherwise false.
@@ -5354,7 +5356,8 @@ int PlatformEvent::park(jlong Millis) {
phri = new HighResolutionInterval(prd);
}
rv = ::WaitForSingleObject(_ParkHandle, prd);
- assert(rv == WAIT_OBJECT_0 || rv == WAIT_TIMEOUT, "WaitForSingleObject failed");
+ assert(rv != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ assert(rv == WAIT_OBJECT_0 || rv == WAIT_TIMEOUT, "WaitForSingleObject failed with return value: %lu", rv);
if (rv == WAIT_TIMEOUT) {
Millis -= prd;
}
@@ -5393,7 +5396,8 @@ void PlatformEvent::park() {
// spin attempts by this thread.
while (_Event < 0) {
DWORD rv = ::WaitForSingleObject(_ParkHandle, INFINITE);
- assert(rv == WAIT_OBJECT_0, "WaitForSingleObject failed");
+ assert(rv != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ assert(rv == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", rv);
}
// Usually we'll find _Event == 0 at this point, but as
@@ -5456,16 +5460,25 @@ void Parker::park(bool isAbsolute, jlong time) {
JavaThread* thread = JavaThread::current();
// Don't wait if interrupted or already triggered
- if (thread->is_interrupted(false) ||
- WaitForSingleObject(_ParkHandle, 0) == WAIT_OBJECT_0) {
+ if (thread->is_interrupted(false)) {
ResetEvent(_ParkHandle);
return;
} else {
- ThreadBlockInVM tbivm(thread);
- OSThreadWaitState osts(thread->osthread(), false /* not Object.wait() */);
+ DWORD rv = WaitForSingleObject(_ParkHandle, 0);
+ assert(rv != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ assert(rv == WAIT_OBJECT_0 || rv == WAIT_TIMEOUT, "WaitForSingleObject failed with return value: %lu", rv);
+ if (rv == WAIT_OBJECT_0) {
+ ResetEvent(_ParkHandle);
+ return;
+ } else {
+ ThreadBlockInVM tbivm(thread);
+ OSThreadWaitState osts(thread->osthread(), false /* not Object.wait() */);
- WaitForSingleObject(_ParkHandle, time);
- ResetEvent(_ParkHandle);
+ rv = WaitForSingleObject(_ParkHandle, time);
+ assert(rv != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ assert(rv == WAIT_OBJECT_0 || rv == WAIT_TIMEOUT, "WaitForSingleObject failed with return value: %lu", rv);
+ ResetEvent(_ParkHandle);
+ }
}
}
@@ -5541,7 +5554,9 @@ int os::fork_and_exec(const char* cmd) {
if (rslt) {
// Wait until child process exits.
- WaitForSingleObject(pi.hProcess, INFINITE);
+ DWORD rv = WaitForSingleObject(pi.hProcess, INFINITE);
+ assert(rv != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ assert(rv == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", rv);
GetExitCodeProcess(pi.hProcess, &exit_code);
diff --git a/src/hotspot/os/windows/perfMemory_windows.cpp b/src/hotspot/os/windows/perfMemory_windows.cpp
index 5a4f95d044fc8..a71101731fb98 100644
--- a/src/hotspot/os/windows/perfMemory_windows.cpp
+++ b/src/hotspot/os/windows/perfMemory_windows.cpp
@@ -1803,9 +1803,9 @@ void PerfMemory::detach(char* addr, size_t bytes) {
if (MemTracker::enabled()) {
// it does not go through os api, the operation has to record from here
- Tracker tkr(Tracker::release);
+ ThreadCritical tc;
remove_file_mapping(addr);
- tkr.record((address)addr, bytes);
+ MemTracker::record_virtual_memory_release((address)addr, bytes);
} else {
remove_file_mapping(addr);
}
diff --git a/src/hotspot/os/windows/threadCritical_windows.cpp b/src/hotspot/os/windows/threadCritical_windows.cpp
index f781a4224849b..c85143f80930d 100644
--- a/src/hotspot/os/windows/threadCritical_windows.cpp
+++ b/src/hotspot/os/windows/threadCritical_windows.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -64,7 +64,8 @@ ThreadCritical::ThreadCritical() {
if (lock_owner != current_thread) {
// Grab the lock before doing anything.
DWORD ret = WaitForSingleObject(lock_event, INFINITE);
- assert(ret == WAIT_OBJECT_0, "unexpected return value from WaitForSingleObject");
+ assert(ret != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
+ assert(ret == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", ret);
lock_owner = current_thread;
}
// Atomicity isn't required. Bump the recursion count.
diff --git a/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp
index aeed103c4deb3..a4d34d05df59e 100644
--- a/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp
@@ -37,7 +37,8 @@ frame JavaThread::pd_last_frame() {
intptr_t* sp = last_Java_sp();
address pc = _anchor.last_Java_pc();
- return frame(sp, pc);
+ // Likely the frame of a RuntimeStub.
+ return frame(sp, pc, frame::kind::code_blob);
}
bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, bool isInJava) {
@@ -50,7 +51,7 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
// pc can be seen as null because not all writers use store pc + release store sp.
// Simply discard the sample in this very rare case.
if (pc == nullptr) return false;
- *fr_addr = frame(sp, pc);
+ *fr_addr = frame(sp, pc, frame::kind::code_blob);
return true;
}
@@ -66,7 +67,8 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
return false;
}
- frame ret_frame((intptr_t*)uc->uc_mcontext.jmp_context.gpr[1/*REG_SP*/], pc);
+ // pc could refer to a native address outside the code cache even though the thread isInJava.
+ frame ret_frame((intptr_t*)uc->uc_mcontext.jmp_context.gpr[1/*REG_SP*/], pc, frame::kind::unknown);
if (ret_frame.fp() == nullptr) {
// The found frame does not have a valid frame pointer.
diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
index 69dc9dffd80f0..242042d4247aa 100644
--- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
@@ -28,7 +28,6 @@
#include "asm/assembler.inline.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
@@ -126,7 +125,7 @@ frame os::fetch_frame_from_context(const void* ucVoid) {
address epc = fetch_frame_from_context(ucVoid, &sp, &fp);
// Avoid crash during crash if pc broken.
if (epc) {
- frame fr(sp, epc);
+ frame fr(sp, epc, frame::kind::unknown);
return fr;
}
frame fr(sp);
@@ -137,21 +136,21 @@ frame os::fetch_compiled_frame_from_context(const void* ucVoid) {
const ucontext_t* uc = (const ucontext_t*)ucVoid;
intptr_t* sp = os::Aix::ucontext_get_sp(uc);
address lr = ucontext_get_lr(uc);
- return frame(sp, lr);
+ return frame(sp, lr, frame::kind::unknown);
}
frame os::get_sender_for_C_frame(frame* fr) {
if (*fr->sp() == (intptr_t) nullptr) {
// fr is the last C frame
- return frame(nullptr, nullptr);
+ return frame();
}
- return frame(fr->sender_sp(), fr->sender_pc());
+ return frame(fr->sender_sp(), fr->sender_pc(), frame::kind::unknown);
}
frame os::current_frame() {
intptr_t* csp = *(intptr_t**) __builtin_frame_address(0);
- frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame));
+ frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame), frame::kind::unknown);
return os::get_sender_for_C_frame(&topframe);
}
diff --git a/src/hotspot/os_cpu/bsd_aarch64/copy_bsd_aarch64.S b/src/hotspot/os_cpu/bsd_aarch64/copy_bsd_aarch64.S
index 7b286820a9a8f..187cd20ddbdad 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/copy_bsd_aarch64.S
+++ b/src/hotspot/os_cpu/bsd_aarch64/copy_bsd_aarch64.S
@@ -28,6 +28,11 @@
.global CFUNC(_Copy_conjoint_words)
.global CFUNC(_Copy_disjoint_words)
+#ifdef __APPLE__
+ .private_extern CFUNC(_Copy_conjoint_words)
+ .private_extern CFUNC(_Copy_disjoint_words)
+#endif
+
s .req x0
d .req x1
count .req x2
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index fbd7c4eccd403..4750ed8805644 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -29,7 +29,6 @@
#include "classfile/classLoader.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S b/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S
index 34d7b8e34a739..b9b6df9b23aa0 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S
+++ b/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,15 @@
.global SYMBOL(_SafeFetch32_fault)
.global SYMBOL(_SafeFetch32_continuation)
+#ifdef __APPLE__
+ .private_extern SYMBOL(SafeFetchN_impl)
+ .private_extern SYMBOL(_SafeFetchN_fault)
+ .private_extern SYMBOL(_SafeFetchN_continuation)
+ .private_extern SYMBOL(SafeFetch32_impl)
+ .private_extern SYMBOL(_SafeFetch32_fault)
+ .private_extern SYMBOL(_SafeFetch32_continuation)
+#endif
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# x0 : address
diff --git a/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S b/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S
index 02231040e15bd..5cad379df3f2b 100644
--- a/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S
+++ b/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2004, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#endif
.globl SYMBOL(fixcw)
+ .globl SYMBOL(SpinPause)
# NOTE WELL! The _Copy functions are called directly
# from server-compiler-generated code via CallLeafNoFP,
@@ -50,6 +51,20 @@
.globl SYMBOL(_Atomic_cmpxchg_long)
.globl SYMBOL(_Atomic_move_long)
+#ifdef __APPLE__
+ .private_extern SYMBOL(fixcw)
+ .private_extern SYMBOL(SpinPause)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_bytes)
+ .private_extern SYMBOL(_Copy_conjoint_jshorts_atomic)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jshorts)
+ .private_extern SYMBOL(_Copy_conjoint_jints_atomic)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jints)
+ .private_extern SYMBOL(_Copy_conjoint_jlongs_atomic)
+ .private_extern SYMBOL(_mmx_Copy_arrayof_conjoint_jshorts)
+ .private_extern SYMBOL(_Atomic_cmpxchg_long)
+ .private_extern SYMBOL(_Atomic_move_long)
+#endif
+
.text
# Support for void os::Solaris::init_thread_fpu_state() in os_solaris_i486.cpp
@@ -62,7 +77,6 @@ SYMBOL(fixcw):
popl %eax
ret
- .globl SYMBOL(SpinPause)
ELF_TYPE(SpinPause,@function)
.p2align 4,,15
SYMBOL(SpinPause):
diff --git a/src/hotspot/os_cpu/bsd_x86/bsd_x86_64.S b/src/hotspot/os_cpu/bsd_x86/bsd_x86_64.S
index 95cea3bf2a3d3..5e2addc4e6f43 100644
--- a/src/hotspot/os_cpu/bsd_x86/bsd_x86_64.S
+++ b/src/hotspot/os_cpu/bsd_x86/bsd_x86_64.S
@@ -1,5 +1,5 @@
-#
-# Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
+#
+# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -31,22 +31,33 @@
#endif
# NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
-
+ # from server-compiler-generated code via CallLeafNoFP,
+ # which means that they *must* either not use floating
+ # point or use it in the same manner as does the server
+ # compiler.
+
+ .globl SYMBOL(SpinPause)
.globl SYMBOL(_Copy_arrayof_conjoint_bytes)
- .globl SYMBOL(_Copy_arrayof_conjoint_jshorts)
+ .globl SYMBOL(_Copy_arrayof_conjoint_jshorts)
.globl SYMBOL(_Copy_conjoint_jshorts_atomic)
.globl SYMBOL(_Copy_arrayof_conjoint_jints)
.globl SYMBOL(_Copy_conjoint_jints_atomic)
.globl SYMBOL(_Copy_arrayof_conjoint_jlongs)
.globl SYMBOL(_Copy_conjoint_jlongs_atomic)
- .text
+#ifdef __APPLE__
+ .private_extern SYMBOL(SpinPause)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_bytes)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jshorts)
+ .private_extern SYMBOL(_Copy_conjoint_jshorts_atomic)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jints)
+ .private_extern SYMBOL(_Copy_conjoint_jints_atomic)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jlongs)
+ .private_extern SYMBOL(_Copy_conjoint_jlongs_atomic)
+#endif
+
+ .text
- .globl SYMBOL(SpinPause)
.p2align 4,,15
ELF_TYPE(SpinPause,@function)
SYMBOL(SpinPause):
@@ -63,7 +74,7 @@ SYMBOL(SpinPause):
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- ELF_TYPE(_Copy_arrayof_conjoint_bytes,@function)
+ ELF_TYPE(_Copy_arrayof_conjoint_bytes,@function)
SYMBOL(_Copy_arrayof_conjoint_bytes):
movq %rdx,%r8 # byte count
shrq $3,%rdx # qword count
@@ -71,7 +82,7 @@ SYMBOL(_Copy_arrayof_conjoint_bytes):
leaq -1(%rdi,%r8,1),%rax # from + bcount*1 - 1
jbe acb_CopyRight
cmpq %rax,%rsi
- jbe acb_CopyLeft
+ jbe acb_CopyLeft
acb_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -165,8 +176,8 @@ acb_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- ELF_TYPE(_Copy_arrayof_conjoint_jshorts,@function)
- ELF_TYPE(_Copy_conjoint_jshorts_atomic,@function)
+ ELF_TYPE(_Copy_arrayof_conjoint_jshorts,@function)
+ ELF_TYPE(_Copy_conjoint_jshorts_atomic,@function)
SYMBOL(_Copy_arrayof_conjoint_jshorts):
SYMBOL(_Copy_conjoint_jshorts_atomic):
movq %rdx,%r8 # word count
@@ -175,7 +186,7 @@ SYMBOL(_Copy_conjoint_jshorts_atomic):
leaq -2(%rdi,%r8,2),%rax # from + wcount*2 - 2
jbe acs_CopyRight
cmpq %rax,%rsi
- jbe acs_CopyLeft
+ jbe acs_CopyLeft
acs_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -255,8 +266,8 @@ acs_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- ELF_TYPE(_Copy_arrayof_conjoint_jints,@function)
- ELF_TYPE(_Copy_conjoint_jints_atomic,@function)
+ ELF_TYPE(_Copy_arrayof_conjoint_jints,@function)
+ ELF_TYPE(_Copy_conjoint_jints_atomic,@function)
SYMBOL(_Copy_arrayof_conjoint_jints):
SYMBOL(_Copy_conjoint_jints_atomic):
movq %rdx,%r8 # dword count
@@ -265,7 +276,7 @@ SYMBOL(_Copy_conjoint_jints_atomic):
leaq -4(%rdi,%r8,4),%rax # from + dcount*4 - 4
jbe aci_CopyRight
cmpq %rax,%rsi
- jbe aci_CopyLeft
+ jbe aci_CopyLeft
aci_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -334,15 +345,15 @@ aci_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- ELF_TYPE(_Copy_arrayof_conjoint_jlongs,@function)
- ELF_TYPE(_Copy_conjoint_jlongs_atomic,@function)
+ ELF_TYPE(_Copy_arrayof_conjoint_jlongs,@function)
+ ELF_TYPE(_Copy_conjoint_jlongs_atomic,@function)
SYMBOL(_Copy_arrayof_conjoint_jlongs):
SYMBOL(_Copy_conjoint_jlongs_atomic):
cmpq %rdi,%rsi
leaq -8(%rdi,%rdx,8),%rax # from + count*8 - 8
jbe acl_CopyRight
cmpq %rax,%rsi
- jbe acl_CopyLeft
+ jbe acl_CopyLeft
acl_CopyRight:
leaq -8(%rsi,%rdx,8),%rcx # to + count*8 - 8
negq %rdx
diff --git a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
index 37b92bc7ffd48..c73e83996ff57 100644
--- a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
+++ b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
@@ -351,7 +350,7 @@ frame os::get_sender_for_C_frame(frame* fr) {
return frame(fr->sender_sp(), fr->link(), fr->sender_pc());
}
-intptr_t* _get_previous_fp() {
+static intptr_t* _get_previous_fp() {
#if defined(__clang__) || defined(__llvm__)
intptr_t **ebp;
__asm__("mov %%" SPELL_REG_FP ", %0":"=r"(ebp));
diff --git a/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S b/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S
index 2a75f3dac94b3..1697f6f03b581 100644
--- a/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S
+++ b/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S
@@ -1,6 +1,6 @@
#
# Copyright (c) 2022 SAP SE. All rights reserved.
-# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -38,13 +38,22 @@
.globl SYMBOL(_SafeFetch32_continuation)
.globl SYMBOL(_SafeFetchN_continuation)
+#ifdef __APPLE__
+ .private_extern SYMBOL(SafeFetch32_impl)
+ .private_extern SYMBOL(SafeFetchN_impl)
+ .private_extern SYMBOL(_SafeFetch32_fault)
+ .private_extern SYMBOL(_SafeFetchN_fault)
+ .private_extern SYMBOL(_SafeFetch32_continuation)
+ .private_extern SYMBOL(_SafeFetchN_continuation)
+#endif
+
.text
# Support for int SafeFetch32(int* address, int defaultval);
#
# %rdi : address
# %esi : defaultval
- ELF_TYPE(SafeFetch32_impl,@function)
+ ELF_TYPE(SafeFetch32_impl,@function)
SYMBOL(SafeFetch32_impl:)
SYMBOL(_SafeFetch32_fault:)
movl (%rdi), %eax
diff --git a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
index 012f85ac0ff4a..0fc9484ce23ef 100644
--- a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
+++ b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
@@ -27,7 +27,6 @@
#include "asm/assembler.inline.hpp"
#include "atomic_bsd_zero.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
index 4621e44ca3c47..e67206a9d497f 100644
--- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
@@ -24,6 +24,7 @@
.text
.globl aarch64_atomic_fetch_add_8_default_impl
+ .hidden aarch64_atomic_fetch_add_8_default_impl
.align 5
aarch64_atomic_fetch_add_8_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -40,6 +41,7 @@ aarch64_atomic_fetch_add_8_default_impl:
ret
.globl aarch64_atomic_fetch_add_4_default_impl
+ .hidden aarch64_atomic_fetch_add_4_default_impl
.align 5
aarch64_atomic_fetch_add_4_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -56,6 +58,7 @@ aarch64_atomic_fetch_add_4_default_impl:
ret
.global aarch64_atomic_fetch_add_8_relaxed_default_impl
+ .hidden aarch64_atomic_fetch_add_8_relaxed_default_impl
.align 5
aarch64_atomic_fetch_add_8_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -71,6 +74,7 @@ aarch64_atomic_fetch_add_8_relaxed_default_impl:
ret
.global aarch64_atomic_fetch_add_4_relaxed_default_impl
+ .hidden aarch64_atomic_fetch_add_4_relaxed_default_impl
.align 5
aarch64_atomic_fetch_add_4_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -86,6 +90,7 @@ aarch64_atomic_fetch_add_4_relaxed_default_impl:
ret
.globl aarch64_atomic_xchg_4_default_impl
+ .hidden aarch64_atomic_xchg_4_default_impl
.align 5
aarch64_atomic_xchg_4_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -101,6 +106,7 @@ aarch64_atomic_xchg_4_default_impl:
ret
.globl aarch64_atomic_xchg_8_default_impl
+ .hidden aarch64_atomic_xchg_8_default_impl
.align 5
aarch64_atomic_xchg_8_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -116,6 +122,7 @@ aarch64_atomic_xchg_8_default_impl:
ret
.globl aarch64_atomic_cmpxchg_1_default_impl
+ .hidden aarch64_atomic_cmpxchg_1_default_impl
.align 5
aarch64_atomic_cmpxchg_1_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -136,6 +143,7 @@ aarch64_atomic_cmpxchg_1_default_impl:
ret
.globl aarch64_atomic_cmpxchg_4_default_impl
+ .hidden aarch64_atomic_cmpxchg_4_default_impl
.align 5
aarch64_atomic_cmpxchg_4_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -155,6 +163,7 @@ aarch64_atomic_cmpxchg_4_default_impl:
ret
.globl aarch64_atomic_cmpxchg_8_default_impl
+ .hidden aarch64_atomic_cmpxchg_8_default_impl
.align 5
aarch64_atomic_cmpxchg_8_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -174,6 +183,7 @@ aarch64_atomic_cmpxchg_8_default_impl:
ret
.globl aarch64_atomic_cmpxchg_4_release_default_impl
+ .hidden aarch64_atomic_cmpxchg_4_release_default_impl
.align 5
aarch64_atomic_cmpxchg_4_release_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -191,6 +201,7 @@ aarch64_atomic_cmpxchg_4_release_default_impl:
ret
.globl aarch64_atomic_cmpxchg_8_release_default_impl
+ .hidden aarch64_atomic_cmpxchg_8_release_default_impl
.align 5
aarch64_atomic_cmpxchg_8_release_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -208,6 +219,7 @@ aarch64_atomic_cmpxchg_8_release_default_impl:
ret
.globl aarch64_atomic_cmpxchg_4_seq_cst_default_impl
+ .hidden aarch64_atomic_cmpxchg_4_seq_cst_default_impl
.align 5
aarch64_atomic_cmpxchg_4_seq_cst_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -225,6 +237,7 @@ aarch64_atomic_cmpxchg_4_seq_cst_default_impl:
ret
.globl aarch64_atomic_cmpxchg_8_seq_cst_default_impl
+ .hidden aarch64_atomic_cmpxchg_8_seq_cst_default_impl
.align 5
aarch64_atomic_cmpxchg_8_seq_cst_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -242,6 +255,7 @@ aarch64_atomic_cmpxchg_8_seq_cst_default_impl:
ret
.globl aarch64_atomic_cmpxchg_1_relaxed_default_impl
+.hidden aarch64_atomic_cmpxchg_1_relaxed_default_impl
.align 5
aarch64_atomic_cmpxchg_1_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -260,6 +274,7 @@ aarch64_atomic_cmpxchg_1_relaxed_default_impl:
ret
.globl aarch64_atomic_cmpxchg_4_relaxed_default_impl
+ .hidden aarch64_atomic_cmpxchg_4_relaxed_default_impl
.align 5
aarch64_atomic_cmpxchg_4_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -277,6 +292,7 @@ aarch64_atomic_cmpxchg_4_relaxed_default_impl:
ret
.globl aarch64_atomic_cmpxchg_8_relaxed_default_impl
+ .hidden aarch64_atomic_cmpxchg_8_relaxed_default_impl
.align 5
aarch64_atomic_cmpxchg_8_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
diff --git a/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
index 4b8ed597c59fc..ade867ace016b 100644
--- a/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
@@ -24,6 +24,9 @@
.global _Copy_conjoint_words
.global _Copy_disjoint_words
+ .hidden _Copy_conjoint_words
+ .hidden _Copy_disjoint_words
+
s .req x0
d .req x1
count .req x2
diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
index 4835eb9405a1b..3698896abb78a 100644
--- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
@@ -27,7 +27,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "code/nativeInst.hpp"
#include "interpreter/interpreter.hpp"
diff --git a/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
index fcb7e62e6d5e9..cfbd8f45f285f 100644
--- a/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,13 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetchN_continuation
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# x0 : address
diff --git a/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
index ac60d6aa94168..f9f5aab2a6bd8 100644
--- a/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
@@ -25,22 +25,23 @@
// Clobber x1, flags.
// All other registers are preserved,
- .global _ZN10JavaThread25aarch64_get_thread_helperEv
- .type _ZN10JavaThread25aarch64_get_thread_helperEv, %function
+ .global _ZN10JavaThread25aarch64_get_thread_helperEv
+ .hidden _ZN10JavaThread25aarch64_get_thread_helperEv
+ .type _ZN10JavaThread25aarch64_get_thread_helperEv, %function
_ZN10JavaThread25aarch64_get_thread_helperEv:
- hint #0x19 // paciasp
- stp x29, x30, [sp, -16]!
- adrp x0, :tlsdesc:_ZN6Thread12_thr_currentE
- ldr x1, [x0, #:tlsdesc_lo12:_ZN6Thread12_thr_currentE]
- add x0, x0, :tlsdesc_lo12:_ZN6Thread12_thr_currentE
- .tlsdesccall _ZN6Thread12_thr_currentE
- blr x1
- mrs x1, tpidr_el0
- add x0, x1, x0
- ldr x0, [x0]
- ldp x29, x30, [sp], 16
- hint #0x1d // autiasp
- ret
+ hint #0x19 // paciasp
+ stp x29, x30, [sp, -16]!
+ adrp x0, :tlsdesc:_ZN6Thread12_thr_currentE
+ ldr x1, [x0, #:tlsdesc_lo12:_ZN6Thread12_thr_currentE]
+ add x0, x0, :tlsdesc_lo12:_ZN6Thread12_thr_currentE
+ .tlsdesccall _ZN6Thread12_thr_currentE
+ blr x1
+ mrs x1, tpidr_el0
+ add x0, x1, x0
+ ldr x0, [x0]
+ ldp x29, x30, [sp], 16
+ hint #0x1d // autiasp
+ ret
- .size _ZN10JavaThread25aarch64_get_thread_helperEv, .-_ZN10JavaThread25aarch64_get_thread_helperEv
+ .size _ZN10JavaThread25aarch64_get_thread_helperEv, .-_ZN10JavaThread25aarch64_get_thread_helperEv
diff --git a/src/hotspot/os_cpu/linux_arm/linux_arm_32.S b/src/hotspot/os_cpu/linux_arm/linux_arm_32.S
index eb560d8f0c78b..ad88c58ce78ce 100644
--- a/src/hotspot/os_cpu/linux_arm/linux_arm_32.S
+++ b/src/hotspot/os_cpu/linux_arm/linux_arm_32.S
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -23,38 +23,46 @@
# NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
+ # from server-compiler-generated code via CallLeafNoFP,
+ # which means that they *must* either not use floating
+ # point or use it in the same manner as does the server
+ # compiler.
- .globl _Copy_conjoint_bytes
- .type _Copy_conjoint_bytes, %function
+ .globl SpinPause
+ .hidden SpinPause
+ .type SpinPause, %function
.globl _Copy_arrayof_conjoint_bytes
- .type _Copy_arrayof_conjoint_bytes, %function
- .globl _Copy_disjoint_words
- .type _Copy_disjoint_words, %function
- .globl _Copy_conjoint_words
- .type _Copy_conjoint_words, %function
+ .hidden _Copy_arrayof_conjoint_bytes
+ .type _Copy_arrayof_conjoint_bytes, %function
+ .globl _Copy_disjoint_words
+ .hidden _Copy_disjoint_words
+ .type _Copy_disjoint_words, %function
+ .globl _Copy_conjoint_words
+ .hidden _Copy_conjoint_words
+ .type _Copy_conjoint_words, %function
.globl _Copy_conjoint_jshorts_atomic
- .type _Copy_conjoint_jshorts_atomic, %function
- .globl _Copy_arrayof_conjoint_jshorts
- .type _Copy_arrayof_conjoint_jshorts, %function
+ .hidden _Copy_conjoint_jshorts_atomic
+ .type _Copy_conjoint_jshorts_atomic, %function
+ .globl _Copy_arrayof_conjoint_jshorts
+ .hidden _Copy_arrayof_conjoint_jshorts
+ .type _Copy_arrayof_conjoint_jshorts, %function
.globl _Copy_conjoint_jints_atomic
- .type _Copy_conjoint_jints_atomic, %function
+ .hidden _Copy_conjoint_jints_atomic
+ .type _Copy_conjoint_jints_atomic, %function
.globl _Copy_arrayof_conjoint_jints
- .type _Copy_arrayof_conjoint_jints, %function
- .globl _Copy_conjoint_jlongs_atomic
- .type _Copy_conjoint_jlongs_atomic, %function
- .globl _Copy_arrayof_conjoint_jlongs
- .type _Copy_arrayof_conjoint_jlongs, %function
+ .hidden _Copy_arrayof_conjoint_jints
+ .type _Copy_arrayof_conjoint_jints, %function
+ .globl _Copy_conjoint_jlongs_atomic
+ .hidden _Copy_conjoint_jlongs_atomic
+ .type _Copy_conjoint_jlongs_atomic, %function
+ .globl _Copy_arrayof_conjoint_jlongs
+ .hidden _Copy_arrayof_conjoint_jlongs
+ .type _Copy_arrayof_conjoint_jlongs, %function
from .req r0
to .req r1
- .text
- .globl SpinPause
- .type SpinPause, %function
+ .text
SpinPause:
bx LR
@@ -70,7 +78,7 @@ _Copy_arrayof_conjoint_bytes:
# size_t count)
_Copy_disjoint_words:
stmdb sp!, {r3 - r9, ip}
-
+
cmp r2, #0
beq disjoint_words_finish
@@ -81,17 +89,17 @@ _Copy_disjoint_words:
.align 3
dw_f2b_loop_32:
subs r2, #32
- blt dw_f2b_loop_32_finish
+ blt dw_f2b_loop_32_finish
ldmia from!, {r3 - r9, ip}
nop
- pld [from]
+ pld [from]
stmia to!, {r3 - r9, ip}
bgt dw_f2b_loop_32
dw_f2b_loop_32_finish:
addlts r2, #32
beq disjoint_words_finish
cmp r2, #16
- blt disjoint_words_small
+ blt disjoint_words_small
ldmia from!, {r3 - r6}
subge r2, r2, #16
stmia to!, {r3 - r6}
@@ -116,8 +124,8 @@ disjoint_words_finish:
_Copy_conjoint_words:
stmdb sp!, {r3 - r9, ip}
- cmp r2, #0
- beq conjoint_words_finish
+ cmp r2, #0
+ beq conjoint_words_finish
pld [from, #0]
cmp r2, #12
@@ -129,17 +137,17 @@ _Copy_conjoint_words:
.align 3
cw_f2b_loop_32:
subs r2, #32
- blt cw_f2b_loop_32_finish
+ blt cw_f2b_loop_32_finish
ldmia from!, {r3 - r9, ip}
nop
- pld [from]
+ pld [from]
stmia to!, {r3 - r9, ip}
bgt cw_f2b_loop_32
cw_f2b_loop_32_finish:
addlts r2, #32
beq conjoint_words_finish
cmp r2, #16
- blt conjoint_words_small
+ blt conjoint_words_small
ldmia from!, {r3 - r6}
subge r2, r2, #16
stmia to!, {r3 - r6}
@@ -154,7 +162,7 @@ conjoint_words_small:
strgt r9, [to], #4
b conjoint_words_finish
- # Src and dest overlap, copy in a descending order
+ # Src and dest overlap, copy in a descending order
cw_b2f_copy:
add from, r2
pld [from, #-32]
@@ -162,17 +170,17 @@ cw_b2f_copy:
.align 3
cw_b2f_loop_32:
subs r2, #32
- blt cw_b2f_loop_32_finish
+ blt cw_b2f_loop_32_finish
ldmdb from!, {r3-r9,ip}
nop
- pld [from, #-32]
+ pld [from, #-32]
stmdb to!, {r3-r9,ip}
bgt cw_b2f_loop_32
cw_b2f_loop_32_finish:
addlts r2, #32
beq conjoint_words_finish
cmp r2, #16
- blt cw_b2f_copy_small
+ blt cw_b2f_copy_small
ldmdb from!, {r3 - r6}
subge r2, r2, #16
stmdb to!, {r3 - r6}
@@ -196,8 +204,8 @@ conjoint_words_finish:
_Copy_conjoint_jshorts_atomic:
stmdb sp!, {r3 - r9, ip}
- cmp r2, #0
- beq conjoint_shorts_finish
+ cmp r2, #0
+ beq conjoint_shorts_finish
subs r3, to, from
cmphi r2, r3
@@ -210,11 +218,11 @@ _Copy_conjoint_jshorts_atomic:
ands r3, from, #3
bne cs_f2b_src_u
- # Aligned source address
+ # Aligned source address
.align 3
cs_f2b_loop_32:
subs r2, #32
- blt cs_f2b_loop_32_finish
+ blt cs_f2b_loop_32_finish
ldmia from!, {r3 - r9, ip}
nop
pld [from]
@@ -244,14 +252,14 @@ cs_f2b_4:
strgth r5, [to], #2
b conjoint_shorts_finish
- # Destination not aligned
+ # Destination not aligned
cs_f2b_dest_u:
ldrh r3, [from], #2
subs r2, #2
strh r3, [to], #2
beq conjoint_shorts_finish
- # Check to see if source is not aligned ether
+ # Check to see if source is not aligned ether
ands r3, from, #3
beq cs_f2b_loop_32
@@ -259,11 +267,11 @@ cs_f2b_src_u:
cmp r2, #16
blt cs_f2b_8_u
- # Load 2 first bytes to r7 and make src ptr word aligned
+ # Load 2 first bytes to r7 and make src ptr word aligned
bic from, #3
ldr r7, [from], #4
- # Destination aligned, source not
+ # Destination aligned, source not
mov r8, r2, lsr #4
.align 3
cs_f2b_16_u_loop:
@@ -306,7 +314,7 @@ cs_f2b_4_u:
strgth r5, [to], #2
b conjoint_shorts_finish
- # Src and dest overlap, copy in a descending order
+ # Src and dest overlap, copy in a descending order
cs_b2f_copy:
add from, r2
pld [from, #-32]
@@ -319,7 +327,7 @@ cs_b2f_copy:
.align 3
cs_b2f_loop_32:
subs r2, #32
- blt cs_b2f_loop_32_finish
+ blt cs_b2f_loop_32_finish
ldmdb from!, {r3-r9,ip}
nop
pld [from, #-32]
@@ -359,16 +367,16 @@ cs_b2f_all_copy:
strgth r5, [to, #-2]!
b conjoint_shorts_finish
- # Destination not aligned
+ # Destination not aligned
cs_b2f_dest_u:
ldrh r3, [from, #-2]!
strh r3, [to, #-2]!
sub r2, #2
- # Check source alignment as well
+ # Check source alignment as well
ands r3, from, #3
beq cs_b2f_loop_32
- # Source not aligned
+ # Source not aligned
cs_b2f_src_u:
bic from, #3
.align 3
@@ -393,7 +401,7 @@ cs_b2f_16_loop_u:
cs_b2f_16_loop_u_finished:
addlts r2, #16
ldr r3, [from]
- cmp r2, #10
+ cmp r2, #10
blt cs_b2f_2_u_loop
ldmdb from!, {r4 - r5}
mov r6, r4, lsr #16
@@ -402,7 +410,7 @@ cs_b2f_16_loop_u_finished:
orr r7, r7, r3, lsl #16
stmdb to!, {r6-r7}
sub r2, #8
- .align 3
+ .align 3
cs_b2f_2_u_loop:
subs r2, #2
ldrh r3, [from], #-2
@@ -426,7 +434,7 @@ _Copy_arrayof_conjoint_jshorts:
_Copy_conjoint_jints_atomic:
_Copy_arrayof_conjoint_jints:
swi 0x9f0001
-
+
# Support for void Copy::conjoint_jlongs_atomic(jlong* from,
# jlong* to,
# size_t count)
@@ -434,8 +442,8 @@ _Copy_conjoint_jlongs_atomic:
_Copy_arrayof_conjoint_jlongs:
stmdb sp!, {r3 - r9, ip}
- cmp r2, #0
- beq conjoint_longs_finish
+ cmp r2, #0
+ beq conjoint_longs_finish
pld [from, #0]
cmp r2, #24
@@ -447,10 +455,10 @@ _Copy_arrayof_conjoint_jlongs:
.align 3
cl_f2b_loop_32:
subs r2, #32
- blt cl_f2b_loop_32_finish
+ blt cl_f2b_loop_32_finish
ldmia from!, {r3 - r9, ip}
nop
- pld [from]
+ pld [from]
stmia to!, {r3 - r9, ip}
bgt cl_f2b_loop_32
cl_f2b_loop_32_finish:
@@ -458,21 +466,21 @@ cl_f2b_loop_32_finish:
beq conjoint_longs_finish
conjoint_longs_small:
cmp r2, #16
- blt cl_f2b_copy_8
- bgt cl_f2b_copy_24
+ blt cl_f2b_copy_8
+ bgt cl_f2b_copy_24
ldmia from!, {r3 - r6}
stmia to!, {r3 - r6}
- b conjoint_longs_finish
+ b conjoint_longs_finish
cl_f2b_copy_8:
ldmia from!, {r3 - r4}
stmia to!, {r3 - r4}
b conjoint_longs_finish
cl_f2b_copy_24:
- ldmia from!, {r3 - r8}
+ ldmia from!, {r3 - r8}
stmia to!, {r3 - r8}
b conjoint_longs_finish
- # Src and dest overlap, copy in a descending order
+ # Src and dest overlap, copy in a descending order
cl_b2f_copy:
add from, r2
pld [from, #-32]
@@ -480,31 +488,29 @@ cl_b2f_copy:
.align 3
cl_b2f_loop_32:
subs r2, #32
- blt cl_b2f_loop_32_finish
+ blt cl_b2f_loop_32_finish
ldmdb from!, {r3 - r9, ip}
nop
- pld [from]
+ pld [from]
stmdb to!, {r3 - r9, ip}
bgt cl_b2f_loop_32
cl_b2f_loop_32_finish:
addlts r2, #32
beq conjoint_longs_finish
cmp r2, #16
- blt cl_b2f_copy_8
- bgt cl_b2f_copy_24
+ blt cl_b2f_copy_8
+ bgt cl_b2f_copy_24
ldmdb from!, {r3 - r6}
stmdb to!, {r3 - r6}
b conjoint_longs_finish
cl_b2f_copy_8:
- ldmdb from!, {r3 - r4}
+ ldmdb from!, {r3 - r4}
stmdb to!, {r3 - r4}
b conjoint_longs_finish
cl_b2f_copy_24:
- ldmdb from!, {r3 - r8}
+ ldmdb from!, {r3 - r8}
stmdb to!, {r3 - r8}
conjoint_longs_finish:
ldmia sp!, {r3 - r9, ip}
bx lr
-
-
diff --git a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
index 86e8ed25618c1..551270588438e 100644
--- a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
+++ b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
@@ -25,7 +25,6 @@
// no precompiled headers
#include "asm/assembler.inline.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S b/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S
index 5196b199f05f6..07e90fa3079f2 100644
--- a/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S
+++ b/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,11 @@
.globl SafeFetch32_impl
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
.type SafeFetch32_impl, %function
# Support for int SafeFetch32(int* address, int defaultval);
diff --git a/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp
index d16788222a142..191b7315205f7 100644
--- a/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp
+++ b/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp
@@ -36,7 +36,8 @@ frame JavaThread::pd_last_frame() {
intptr_t* sp = last_Java_sp();
address pc = _anchor.last_Java_pc();
- return frame(sp, pc);
+ // Likely the frame of a RuntimeStub.
+ return frame(sp, pc, frame::kind::code_blob);
}
bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, bool isInJava) {
@@ -49,7 +50,7 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
// pc can be seen as null because not all writers use store pc + release store sp.
// Simply discard the sample in this very rare case.
if (pc == nullptr) return false;
- *fr_addr = frame(sp, pc);
+ *fr_addr = frame(sp, pc, frame::kind::code_blob);
return true;
}
@@ -65,7 +66,8 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
return false;
}
- frame ret_frame((intptr_t*)uc->uc_mcontext.regs->gpr[1/*REG_SP*/], pc);
+ // pc could refer to a native address outside the code cache even though the thread isInJava.
+ frame ret_frame((intptr_t*)uc->uc_mcontext.regs->gpr[1/*REG_SP*/], pc, frame::kind::unknown);
if (ret_frame.fp() == nullptr) {
// The found frame does not have a valid frame pointer.
diff --git a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
index 109f2b98d645c..0b666f29c312b 100644
--- a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
+++ b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
@@ -28,7 +28,6 @@
#include "asm/assembler.inline.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
@@ -156,28 +155,28 @@ frame os::fetch_frame_from_context(const void* ucVoid) {
intptr_t* sp;
intptr_t* fp;
address epc = fetch_frame_from_context(ucVoid, &sp, &fp);
- return frame(sp, epc);
+ return frame(sp, epc, frame::kind::unknown);
}
frame os::fetch_compiled_frame_from_context(const void* ucVoid) {
const ucontext_t* uc = (const ucontext_t*)ucVoid;
intptr_t* sp = os::Linux::ucontext_get_sp(uc);
address lr = ucontext_get_lr(uc);
- return frame(sp, lr);
+ return frame(sp, lr, frame::kind::unknown);
}
frame os::get_sender_for_C_frame(frame* fr) {
if (*fr->sp() == 0) {
// fr is the last C frame
- return frame(nullptr, nullptr);
+ return frame();
}
- return frame(fr->sender_sp(), fr->sender_pc());
+ return frame(fr->sender_sp(), fr->sender_pc(), frame::kind::unknown);
}
frame os::current_frame() {
intptr_t* csp = *(intptr_t**) __builtin_frame_address(0);
- frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame));
+ frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame), frame::kind::unknown);
return os::get_sender_for_C_frame(&topframe);
}
diff --git a/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S b/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S
index c8d20cc1b4328..8c96edf01b4d0 100644
--- a/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S
+++ b/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,13 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetchN_continuation
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# r3 : address
diff --git a/src/hotspot/os_cpu/linux_riscv/atomic_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/atomic_linux_riscv.hpp
index e85bd60b22659..6546adb6ff33a 100644
--- a/src/hotspot/os_cpu/linux_riscv/atomic_linux_riscv.hpp
+++ b/src/hotspot/os_cpu/linux_riscv/atomic_linux_riscv.hpp
@@ -39,6 +39,12 @@
#define FULL_COMPILER_ATOMIC_SUPPORT
#endif
+#if defined(__clang_major__)
+#define CORRECT_COMPILER_ATOMIC_SUPPORT
+#elif defined(__GNUC__) && (__riscv_xlen <= 32 || __GNUC__ > 13)
+#define CORRECT_COMPILER_ATOMIC_SUPPORT
+#endif
+
template
struct Atomic::PlatformAdd {
template
@@ -114,6 +120,44 @@ inline T Atomic::PlatformCmpxchg<1>::operator()(T volatile* dest __attribute__((
}
#endif
+#ifndef CORRECT_COMPILER_ATOMIC_SUPPORT
+// The implementation of `__atomic_compare_exchange` lacks sign extensions
+// in GCC 13 and lower when using with 32-bit unsigned integers on RV64,
+// so we should implement it manually.
+// GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114130.
+// See also JDK-8326936.
+template<>
+template
+inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest __attribute__((unused)),
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+
+ int32_t old_value;
+ uint64_t rc_temp;
+
+ if (order != memory_order_relaxed) {
+ FULL_MEM_BARRIER;
+ }
+
+ __asm__ __volatile__ (
+ "1: lr.w %0, %2 \n\t"
+ " bne %0, %3, 2f \n\t"
+ " sc.w %1, %4, %2 \n\t"
+ " bnez %1, 1b \n\t"
+ "2: \n\t"
+ : /*%0*/"=&r" (old_value), /*%1*/"=&r" (rc_temp), /*%2*/"+A" (*dest)
+ : /*%3*/"r" ((int64_t)(int32_t)compare_value), /*%4*/"r" (exchange_value)
+ : "memory" );
+
+ if (order != memory_order_relaxed) {
+ FULL_MEM_BARRIER;
+ }
+ return (T)old_value;
+}
+#endif
+
template
template
inline T Atomic::PlatformXchg::operator()(T volatile* dest,
@@ -151,6 +195,10 @@ inline T Atomic::PlatformCmpxchg::operator()(T volatile* dest __attri
STATIC_ASSERT(byte_size >= 4);
#endif
+#ifndef CORRECT_COMPILER_ATOMIC_SUPPORT
+ STATIC_ASSERT(byte_size != 4);
+#endif
+
STATIC_ASSERT(byte_size == sizeof(T));
if (order != memory_order_relaxed) {
FULL_MEM_BARRIER;
@@ -187,5 +235,6 @@ struct Atomic::PlatformOrderedStore
};
#undef FULL_COMPILER_ATOMIC_SUPPORT
+#undef CORRECT_COMPILER_ATOMIC_SUPPORT
#endif // OS_CPU_LINUX_RISCV_ATOMIC_LINUX_RISCV_HPP
diff --git a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
index 282467bc9e096..9f13e2bdd2cb7 100644
--- a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
@@ -27,7 +27,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
@@ -39,6 +38,7 @@
#include "prims/jvm_misc.hpp"
#include "runtime/arguments.hpp"
#include "runtime/frame.inline.hpp"
+#include "runtime/globals.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/java.hpp"
#include "runtime/javaCalls.hpp"
@@ -406,6 +406,14 @@ static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
extern "C" {
int SpinPause() {
+ if (UseZihintpause) {
+ // PAUSE is encoded as a FENCE instruction with pred=W, succ=0, fm=0, rd=x0, and rs1=x0.
+ // To do: __asm__ volatile("pause " : : : );
+ // Since we're currently not passing '-march=..._zihintpause' to the compiler,
+ // it will not recognize the "pause" instruction, hence the hard-coded instruction.
+ __asm__ volatile(".word 0x0100000f " : : : );
+ return 1;
+ }
return 0;
}
diff --git a/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp b/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
index 243c4b850ee43..df4a2e347ccac 100644
--- a/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
@@ -49,6 +49,36 @@
#define RISCV_HWPROBE_EXT_ZBA (1 << 3)
#define RISCV_HWPROBE_EXT_ZBB (1 << 4)
#define RISCV_HWPROBE_EXT_ZBS (1 << 5)
+#define RISCV_HWPROBE_EXT_ZICBOZ (1 << 6)
+#define RISCV_HWPROBE_EXT_ZBC (1 << 7)
+#define RISCV_HWPROBE_EXT_ZBKB (1 << 8)
+#define RISCV_HWPROBE_EXT_ZBKC (1 << 9)
+#define RISCV_HWPROBE_EXT_ZBKX (1 << 10)
+#define RISCV_HWPROBE_EXT_ZKND (1 << 11)
+#define RISCV_HWPROBE_EXT_ZKNE (1 << 12)
+#define RISCV_HWPROBE_EXT_ZKNH (1 << 13)
+#define RISCV_HWPROBE_EXT_ZKSED (1 << 14)
+#define RISCV_HWPROBE_EXT_ZKSH (1 << 15)
+#define RISCV_HWPROBE_EXT_ZKT (1 << 16)
+#define RISCV_HWPROBE_EXT_ZVBB (1 << 17)
+#define RISCV_HWPROBE_EXT_ZVBC (1 << 18)
+#define RISCV_HWPROBE_EXT_ZVKB (1 << 19)
+#define RISCV_HWPROBE_EXT_ZVKG (1 << 20)
+#define RISCV_HWPROBE_EXT_ZVKNED (1 << 21)
+#define RISCV_HWPROBE_EXT_ZVKNHA (1 << 22)
+#define RISCV_HWPROBE_EXT_ZVKNHB (1 << 23)
+#define RISCV_HWPROBE_EXT_ZVKSED (1 << 24)
+#define RISCV_HWPROBE_EXT_ZVKSH (1 << 25)
+#define RISCV_HWPROBE_EXT_ZVKT (1 << 26)
+#define RISCV_HWPROBE_EXT_ZFH (1 << 27)
+#define RISCV_HWPROBE_EXT_ZFHMIN (1 << 28)
+#define RISCV_HWPROBE_EXT_ZIHINTNTL (1 << 29)
+#define RISCV_HWPROBE_EXT_ZVFH (1 << 30)
+#define RISCV_HWPROBE_EXT_ZVFHMIN (1 << 31)
+#define RISCV_HWPROBE_EXT_ZFA (1ULL << 32)
+#define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33)
+#define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34)
+#define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35)
#define RISCV_HWPROBE_KEY_CPUPERF_0 5
#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
@@ -145,6 +175,9 @@ void RiscvHwprobe::add_features_from_query_result() {
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZBS)) {
VM_Version::ext_Zbs.enable_feature();
}
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZFH)) {
+ VM_Version::ext_Zfh.enable_feature();
+ }
if (is_valid(RISCV_HWPROBE_KEY_CPUPERF_0)) {
VM_Version::unaligned_access.enable_feature(
query[RISCV_HWPROBE_KEY_CPUPERF_0].value & RISCV_HWPROBE_MISALIGNED_MASK);
diff --git a/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S b/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S
index ecf0bac6f9e78..150df7567bdb8 100644
--- a/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S
+++ b/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,13 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetchN_continuation
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# x10 (a0) : address
diff --git a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
index 354dbd70bb4e1..c7d61b33829ab 100644
--- a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
@@ -169,13 +169,13 @@ void VM_Version::os_aux_features() {
}
VM_Version::VM_MODE VM_Version::parse_satp_mode(const char* vm_mode) {
- if (!strcmp(vm_mode, "sv39")) {
+ if (!strncmp(vm_mode, "sv39", sizeof "sv39" - 1)) {
return VM_SV39;
- } else if (!strcmp(vm_mode, "sv48")) {
+ } else if (!strncmp(vm_mode, "sv48", sizeof "sv48" - 1)) {
return VM_SV48;
- } else if (!strcmp(vm_mode, "sv57")) {
+ } else if (!strncmp(vm_mode, "sv57", sizeof "sv57" - 1)) {
return VM_SV57;
- } else if (!strcmp(vm_mode, "sv64")) {
+ } else if (!strncmp(vm_mode, "sv64", sizeof "sv64" - 1)) {
return VM_SV64;
} else {
return VM_MBARE;
@@ -197,7 +197,7 @@ char* VM_Version::os_uarch_additional_features() {
if ((p = strchr(buf, ':')) != nullptr) {
if (mode == VM_NOTSET) {
if (strncmp(buf, "mmu", sizeof "mmu" - 1) == 0) {
- mode = VM_Version::parse_satp_mode(p);
+ mode = VM_Version::parse_satp_mode(p + 2);
}
}
if (ret == nullptr) {
@@ -242,6 +242,10 @@ void VM_Version::rivos_features() {
ext_Zcb.enable_feature();
+ ext_Zfh.enable_feature();
+
+ ext_Zacas.enable_feature();
+ ext_Zicboz.enable_feature();
ext_Zicsr.enable_feature();
ext_Zifencei.enable_feature();
ext_Zic64b.enable_feature();
diff --git a/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp b/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
index 033ea14ead6a4..5aa65e705d9ed 100644
--- a/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
+++ b/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
@@ -28,7 +28,6 @@
// no precompiled headers
#include "asm/assembler.inline.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/disassembler.hpp"
diff --git a/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S b/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S
index 47fe82f5a278b..43d50c798e534 100644
--- a/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S
+++ b/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,13 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetchN_continuation
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# r2 : address
diff --git a/src/hotspot/os_cpu/linux_x86/linux_x86_32.S b/src/hotspot/os_cpu/linux_x86/linux_x86_32.S
index 344358172defd..e23cd2b9164ae 100644
--- a/src/hotspot/os_cpu/linux_x86/linux_x86_32.S
+++ b/src/hotspot/os_cpu/linux_x86/linux_x86_32.S
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2004, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -21,28 +21,41 @@
# questions.
#
+ .globl SpinPause
# NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
+ # from server-compiler-generated code via CallLeafNoFP,
+ # which means that they *must* either not use floating
+ # point or use it in the same manner as does the server
+ # compiler.
.globl _Copy_arrayof_conjoint_bytes
.globl _Copy_conjoint_jshorts_atomic
- .globl _Copy_arrayof_conjoint_jshorts
+ .globl _Copy_arrayof_conjoint_jshorts
.globl _Copy_conjoint_jints_atomic
.globl _Copy_arrayof_conjoint_jints
- .globl _Copy_conjoint_jlongs_atomic
- .globl _mmx_Copy_arrayof_conjoint_jshorts
+ .globl _Copy_conjoint_jlongs_atomic
+ .globl _mmx_Copy_arrayof_conjoint_jshorts
.globl _Atomic_cmpxchg_long
.globl _Atomic_move_long
- .text
+ .hidden SpinPause
- .globl SpinPause
- .type SpinPause,@function
+ .hidden _Copy_arrayof_conjoint_bytes
+ .hidden _Copy_conjoint_jshorts_atomic
+ .hidden _Copy_arrayof_conjoint_jshorts
+ .hidden _Copy_conjoint_jints_atomic
+ .hidden _Copy_arrayof_conjoint_jints
+ .hidden _Copy_conjoint_jlongs_atomic
+ .hidden _mmx_Copy_arrayof_conjoint_jshorts
+
+ .hidden _Atomic_cmpxchg_long
+ .hidden _Atomic_move_long
+
+ .text
+
+ .type SpinPause,@function
.p2align 4,,15
SpinPause:
rep
@@ -55,7 +68,7 @@ SpinPause:
# size_t count)
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_bytes,@function
+ .type _Copy_arrayof_conjoint_bytes,@function
_Copy_arrayof_conjoint_bytes:
pushl %esi
movl 4+12(%esp),%ecx # count
@@ -115,7 +128,7 @@ acb_CopyLeft:
jbe 2f # <= 32 dwords
rep; smovl
jmp 4f
- .space 8
+ .space 8
2: subl %esi,%edi
.p2align 4,,15
3: movl (%esi),%edx
@@ -131,7 +144,7 @@ acb_CopyLeft:
addl $3,%esi
6: movb (%esi),%dl
movb %dl,(%edi,%esi,1)
- subl $1,%esi
+ subl $1,%esi
subl $1,%ecx
jnz 6b
7: cld
@@ -143,7 +156,7 @@ acb_CopyLeft:
# void* to,
# size_t count)
.p2align 4,,15
- .type _Copy_conjoint_jshorts_atomic,@function
+ .type _Copy_conjoint_jshorts_atomic,@function
_Copy_conjoint_jshorts_atomic:
pushl %esi
movl 4+12(%esp),%ecx # count
@@ -230,7 +243,7 @@ cs_CopyLeft:
# void* to,
# size_t count)
.p2align 4,,15
- .type _Copy_arrayof_conjoint_jshorts,@function
+ .type _Copy_arrayof_conjoint_jshorts,@function
_Copy_arrayof_conjoint_jshorts:
pushl %esi
movl 4+12(%esp),%ecx # count
@@ -307,8 +320,8 @@ acs_CopyLeft:
# Equivalent to
# arrayof_conjoint_jints
.p2align 4,,15
- .type _Copy_conjoint_jints_atomic,@function
- .type _Copy_arrayof_conjoint_jints,@function
+ .type _Copy_conjoint_jints_atomic,@function
+ .type _Copy_arrayof_conjoint_jints,@function
_Copy_conjoint_jints_atomic:
_Copy_arrayof_conjoint_jints:
pushl %esi
@@ -384,7 +397,7 @@ ci_CopyLeft:
# }
*/
.p2align 4,,15
- .type _Copy_conjoint_jlongs_atomic,@function
+ .type _Copy_conjoint_jlongs_atomic,@function
_Copy_conjoint_jlongs_atomic:
movl 4+8(%esp),%ecx # count
movl 4+0(%esp),%eax # from
@@ -413,7 +426,7 @@ cla_CopyLeft:
# void* to,
# size_t count)
.p2align 4,,15
- .type _mmx_Copy_arrayof_conjoint_jshorts,@function
+ .type _mmx_Copy_arrayof_conjoint_jshorts,@function
_mmx_Copy_arrayof_conjoint_jshorts:
pushl %esi
movl 4+12(%esp),%ecx
@@ -465,8 +478,8 @@ mmx_acs_CopyRight:
cmpl $16,%ecx
jge 4b
emms
- testl %ecx,%ecx
- ja 1b
+ testl %ecx,%ecx
+ ja 1b
5: andl $1,%eax
je 7f
6: movw (%esi),%dx
@@ -511,7 +524,7 @@ mmx_acs_CopyLeft:
# jlong exchange_value)
#
.p2align 4,,15
- .type _Atomic_cmpxchg_long,@function
+ .type _Atomic_cmpxchg_long,@function
_Atomic_cmpxchg_long:
# 8(%esp) : return PC
pushl %ebx # 4(%esp) : old %ebx
@@ -530,7 +543,7 @@ _Atomic_cmpxchg_long:
# Support for jlong Atomic::load and Atomic::store.
# void _Atomic_move_long(const volatile jlong* src, volatile jlong* dst)
.p2align 4,,15
- .type _Atomic_move_long,@function
+ .type _Atomic_move_long,@function
_Atomic_move_long:
movl 4(%esp), %eax # src
fildll (%eax)
diff --git a/src/hotspot/os_cpu/linux_x86/linux_x86_64.S b/src/hotspot/os_cpu/linux_x86/linux_x86_64.S
index 89d98cb583786..65580a194afab 100644
--- a/src/hotspot/os_cpu/linux_x86/linux_x86_64.S
+++ b/src/hotspot/os_cpu/linux_x86/linux_x86_64.S
@@ -1,5 +1,5 @@
-#
-# Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
+#
+# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -21,24 +21,34 @@
# questions.
#
+ .globl SpinPause
# NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
-
+ # from server-compiler-generated code via CallLeafNoFP,
+ # which means that they *must* either not use floating
+ # point or use it in the same manner as does the server
+ # compiler.
+
.globl _Copy_arrayof_conjoint_bytes
- .globl _Copy_arrayof_conjoint_jshorts
+ .globl _Copy_arrayof_conjoint_jshorts
.globl _Copy_conjoint_jshorts_atomic
.globl _Copy_arrayof_conjoint_jints
.globl _Copy_conjoint_jints_atomic
.globl _Copy_arrayof_conjoint_jlongs
.globl _Copy_conjoint_jlongs_atomic
- .text
+ .hidden SpinPause
+
+ .hidden _Copy_arrayof_conjoint_bytes
+ .hidden _Copy_arrayof_conjoint_jshorts
+ .hidden _Copy_conjoint_jshorts_atomic
+ .hidden _Copy_arrayof_conjoint_jints
+ .hidden _Copy_conjoint_jints_atomic
+ .hidden _Copy_arrayof_conjoint_jlongs
+ .hidden _Copy_conjoint_jlongs_atomic
+
+ .text
- .globl SpinPause
.align 16
.type SpinPause,@function
SpinPause:
@@ -55,7 +65,7 @@ SpinPause:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_bytes,@function
+ .type _Copy_arrayof_conjoint_bytes,@function
_Copy_arrayof_conjoint_bytes:
movq %rdx,%r8 # byte count
shrq $3,%rdx # qword count
@@ -63,7 +73,7 @@ _Copy_arrayof_conjoint_bytes:
leaq -1(%rdi,%r8,1),%rax # from + bcount*1 - 1
jbe acb_CopyRight
cmpq %rax,%rsi
- jbe acb_CopyLeft
+ jbe acb_CopyLeft
acb_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -157,8 +167,8 @@ acb_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_jshorts,@function
- .type _Copy_conjoint_jshorts_atomic,@function
+ .type _Copy_arrayof_conjoint_jshorts,@function
+ .type _Copy_conjoint_jshorts_atomic,@function
_Copy_arrayof_conjoint_jshorts:
_Copy_conjoint_jshorts_atomic:
movq %rdx,%r8 # word count
@@ -167,7 +177,7 @@ _Copy_conjoint_jshorts_atomic:
leaq -2(%rdi,%r8,2),%rax # from + wcount*2 - 2
jbe acs_CopyRight
cmpq %rax,%rsi
- jbe acs_CopyLeft
+ jbe acs_CopyLeft
acs_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -247,8 +257,8 @@ acs_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_jints,@function
- .type _Copy_conjoint_jints_atomic,@function
+ .type _Copy_arrayof_conjoint_jints,@function
+ .type _Copy_conjoint_jints_atomic,@function
_Copy_arrayof_conjoint_jints:
_Copy_conjoint_jints_atomic:
movq %rdx,%r8 # dword count
@@ -257,7 +267,7 @@ _Copy_conjoint_jints_atomic:
leaq -4(%rdi,%r8,4),%rax # from + dcount*4 - 4
jbe aci_CopyRight
cmpq %rax,%rsi
- jbe aci_CopyLeft
+ jbe aci_CopyLeft
aci_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -326,15 +336,15 @@ aci_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_jlongs,@function
- .type _Copy_conjoint_jlongs_atomic,@function
+ .type _Copy_arrayof_conjoint_jlongs,@function
+ .type _Copy_conjoint_jlongs_atomic,@function
_Copy_arrayof_conjoint_jlongs:
_Copy_conjoint_jlongs_atomic:
cmpq %rdi,%rsi
leaq -8(%rdi,%rdx,8),%rax # from + count*8 - 8
jbe acl_CopyRight
cmpq %rax,%rsi
- jbe acl_CopyLeft
+ jbe acl_CopyLeft
acl_CopyRight:
leaq -8(%rsi,%rdx,8),%rcx # to + count*8 - 8
negq %rdx
diff --git a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
index b211330409d59..4dcaedf71da8c 100644
--- a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
+++ b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
@@ -165,7 +164,7 @@ frame os::get_sender_for_C_frame(frame* fr) {
return frame(fr->sender_sp(), fr->link(), fr->sender_pc());
}
-intptr_t* _get_previous_fp() {
+static intptr_t* _get_previous_fp() {
#if defined(__clang__)
intptr_t **ebp;
__asm__ __volatile__ ("mov %%" SPELL_REG_FP ", %0":"=r"(ebp):);
diff --git a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S
index 492b1207db6e2..54775cb7e8ede 100644
--- a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S
+++ b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S
@@ -1,6 +1,6 @@
#
# Copyright (c) 2022 SAP SE. All rights reserved.
-# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,11 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
- .text
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
+ .text
# Support for int SafeFetch32(int* address, int defaultval);
#
diff --git a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S
index 617851e8327d4..1937e71708897 100644
--- a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S
+++ b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S
@@ -1,6 +1,6 @@
#
# Copyright (c) 2022 SAP SE. All rights reserved.
-# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,14 @@
.globl _SafeFetch32_continuation
.globl _SafeFetchN_continuation
- .text
+ .hidden SafeFetch32_impl
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetch32_continuation
+ .hidden _SafeFetchN_continuation
+
+ .text
# Support for int SafeFetch32(int* address, int defaultval);
diff --git a/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp b/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
index 1ce73588524c1..d593c46d15d91 100644
--- a/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
+++ b/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
@@ -27,7 +27,6 @@
#include "asm/assembler.inline.hpp"
#include "atomic_linux_zero.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp
index 46f718a9cd0f5..78e98609b6bdc 100644
--- a/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp
+++ b/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp
@@ -27,7 +27,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "code/nativeInst.hpp"
#include "interpreter/interpreter.hpp"
diff --git a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
index 4e18334315a37..7e0814c014bec 100644
--- a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
+++ b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
@@ -25,7 +25,6 @@
// no precompiled headers
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/share/adlc/archDesc.cpp b/src/hotspot/share/adlc/archDesc.cpp
index d27bf0865608a..93fa7451dc0b9 100644
--- a/src/hotspot/share/adlc/archDesc.cpp
+++ b/src/hotspot/share/adlc/archDesc.cpp
@@ -1,5 +1,5 @@
//
-// Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
// archDesc.cpp - Internal format for architecture definition
+#include
#include "adlc.hpp"
static FILE *errfile = stderr;
@@ -684,6 +685,98 @@ bool ArchDesc::verify() {
return true;
}
+class MarkUsageFormClosure : public FormClosure {
+private:
+ ArchDesc* _ad;
+ std::unordered_set *_visited;
+
+public:
+ MarkUsageFormClosure(ArchDesc* ad, std::unordered_set *visit_map) {
+ _ad = ad;
+ _visited = visit_map;
+ }
+ virtual ~MarkUsageFormClosure() = default;
+
+ virtual void do_form(Form *form) {
+ if (_visited->find(form) == _visited->end()) {
+ _visited->insert(form);
+ form->forms_do(this);
+ }
+ }
+
+ virtual void do_form_by_name(const char* name) {
+ const Form* form = _ad->globalNames()[name];
+ if (form) {
+ do_form(const_cast(form));
+ return;
+ }
+ RegisterForm* regs = _ad->get_registers();
+ if (regs->getRegClass(name)) {
+ do_form(regs->getRegClass(name));
+ return;
+ }
+ }
+};
+
+// check unused operands
+bool ArchDesc::check_usage() {
+ std::unordered_set visited;
+ MarkUsageFormClosure callback(this, &visited);
+ _instructions.reset();
+ // iterate all instruction to mark used form
+ InstructForm* instr;
+ for ( ; (instr = (InstructForm*)_instructions.iter()) != nullptr; ) {
+ callback.do_form(instr);
+ }
+
+ // these forms are coded in OperandForm::is_user_name_for_sReg
+ // it may happen no instruction use these operands, like stackSlotP in aarch64,
+ // but we can not desclare they are useless.
+ callback.do_form_by_name("stackSlotI");
+ callback.do_form_by_name("stackSlotP");
+ callback.do_form_by_name("stackSlotD");
+ callback.do_form_by_name("stackSlotF");
+ callback.do_form_by_name("stackSlotL");
+
+ // sReg* are initial created by adlc in ArchDesc::initBaseOpTypes()
+ // In ARM, no definition or usage in adfile, but they are reported as unused
+ callback.do_form_by_name("sRegI");
+ callback.do_form_by_name("sRegP");
+ callback.do_form_by_name("sRegD");
+ callback.do_form_by_name("sRegF");
+ callback.do_form_by_name("sRegL");
+
+ // special generic vector operands only used in Matcher::pd_specialize_generic_vector_operand
+#if defined(AARCH64)
+ callback.do_form_by_name("vecA");
+ callback.do_form_by_name("vecD");
+ callback.do_form_by_name("vecX");
+#elif defined(AMD64)
+ callback.do_form_by_name("vecS");
+ callback.do_form_by_name("vecD");
+ callback.do_form_by_name("vecX");
+ callback.do_form_by_name("vecY");
+ callback.do_form_by_name("vecZ");
+ callback.do_form_by_name("legVecS");
+ callback.do_form_by_name("legVecD");
+ callback.do_form_by_name("legVecX");
+ callback.do_form_by_name("legVecY");
+ callback.do_form_by_name("legVecZ");
+#endif
+
+ int cnt = 0;
+ _operands.reset();
+ OperandForm* operand;
+ for ( ; (operand = (OperandForm*)_operands.iter()) != nullptr; ) {
+ if(visited.find(operand) == visited.end() && !operand->ideal_only()) {
+ fprintf(stderr, "\nWarning: unused operand (%s)", operand->_ident);
+ cnt++;
+ }
+ }
+ if (cnt) fprintf(stderr, "\n-------Warning: total %d unused operands\n", cnt);
+
+ return true;
+}
void ArchDesc::dump() {
_pre_header.dump();
diff --git a/src/hotspot/share/adlc/archDesc.hpp b/src/hotspot/share/adlc/archDesc.hpp
index 42369f6c2d13a..99e4947e110e1 100644
--- a/src/hotspot/share/adlc/archDesc.hpp
+++ b/src/hotspot/share/adlc/archDesc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -226,6 +226,7 @@ class ArchDesc {
inline void getForm(EncodeForm **ptr) { *ptr = _encode; }
bool verify();
+ bool check_usage();
void dump();
// Helper utility that gets MatchList components from inside MatchRule
diff --git a/src/hotspot/share/adlc/forms.cpp b/src/hotspot/share/adlc/forms.cpp
index fd8e5c4fd50b1..068d745254e3d 100644
--- a/src/hotspot/share/adlc/forms.cpp
+++ b/src/hotspot/share/adlc/forms.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -362,6 +362,15 @@ void FormDict::dump() {
_form.print(dumpkey, dumpform);
}
+void FormDict::forms_do(FormClosure* f) {;
+ DictI iter(&_form);
+ for( ; iter.test(); ++iter ) {
+ Form* form = (Form*) iter._value;
+ assert(form != nullptr, "sanity");
+ f->do_form(form);
+ }
+}
+
//------------------------------SourceForm-------------------------------------
SourceForm::SourceForm(char* code) : _code(code) { }; // Constructor
SourceForm::~SourceForm() {
@@ -374,3 +383,11 @@ void SourceForm::dump() { // Debug printer
void SourceForm::output(FILE *fp) {
fprintf(fp,"\n//%s\n%s\n",classname(),(_code?_code:""));
}
+
+void FormClosure::do_form(Form* form) {
+ assert(false, "should not reach here");
+}
+
+void FormClosure::do_form_by_name(const char* name) {
+ assert(false, "should not reach here");
+}
diff --git a/src/hotspot/share/adlc/forms.hpp b/src/hotspot/share/adlc/forms.hpp
index c3dd85eb98bae..a82b9bbb3382d 100644
--- a/src/hotspot/share/adlc/forms.hpp
+++ b/src/hotspot/share/adlc/forms.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -58,6 +58,7 @@ class Flag;
class RewriteRule;
class ConstructRule;
class FormatRule;
+class FormClosure;
class Peephole;
class EncClass;
class Interface;
@@ -114,6 +115,8 @@ class FormDict {
const Form *operator [](const char *name) const; // Do a lookup
void dump();
+ // iterate child forms recursively
+ void forms_do(FormClosure *f);
};
// ***** Master Class for ADL Parser Forms *****
@@ -163,6 +166,9 @@ class Form {
// Write info to output files
virtual void output(FILE *fp) { fprintf(fp,"Form Output"); }
+ // iterate child forms recursively
+ virtual void forms_do (FormClosure* f) { return; }
+
public:
// ADLC types, match the last character on ideal operands and instructions
enum DataType {
@@ -255,6 +261,16 @@ class Form {
};
+class FormClosure {
+public:
+ FormClosure() = default;
+ virtual ~FormClosure() = default;
+
+ virtual void do_form(Form* form);
+ virtual void do_form_by_name(const char* name);
+};
+
+
//------------------------------FormList---------------------------------------
class FormList {
private:
diff --git a/src/hotspot/share/adlc/formsopt.cpp b/src/hotspot/share/adlc/formsopt.cpp
index 13d29ef3947cc..e1e4ed96c2ea0 100644
--- a/src/hotspot/share/adlc/formsopt.cpp
+++ b/src/hotspot/share/adlc/formsopt.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -198,6 +198,20 @@ void RegisterForm::output(FILE *fp) { // Write info to output files
fprintf(fp,"-------------------- end RegisterForm --------------------\n");
}
+void RegisterForm::forms_do(FormClosure *f) {
+ const char *name = nullptr;
+ if (_current_ac) f->do_form(_current_ac);
+ for(_rdefs.reset(); (name = _rdefs.iter()) != nullptr;) {
+ f->do_form((RegDef*)_regDef[name]);
+ }
+ for (_rclasses.reset(); (name = _rclasses.iter()) != nullptr;) {
+ f->do_form((RegClass*)_regClass[name]);
+ }
+ for (_aclasses.reset(); (name = _aclasses.iter()) != nullptr;) {
+ f->do_form((AllocClass*)_allocClass[name]);
+ }
+}
+
//------------------------------RegDef-----------------------------------------
// Constructor
RegDef::RegDef(char *regname, char *callconv, char *c_conv, char * idealtype, char * encode, char * concrete)
@@ -322,6 +336,13 @@ void RegClass::output(FILE *fp) { // Write info to output files
fprintf(fp,"--- done with entries for reg_class %s\n\n",_classid);
}
+void RegClass::forms_do(FormClosure *f) {
+ const char *name = nullptr;
+ for( _regDefs.reset(); (name = _regDefs.iter()) != nullptr; ) {
+ f->do_form((RegDef*)_regDef[name]);
+ }
+}
+
void RegClass::declare_register_masks(FILE* fp) {
const char* prefix = "";
const char* rc_name_to_upper = toUpper(_classid);
@@ -436,6 +457,14 @@ void AllocClass::output(FILE *fp) { // Write info to output files
fprintf(fp,"--- done with entries for alloc_class %s\n\n",_classid);
}
+void AllocClass::forms_do(FormClosure* f) {
+ const char *name;
+ for(_regDefs.reset(); (name = _regDefs.iter()) != nullptr;) {
+ f->do_form((RegDef*)_regDef[name]);
+ }
+ return;
+}
+
//==============================Frame Handling=================================
//------------------------------FrameForm--------------------------------------
FrameForm::FrameForm() {
@@ -706,6 +735,15 @@ void Peephole::output(FILE *fp) { // Write info to output files
if( _next ) _next->output(fp);
}
+void Peephole::forms_do(FormClosure *f) {
+ if (_predicate) f->do_form(_predicate);
+ if (_match) f->do_form(_match);
+ if (_procedure) f->do_form(_procedure);
+ if (_constraint) f->do_form(_constraint);
+ if (_replace) f->do_form(_replace);
+ return;
+}
+
//----------------------------PeepPredicate------------------------------------
PeepPredicate::PeepPredicate(const char* rule) : _rule(rule) {
}
diff --git a/src/hotspot/share/adlc/formsopt.hpp b/src/hotspot/share/adlc/formsopt.hpp
index 1a2b5dadd30ee..d183a46b87545 100644
--- a/src/hotspot/share/adlc/formsopt.hpp
+++ b/src/hotspot/share/adlc/formsopt.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -123,6 +123,7 @@ class RegisterForm : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
//------------------------------RegDef-----------------------------------------
@@ -199,6 +200,7 @@ class RegClass : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
virtual bool has_stack_version() {
return _stack_or_reg;
@@ -305,6 +307,11 @@ class ConditionalRegClass : public RegClass {
char* condition_code() {
return _condition_code;
}
+
+ virtual void forms_do(FormClosure* f) {
+ if (_rclasses[0]) f->do_form(_rclasses[0]);
+ if (_rclasses[1]) f->do_form(_rclasses[1]);
+ }
};
//------------------------------AllocClass-------------------------------------
@@ -325,6 +332,7 @@ class AllocClass : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
@@ -568,6 +576,7 @@ class Peephole : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
class PeepPredicate : public Form {
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index ecbf472c59ef4..be97547f8ce11 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -1498,6 +1498,24 @@ void InstructForm::output(FILE *fp) {
if (_peephole) _peephole->output(fp);
}
+void InstructForm::forms_do(FormClosure *f) {
+ if (_cisc_spill_alternate) f->do_form(_cisc_spill_alternate);
+ if (_short_branch_form) f->do_form(_short_branch_form);
+ _localNames.forms_do(f);
+ if (_matrule) f->do_form(_matrule);
+ if (_opcode) f->do_form(_opcode);
+ if (_insencode) f->do_form(_insencode);
+ if (_constant) f->do_form(_constant);
+ if (_attribs) f->do_form(_attribs);
+ if (_predicate) f->do_form(_predicate);
+ _effects.forms_do(f);
+ if (_exprule) f->do_form(_exprule);
+ if (_rewrule) f->do_form(_rewrule);
+ if (_format) f->do_form(_format);
+ if (_peephole) f->do_form(_peephole);
+ assert(_components.count() == 0, "skip components");
+}
+
void MachNodeForm::dump() {
output(stderr);
}
@@ -1615,6 +1633,14 @@ void EncodeForm::output(FILE *fp) { // Write info to output files
}
fprintf(fp,"-------------------- end EncodeForm --------------------\n");
}
+
+void EncodeForm::forms_do(FormClosure* f) {
+ const char *name;
+ for (_eclasses.reset(); (name = _eclasses.iter()) != nullptr;) {
+ f->do_form((EncClass*)_encClass[name]);
+ }
+}
+
//------------------------------EncClass---------------------------------------
EncClass::EncClass(const char *name)
: _localNames(cmpstr,hashstr, Form::arena), _name(name) {
@@ -1705,6 +1731,15 @@ void EncClass::output(FILE *fp) {
}
+void EncClass::forms_do(FormClosure *f) {
+ _parameter_type.reset();
+ const char *type = _parameter_type.iter();
+ for ( ; type != nullptr ; type = _parameter_type.iter() ) {
+ f->do_form_by_name(type);
+ }
+ _localNames.forms_do(f);
+}
+
//------------------------------Opcode-----------------------------------------
Opcode::Opcode(char *primary, char *secondary, char *tertiary)
: _primary(primary), _secondary(secondary), _tertiary(tertiary) {
@@ -1835,6 +1870,15 @@ void InsEncode::output(FILE *fp) {
fprintf(fp,"\n");
}
+void InsEncode::forms_do(FormClosure *f) {
+ _encoding.reset();
+ NameAndList *encoding = (NameAndList*)_encoding.iter();
+ for( ; encoding != nullptr; encoding = (NameAndList*)_encoding.iter() ) {
+ // just check name, other operands will be checked as instruction parameters
+ f->do_form_by_name(encoding->name());
+ }
+}
+
//------------------------------Effect-----------------------------------------
static int effect_lookup(const char *name) {
if (!strcmp(name, "USE")) return Component::USE;
@@ -1968,6 +2012,19 @@ void ExpandRule::output(FILE *fp) { // Write info to output files
}
}
+void ExpandRule::forms_do(FormClosure *f) {
+ NameAndList *expand_instr = nullptr;
+ // Iterate over the instructions 'node' expands into
+ for(reset_instructions(); (expand_instr = iter_instructions()) != nullptr; ) {
+ f->do_form_by_name(expand_instr->name());
+ }
+ _newopers.reset();
+ const char* oper = _newopers.iter();
+ for(; oper != nullptr; oper = _newopers.iter()) {
+ f->do_form_by_name(oper);
+ }
+}
+
//------------------------------RewriteRule------------------------------------
RewriteRule::RewriteRule(char* params, char* block)
: _tempParams(params), _tempBlock(block) { }; // Constructor
@@ -1984,6 +2041,12 @@ void RewriteRule::output(FILE *fp) { // Write info to output files
(_tempBlock?_tempBlock:""));
}
+void RewriteRule::forms_do(FormClosure *f) {
+ if (_condition) f->do_form(_condition);
+ if (_instrs) f->do_form(_instrs);
+ if (_opers) f->do_form(_opers);
+}
+
//==============================MachNodes======================================
//------------------------------MachNodeForm-----------------------------------
@@ -2066,6 +2129,13 @@ void OpClassForm::output(FILE *fp) {
fprintf(fp,"\n");
}
+void OpClassForm::forms_do(FormClosure* f) {
+ const char *name;
+ for(_oplst.reset(); (name = _oplst.iter()) != nullptr;) {
+ f->do_form_by_name(name);
+ }
+}
+
//==============================Operands=======================================
//------------------------------OperandForm------------------------------------
@@ -2691,6 +2761,22 @@ void OperandForm::output(FILE *fp) {
if (_format) _format->dump();
}
+void OperandForm::forms_do(FormClosure* f) {
+ if (_matrule) f->do_form(_matrule);
+ if (_interface) f->do_form(_interface);
+ if (_attribs) f->do_form(_attribs);
+ if (_predicate) f->do_form(_predicate);
+ if (_constraint) f->do_form(_constraint);
+ if (_construct) f->do_form(_construct);
+ if (_format) f->do_form(_format);
+ _localNames.forms_do(f);
+ const char* opclass = nullptr;
+ for ( _classes.reset(); (opclass = _classes.iter()) != nullptr; ) {
+ f->do_form_by_name(opclass);
+ }
+ assert(_components.count() == 0, "skip _compnets");
+}
+
//------------------------------Constraint-------------------------------------
Constraint::Constraint(const char *func, const char *arg)
: _func(func), _arg(arg) {
@@ -2712,6 +2798,10 @@ void Constraint::output(FILE *fp) { // Write info to output files
fprintf(fp,"Constraint: %s ( %s )\n", _func, _arg);
}
+void Constraint::forms_do(FormClosure *f) {
+ f->do_form_by_name(_arg);
+}
+
//------------------------------Predicate--------------------------------------
Predicate::Predicate(char *pr)
: _pred(pr) {
@@ -3539,6 +3629,12 @@ void MatchNode::output(FILE *fp) {
}
}
+void MatchNode::forms_do(FormClosure *f) {
+ f->do_form_by_name(_name);
+ if (_lChild) f->do_form(_lChild);
+ if (_rChild) f->do_form(_rChild);
+}
+
int MatchNode::needs_ideal_memory_edge(FormDict &globals) const {
static const char *needs_ideal_memory_list[] = {
"StoreI","StoreL","StoreP","StoreN","StoreNKlass","StoreD","StoreF" ,
@@ -3608,6 +3704,7 @@ int InstructForm::needs_base_oop_edge(FormDict &globals) const {
}
+
//-------------------------cisc spilling methods-------------------------------
// helper routines and methods for detecting cisc-spilling instructions
//-------------------------cisc_spill_merge------------------------------------
@@ -4334,6 +4431,18 @@ void MatchRule::output(FILE *fp) {
fprintf(fp,"\n");
}
+void MatchRule::forms_do(FormClosure* f) {
+ // keep sync with MatchNode::forms_do
+ f->do_form_by_name(_name);
+ if (_lChild) f->do_form(_lChild);
+ if (_rChild) f->do_form(_rChild);
+
+ // handle next rule
+ if (_next) {
+ f->do_form(_next);
+ }
+}
+
//------------------------------Attribute--------------------------------------
Attribute::Attribute(char *id, char* val, int type)
: _ident(id), _val(val), _atype(type) {
diff --git a/src/hotspot/share/adlc/formssel.hpp b/src/hotspot/share/adlc/formssel.hpp
index eabd94c323a29..61d0fb40f18a8 100644
--- a/src/hotspot/share/adlc/formssel.hpp
+++ b/src/hotspot/share/adlc/formssel.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -310,6 +310,7 @@ class InstructForm : public Form {
virtual void dump(); // Debug printer
virtual void output(FILE *fp); // Write to output files
+ virtual void forms_do(FormClosure *f);
};
//------------------------------EncodeForm-------------------------------------
@@ -333,6 +334,7 @@ class EncodeForm : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure *f);
};
//------------------------------EncClass---------------------------------------
@@ -377,6 +379,7 @@ class EncClass : public Form {
bool verify();
void dump();
void output(FILE *fp);
+ virtual void forms_do(FormClosure* f);
};
//------------------------------MachNode---------------------------------------
@@ -468,6 +471,7 @@ class InsEncode : public Form {
void dump();
void output(FILE *fp);
+ virtual void forms_do(FormClosure *f);
};
//------------------------------Effect-----------------------------------------
@@ -515,6 +519,7 @@ class ExpandRule : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure *f);
};
//---------------------------------Flag----------------------------------------
@@ -554,6 +559,7 @@ class RewriteRule : public Form {
~RewriteRule(); // Destructor
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
@@ -584,6 +590,7 @@ class OpClassForm : public Form {
virtual bool ideal_only() const;
virtual void dump(); // Debug printer
virtual void output(FILE *fp); // Write to output files
+ virtual void forms_do(FormClosure* f);
};
//------------------------------OperandForm------------------------------------
@@ -711,6 +718,7 @@ class OperandForm : public OpClassForm {
virtual void dump(); // Debug printer
virtual void output(FILE *fp); // Write to output files
+ virtual void forms_do(FormClosure* f);
};
//------------------------------Constraint-------------------------------------
@@ -729,6 +737,7 @@ class Constraint : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
//------------------------------Predicate--------------------------------------
@@ -1014,6 +1023,7 @@ class MatchNode : public Form {
void dump();
void output(FILE *fp);
+ virtual void forms_do(FormClosure* f);
};
//------------------------------MatchRule--------------------------------------
@@ -1075,6 +1085,7 @@ class MatchRule : public MatchNode {
void dump();
void output_short(FILE *fp);
void output(FILE *fp);
+ virtual void forms_do(FormClosure* f);
};
//------------------------------Attribute--------------------------------------
diff --git a/src/hotspot/share/adlc/main.cpp b/src/hotspot/share/adlc/main.cpp
index 6d921cf5e7bee..fd270b0944369 100644
--- a/src/hotspot/share/adlc/main.cpp
+++ b/src/hotspot/share/adlc/main.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -186,6 +186,9 @@ int main(int argc, char *argv[])
// Verify that the results of the parse are consistent
AD.verify();
+ // Check defined operands are used
+ AD.check_usage();
+
// Prepare to generate the result files:
AD.generateMatchLists();
AD.identify_unique_operands();
@@ -216,7 +219,6 @@ int main(int argc, char *argv[])
AD.addInclude(AD._CPP_file, "code/nativeInst.hpp");
AD.addInclude(AD._CPP_file, "code/vmreg.inline.hpp");
AD.addInclude(AD._CPP_file, "gc/shared/collectedHeap.inline.hpp");
- AD.addInclude(AD._CPP_file, "oops/compiledICHolder.hpp");
AD.addInclude(AD._CPP_file, "oops/compressedOops.hpp");
AD.addInclude(AD._CPP_file, "oops/markWord.hpp");
AD.addInclude(AD._CPP_file, "oops/method.hpp");
diff --git a/src/hotspot/share/asm/assembler.hpp b/src/hotspot/share/asm/assembler.hpp
index 7b7dbd4ede7dd..a533b96384468 100644
--- a/src/hotspot/share/asm/assembler.hpp
+++ b/src/hotspot/share/asm/assembler.hpp
@@ -359,6 +359,7 @@ class AbstractAssembler : public ResourceObj {
}
static bool is_uimm12(uint64_t x) { return is_uimm(x, 12); }
+ static bool is_uimm32(uint64_t x) { return is_uimm(x, 32); }
// Accessors
CodeSection* code_section() const { return _code_section; }
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index 7a0a31abf597f..5b1e113f15d10 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "asm/codeBuffer.hpp"
+#include "code/compiledIC.hpp"
#include "code/oopRecorder.inline.hpp"
#include "compiler/disassembler.hpp"
#include "logging/log.hpp"
diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp
index 9ef1ed1f37ca4..48476bedfe2f6 100644
--- a/src/hotspot/share/asm/codeBuffer.hpp
+++ b/src/hotspot/share/asm/codeBuffer.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -209,7 +209,7 @@ class CodeSection {
}
void set_locs_point(address pc) {
assert(pc >= locs_point(), "relocation addr may not decrease");
- assert(allocates2(pc), "relocation addr must be in this section");
+ assert(allocates2(pc), "relocation addr " INTPTR_FORMAT " must be in this section from " INTPTR_FORMAT " to " INTPTR_FORMAT, p2i(pc), p2i(_start), p2i(_limit));
_locs_point = pc;
}
diff --git a/src/hotspot/share/asm/codeBuffer.inline.hpp b/src/hotspot/share/asm/codeBuffer.inline.hpp
index 838447ad882dc..06ec9174b34bb 100644
--- a/src/hotspot/share/asm/codeBuffer.inline.hpp
+++ b/src/hotspot/share/asm/codeBuffer.inline.hpp
@@ -48,7 +48,7 @@ bool emit_shared_stubs_to_interp(CodeBuffer* cb, SharedStubToInterpRequests* sha
shared_stub_to_interp_requests->sort(by_shared_method);
MacroAssembler masm(cb);
for (int i = 0; i < shared_stub_to_interp_requests->length();) {
- address stub = __ start_a_stub(CompiledStaticCall::to_interp_stub_size());
+ address stub = __ start_a_stub(CompiledDirectCall::to_interp_stub_size());
if (stub == nullptr) {
return false;
}
diff --git a/src/hotspot/share/c1/c1_Canonicalizer.cpp b/src/hotspot/share/c1/c1_Canonicalizer.cpp
index dab11d4f70e40..f82262d8e529e 100644
--- a/src/hotspot/share/c1/c1_Canonicalizer.cpp
+++ b/src/hotspot/share/c1/c1_Canonicalizer.cpp
@@ -469,9 +469,11 @@ void Canonicalizer::do_CompareOp (CompareOp* x) {
void Canonicalizer::do_IfOp(IfOp* x) {
- // Caution: do not use do_Op2(x) here for now since
- // we map the condition to the op for now!
- move_const_to_right(x);
+ // Currently, Canonicalizer is only used by GraphBuilder,
+ // and IfOp is not created by GraphBuilder but only later
+ // when eliminating conditional expressions with CE_Eliminator,
+ // so this method will not be called.
+ ShouldNotReachHere();
}
@@ -842,9 +844,18 @@ void Canonicalizer::do_LookupSwitch(LookupSwitch* x) {
if (x->tag()->type()->is_constant()) {
int v = x->tag()->type()->as_IntConstant()->value();
BlockBegin* sux = x->default_sux();
- for (int i = 0; i < x->length(); i++) {
- if (v == x->key_at(i)) {
- sux = x->sux_at(i);
+ int low = 0;
+ int high = x->length() - 1;
+ while (low <= high) {
+ int mid = low + ((high - low) >> 1);
+ int key = x->key_at(mid);
+ if (key == v) {
+ sux = x->sux_at(mid);
+ break;
+ } else if (key > v) {
+ high = mid - 1;
+ } else {
+ low = mid + 1;
}
}
set_canonical(new Goto(sux, x->state_before(), is_safepoint(x, sux)));
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp
index 6e5fb99242c8c..396c83c6ab976 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.cpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp
@@ -523,7 +523,7 @@ inline bool BlockListBuilder::is_successor(BlockBegin* block, BlockBegin* sux) {
#ifndef PRODUCT
-int compare_depth_first(BlockBegin** a, BlockBegin** b) {
+static int compare_depth_first(BlockBegin** a, BlockBegin** b) {
return (*a)->depth_first_number() - (*b)->depth_first_number();
}
diff --git a/src/hotspot/share/c1/c1_IR.cpp b/src/hotspot/share/c1/c1_IR.cpp
index 628f14776154e..e375d16aafb18 100644
--- a/src/hotspot/share/c1/c1_IR.cpp
+++ b/src/hotspot/share/c1/c1_IR.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -306,22 +306,22 @@ void IR::eliminate_null_checks() {
}
}
-
-static int sort_pairs(BlockPair** a, BlockPair** b) {
- if ((*a)->from() == (*b)->from()) {
- return (*a)->to()->block_id() - (*b)->to()->block_id();
- } else {
- return (*a)->from()->block_id() - (*b)->from()->block_id();
- }
-}
-
-
+// The functionality of this class is to insert a new block between
+// the 'from' and 'to' block of a critical edge.
+// It first collects the block pairs, and then processes them.
+//
+// Some instructions may introduce more than one edge between two blocks.
+// By checking if the current 'to' block sets critical_edge_split_flag
+// (all new blocks set this flag) we can avoid repeated processing.
+// This is why BlockPair contains the index rather than the original 'to' block.
class CriticalEdgeFinder: public BlockClosure {
BlockPairList blocks;
- IR* _ir;
public:
- CriticalEdgeFinder(IR* ir): _ir(ir) {}
+ CriticalEdgeFinder(IR* ir) {
+ ir->iterate_preorder(this);
+ }
+
void block_do(BlockBegin* bb) {
BlockEnd* be = bb->end();
int nos = be->number_of_sux();
@@ -329,20 +329,22 @@ class CriticalEdgeFinder: public BlockClosure {
for (int i = 0; i < nos; i++) {
BlockBegin* sux = be->sux_at(i);
if (sux->number_of_preds() >= 2) {
- blocks.append(new BlockPair(bb, sux));
+ blocks.append(new BlockPair(bb, i));
}
}
}
}
void split_edges() {
- BlockPair* last_pair = nullptr;
- blocks.sort(sort_pairs);
for (int i = 0; i < blocks.length(); i++) {
BlockPair* pair = blocks.at(i);
- if (last_pair != nullptr && pair->is_same(last_pair)) continue;
BlockBegin* from = pair->from();
- BlockBegin* to = pair->to();
+ int index = pair->index();
+ BlockBegin* to = from->end()->sux_at(index);
+ if (to->is_set(BlockBegin::critical_edge_split_flag)) {
+ // inserted
+ continue;
+ }
BlockBegin* split = from->insert_block_between(to);
#ifndef PRODUCT
if ((PrintIR || PrintIR1) && Verbose) {
@@ -350,15 +352,12 @@ class CriticalEdgeFinder: public BlockClosure {
from->block_id(), to->block_id(), split->block_id());
}
#endif
- last_pair = pair;
}
}
};
void IR::split_critical_edges() {
CriticalEdgeFinder cef(this);
-
- iterate_preorder(&cef);
cef.split_edges();
}
diff --git a/src/hotspot/share/c1/c1_Instruction.cpp b/src/hotspot/share/c1/c1_Instruction.cpp
index b2d31e3c1a83f..431bcea42cb5b 100644
--- a/src/hotspot/share/c1/c1_Instruction.cpp
+++ b/src/hotspot/share/c1/c1_Instruction.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -586,6 +586,8 @@ void BlockBegin::substitute_sux(BlockBegin* old_sux, BlockBegin* new_sux) {
// of the inserted block, without recomputing the values of the other blocks
// in the CFG. Therefore the value of "depth_first_number" in BlockBegin becomes meaningless.
BlockBegin* BlockBegin::insert_block_between(BlockBegin* sux) {
+ assert(!sux->is_set(critical_edge_split_flag), "sanity check");
+
int bci = sux->bci();
// critical edge splitting may introduce a goto after a if and array
// bound check elimination may insert a predicate between the if and
diff --git a/src/hotspot/share/c1/c1_Instruction.hpp b/src/hotspot/share/c1/c1_Instruction.hpp
index 363f4f6e561f1..8f7fd698e7944 100644
--- a/src/hotspot/share/c1/c1_Instruction.hpp
+++ b/src/hotspot/share/c1/c1_Instruction.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2423,15 +2423,11 @@ LEAF(MemBar, Instruction)
class BlockPair: public CompilationResourceObj {
private:
BlockBegin* _from;
- BlockBegin* _to;
+ int _index; // sux index of 'to' block
public:
- BlockPair(BlockBegin* from, BlockBegin* to): _from(from), _to(to) {}
+ BlockPair(BlockBegin* from, int index): _from(from), _index(index) {}
BlockBegin* from() const { return _from; }
- BlockBegin* to() const { return _to; }
- bool is_same(BlockBegin* from, BlockBegin* to) const { return _from == from && _to == to; }
- bool is_same(BlockPair* p) const { return _from == p->from() && _to == p->to(); }
- void set_to(BlockBegin* b) { _to = b; }
- void set_from(BlockBegin* b) { _from = b; }
+ int index() const { return _index; }
};
typedef GrowableArray BlockPairList;
diff --git a/src/hotspot/share/c1/c1_LIRAssembler.cpp b/src/hotspot/share/c1/c1_LIRAssembler.cpp
index 1744aa4448137..51fb851d00c0e 100644
--- a/src/hotspot/share/c1/c1_LIRAssembler.cpp
+++ b/src/hotspot/share/c1/c1_LIRAssembler.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -606,13 +606,14 @@ void LIR_Assembler::emit_op0(LIR_Op0* op) {
Unimplemented();
break;
- case lir_std_entry:
+ case lir_std_entry: {
// init offsets
offsets()->set_value(CodeOffsets::OSR_Entry, _masm->offset());
- _masm->align(CodeEntryAlignment);
if (needs_icache(compilation()->method())) {
- check_icache();
+ int offset = check_icache();
+ offsets()->set_value(CodeOffsets::Entry, offset);
}
+ _masm->align(CodeEntryAlignment);
offsets()->set_value(CodeOffsets::Verified_Entry, _masm->offset());
_masm->verified_entry(compilation()->directive()->BreakAtExecuteOption);
if (needs_clinit_barrier_on_entry(compilation()->method())) {
@@ -621,6 +622,7 @@ void LIR_Assembler::emit_op0(LIR_Op0* op) {
build_frame();
offsets()->set_value(CodeOffsets::Frame_Complete, _masm->offset());
break;
+ }
case lir_osr_entry:
offsets()->set_value(CodeOffsets::OSR_Entry, _masm->offset());
@@ -841,8 +843,6 @@ void LIR_Assembler::verify_oop_map(CodeEmitInfo* info) {
if (v.is_oop()) {
VMReg r = v.reg();
if (!r->is_stack()) {
- stringStream st;
- st.print("bad oop %s at %d", r->as_Register()->name(), _masm->offset());
_masm->verify_oop(r->as_Register());
} else {
_masm->verify_stack_oop(r->reg2stack() * VMRegImpl::stack_slot_size);
diff --git a/src/hotspot/share/c1/c1_LinearScan.cpp b/src/hotspot/share/c1/c1_LinearScan.cpp
index 9e9195a0d60d0..a4d955e52a004 100644
--- a/src/hotspot/share/c1/c1_LinearScan.cpp
+++ b/src/hotspot/share/c1/c1_LinearScan.cpp
@@ -1446,12 +1446,12 @@ int LinearScan::interval_cmp(Interval** a, Interval** b) {
}
}
-#ifndef PRODUCT
-int interval_cmp(Interval* const& l, Interval* const& r) {
+#ifdef ASSERT
+static int interval_cmp(Interval* const& l, Interval* const& r) {
return l->from() - r->from();
}
-bool find_interval(Interval* interval, IntervalArray* intervals) {
+static bool find_interval(Interval* interval, IntervalArray* intervals) {
bool found;
int idx = intervals->find_sorted(interval, found);
@@ -2303,11 +2303,11 @@ void assert_no_register_values(GrowableArray* values) {
}
}
-void assert_equal(Location l1, Location l2) {
+static void assert_equal(Location l1, Location l2) {
assert(l1.where() == l2.where() && l1.type() == l2.type() && l1.offset() == l2.offset(), "");
}
-void assert_equal(ScopeValue* v1, ScopeValue* v2) {
+static void assert_equal(ScopeValue* v1, ScopeValue* v2) {
if (v1->is_location()) {
assert(v2->is_location(), "");
assert_equal(((LocationValue*)v1)->location(), ((LocationValue*)v2)->location());
@@ -2328,12 +2328,12 @@ void assert_equal(ScopeValue* v1, ScopeValue* v2) {
}
}
-void assert_equal(MonitorValue* m1, MonitorValue* m2) {
+static void assert_equal(MonitorValue* m1, MonitorValue* m2) {
assert_equal(m1->owner(), m2->owner());
assert_equal(m1->basic_lock(), m2->basic_lock());
}
-void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) {
+static void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) {
assert(d1->scope() == d2->scope(), "not equal");
assert(d1->bci() == d2->bci(), "not equal");
@@ -2375,7 +2375,7 @@ void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) {
}
}
-void check_stack_depth(CodeEmitInfo* info, int stack_end) {
+static void check_stack_depth(CodeEmitInfo* info, int stack_end) {
if (info->stack()->bci() != SynchronizationEntryBCI && !info->scope()->method()->is_native()) {
Bytecodes::Code code = info->scope()->method()->java_code_at_bci(info->stack()->bci());
switch (code) {
diff --git a/src/hotspot/share/c1/c1_MacroAssembler.hpp b/src/hotspot/share/c1/c1_MacroAssembler.hpp
index 6a8304bd405fa..1e193ce086961 100644
--- a/src/hotspot/share/c1/c1_MacroAssembler.hpp
+++ b/src/hotspot/share/c1/c1_MacroAssembler.hpp
@@ -38,7 +38,6 @@ class C1_MacroAssembler: public MacroAssembler {
//----------------------------------------------------
void explicit_null_check(Register base);
- void inline_cache_check(Register receiver, Register iCache);
void build_frame(int frame_size_in_bytes, int bang_size_in_bytes);
void remove_frame(int frame_size_in_bytes);
diff --git a/src/hotspot/share/c1/c1_Optimizer.cpp b/src/hotspot/share/c1/c1_Optimizer.cpp
index 50009f9425ad0..dd428a5895bc4 100644
--- a/src/hotspot/share/c1/c1_Optimizer.cpp
+++ b/src/hotspot/share/c1/c1_Optimizer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -335,7 +335,7 @@ void Optimizer::eliminate_conditional_expressions() {
}
// This removes others' relation to block, but doesn't empty block's lists
-void disconnect_from_graph(BlockBegin* block) {
+static void disconnect_from_graph(BlockBegin* block) {
for (int p = 0; p < block->number_of_preds(); p++) {
BlockBegin* pred = block->pred_at(p);
int idx;
@@ -714,6 +714,8 @@ class NullCheckEliminator: public ValueVisitor {
void handle_Phi (Phi* x);
void handle_ProfileCall (ProfileCall* x);
void handle_ProfileReturnType (ProfileReturnType* x);
+ void handle_Constant (Constant* x);
+ void handle_IfOp (IfOp* x);
};
@@ -728,7 +730,7 @@ class NullCheckEliminator: public ValueVisitor {
// that in for safety, otherwise should think more about it.
void NullCheckVisitor::do_Phi (Phi* x) { nce()->handle_Phi(x); }
void NullCheckVisitor::do_Local (Local* x) {}
-void NullCheckVisitor::do_Constant (Constant* x) { /* FIXME: handle object constants */ }
+void NullCheckVisitor::do_Constant (Constant* x) { nce()->handle_Constant(x); }
void NullCheckVisitor::do_LoadField (LoadField* x) { nce()->handle_AccessField(x); }
void NullCheckVisitor::do_StoreField (StoreField* x) { nce()->handle_AccessField(x); }
void NullCheckVisitor::do_ArrayLength (ArrayLength* x) { nce()->handle_ArrayLength(x); }
@@ -739,7 +741,7 @@ void NullCheckVisitor::do_ArithmeticOp (ArithmeticOp* x) { if (x->can_trap(
void NullCheckVisitor::do_ShiftOp (ShiftOp* x) {}
void NullCheckVisitor::do_LogicOp (LogicOp* x) {}
void NullCheckVisitor::do_CompareOp (CompareOp* x) {}
-void NullCheckVisitor::do_IfOp (IfOp* x) {}
+void NullCheckVisitor::do_IfOp (IfOp* x) { nce()->handle_IfOp(x); }
void NullCheckVisitor::do_Convert (Convert* x) {}
void NullCheckVisitor::do_NullCheck (NullCheck* x) { nce()->handle_NullCheck(x); }
void NullCheckVisitor::do_TypeCast (TypeCast* x) {}
@@ -882,7 +884,9 @@ void NullCheckEliminator::iterate_one(BlockBegin* block) {
// visiting instructions which are references in other blocks or
// visiting instructions more than once.
mark_visitable(instr);
- if (instr->is_pinned() || instr->can_trap() || (instr->as_NullCheck() != nullptr)) {
+ if (instr->is_pinned() || instr->can_trap() || (instr->as_NullCheck() != nullptr)
+ || (instr->as_Constant() != nullptr && instr->as_Constant()->type()->is_object())
+ || (instr->as_IfOp() != nullptr)) {
mark_visited(instr);
instr->input_values_do(this);
instr->visit(&_visitor);
@@ -1198,6 +1202,28 @@ void NullCheckEliminator::handle_ProfileReturnType(ProfileReturnType* x) {
x->set_needs_null_check(!set_contains(x->ret()));
}
+void NullCheckEliminator::handle_Constant(Constant *x) {
+ ObjectType* ot = x->type()->as_ObjectType();
+ if (ot != nullptr && ot->is_loaded()) {
+ ObjectConstant* oc = ot->as_ObjectConstant();
+ if (oc == nullptr || !oc->value()->is_null_object()) {
+ set_put(x);
+ if (PrintNullCheckElimination) {
+ tty->print_cr("Constant %d is non-null", x->id());
+ }
+ }
+ }
+}
+
+void NullCheckEliminator::handle_IfOp(IfOp *x) {
+ if (x->type()->is_object() && set_contains(x->tval()) && set_contains(x->fval())) {
+ set_put(x);
+ if (PrintNullCheckElimination) {
+ tty->print_cr("IfOp %d is non-null", x->id());
+ }
+ }
+}
+
void Optimizer::eliminate_null_checks() {
ResourceMark rm;
diff --git a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
index 4af9f29f263a4..256f8190b50dc 100644
--- a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
+++ b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
@@ -421,8 +421,11 @@ void RangeCheckEliminator::add_access_indexed_info(InstructionList &indices, int
aii->_max = idx;
aii->_list = new AccessIndexedList();
} else if (idx >= aii->_min && idx <= aii->_max) {
- remove_range_check(ai);
- return;
+ // Guard against underflow/overflow (see 'range_cond' check in RangeCheckEliminator::in_block_motion)
+ if (aii->_max < 0 || (aii->_max + min_jint) <= aii->_min) {
+ remove_range_check(ai);
+ return;
+ }
}
aii->_min = MIN2(aii->_min, idx);
aii->_max = MAX2(aii->_max, idx);
@@ -465,9 +468,9 @@ void RangeCheckEliminator::in_block_motion(BlockBegin *block, AccessIndexedList
}
}
} else {
- int last_integer = 0;
+ jint last_integer = 0;
Instruction *last_instruction = index;
- int base = 0;
+ jint base = 0;
ArithmeticOp *ao = index->as_ArithmeticOp();
while (ao != nullptr && (ao->x()->as_Constant() || ao->y()->as_Constant()) && (ao->op() == Bytecodes::_iadd || ao->op() == Bytecodes::_isub)) {
@@ -479,12 +482,12 @@ void RangeCheckEliminator::in_block_motion(BlockBegin *block, AccessIndexedList
}
if (c) {
- int value = c->type()->as_IntConstant()->value();
+ jint value = c->type()->as_IntConstant()->value();
if (value != min_jint) {
if (ao->op() == Bytecodes::_isub) {
value = -value;
}
- base += value;
+ base = java_add(base, value);
last_integer = base;
last_instruction = other;
}
@@ -506,12 +509,12 @@ void RangeCheckEliminator::in_block_motion(BlockBegin *block, AccessIndexedList
assert(info != nullptr, "Info must not be null");
// if idx < 0, max > 0, max + idx may fall between 0 and
- // length-1 and if min < 0, min + idx may overflow and be >=
+ // length-1 and if min < 0, min + idx may underflow/overflow and be >=
// 0. The predicate wouldn't trigger but some accesses could
// be with a negative index. This test guarantees that for the
// min and max value that are kept the predicate can't let
// some incorrect accesses happen.
- bool range_cond = (info->_max < 0 || info->_max + min_jint <= info->_min);
+ bool range_cond = (info->_max < 0 || (info->_max + min_jint) <= info->_min);
// Generate code only if more than 2 range checks can be eliminated because of that.
// 2 because at least 2 comparisons are done
@@ -859,7 +862,7 @@ void RangeCheckEliminator::process_access_indexed(BlockBegin *loop_header, Block
);
remove_range_check(ai);
- } else if (_optimistic && loop_header) {
+ } else if (false && _optimistic && loop_header) {
assert(ai->array(), "Array must not be null!");
assert(ai->index(), "Index must not be null!");
diff --git a/src/hotspot/share/cds/archiveHeapWriter.cpp b/src/hotspot/share/cds/archiveHeapWriter.cpp
index 3e7d42dd8f157..131b43307d5c6 100644
--- a/src/hotspot/share/cds/archiveHeapWriter.cpp
+++ b/src/hotspot/share/cds/archiveHeapWriter.cpp
@@ -44,7 +44,7 @@
#if INCLUDE_G1GC
#include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#endif
#if INCLUDE_CDS_JAVA_HEAP
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index c2bfec3dfeb18..4c3b9fe018ee4 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -71,7 +71,7 @@
#include "utilities/ostream.hpp"
#if INCLUDE_G1GC
#include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#endif
# include
@@ -1664,9 +1664,9 @@ void FileMapInfo::close() {
/*
* Same as os::map_memory() but also pretouches if AlwaysPreTouch is enabled.
*/
-char* map_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec, MEMFLAGS flags = mtNone) {
+static char* map_memory(int fd, const char* file_name, size_t file_offset,
+ char *addr, size_t bytes, bool read_only,
+ bool allow_exec, MEMFLAGS flags = mtNone) {
char* mem = os::map_memory(fd, file_name, file_offset, addr, bytes,
AlwaysPreTouch ? false : read_only,
allow_exec, flags);
@@ -1690,9 +1690,12 @@ bool FileMapInfo::remap_shared_readonly_as_readwrite() {
return false;
}
char *addr = r->mapped_base();
- char *base = os::remap_memory(_fd, _full_path, r->file_offset(),
- addr, size, false /* !read_only */,
- r->allow_exec());
+ // This path should not be reached for Windows; see JDK-8222379.
+ assert(WINDOWS_ONLY(false) NOT_WINDOWS(true), "Don't call on Windows");
+ // Replace old mapping with new one that is writable.
+ char *base = os::map_memory(_fd, _full_path, r->file_offset(),
+ addr, size, false /* !read_only */,
+ r->allow_exec());
close();
// These have to be errors because the shared region is now unmapped.
if (base == nullptr) {
@@ -2156,7 +2159,7 @@ bool FileMapInfo::map_heap_region_impl() {
if (_heap_pointers_need_patching) {
char* bitmap_base = map_bitmap_region();
- if (bitmap_base == NULL) {
+ if (bitmap_base == nullptr) {
log_info(cds)("CDS heap cannot be used because bitmap region cannot be mapped");
dealloc_heap_region();
unmap_region(MetaspaceShared::hp);
diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp
index fedbc5841b682..ae5d92fedff36 100644
--- a/src/hotspot/share/cds/metaspaceShared.cpp
+++ b/src/hotspot/share/cds/metaspaceShared.cpp
@@ -1327,6 +1327,9 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma
release_reserved_spaces(total_space_rs, archive_space_rs, class_space_rs);
return nullptr;
}
+ // NMT: fix up the space tags
+ MemTracker::record_virtual_memory_type(archive_space_rs.base(), mtClassShared);
+ MemTracker::record_virtual_memory_type(class_space_rs.base(), mtClass);
} else {
if (use_archive_base_addr && base_address != nullptr) {
total_space_rs = ReservedSpace(total_range_size, archive_space_alignment,
@@ -1356,16 +1359,13 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma
(size_t)archive_space_alignment);
class_space_rs = total_space_rs.last_part(ccs_begin_offset);
MemTracker::record_virtual_memory_split_reserved(total_space_rs.base(), total_space_rs.size(),
- ccs_begin_offset);
+ ccs_begin_offset, mtClassShared, mtClass);
}
assert(is_aligned(archive_space_rs.base(), archive_space_alignment), "Sanity");
assert(is_aligned(archive_space_rs.size(), archive_space_alignment), "Sanity");
assert(is_aligned(class_space_rs.base(), class_space_alignment), "Sanity");
assert(is_aligned(class_space_rs.size(), class_space_alignment), "Sanity");
- // NMT: fix up the space tags
- MemTracker::record_virtual_memory_type(archive_space_rs.base(), mtClassShared);
- MemTracker::record_virtual_memory_type(class_space_rs.base(), mtClass);
return archive_space_rs.base();
diff --git a/src/hotspot/share/ci/ciEnv.cpp b/src/hotspot/share/ci/ciEnv.cpp
index 75d46abd5178f..b7db2a6860f4f 100644
--- a/src/hotspot/share/ci/ciEnv.cpp
+++ b/src/hotspot/share/ci/ciEnv.cpp
@@ -121,7 +121,6 @@ ciEnv::ciEnv(CompileTask* task)
_oop_recorder = nullptr;
_debug_info = nullptr;
_dependencies = nullptr;
- _failure_reason = nullptr;
_inc_decompile_count_on_failure = true;
_compilable = MethodCompilable;
_break_at_compile = false;
@@ -250,7 +249,6 @@ ciEnv::ciEnv(Arena* arena) : _ciEnv_arena(mtCompiler) {
_oop_recorder = nullptr;
_debug_info = nullptr;
_dependencies = nullptr;
- _failure_reason = nullptr;
_inc_decompile_count_on_failure = true;
_compilable = MethodCompilable_never;
_break_at_compile = false;
@@ -1233,9 +1231,9 @@ int ciEnv::num_inlined_bytecodes() const {
// ------------------------------------------------------------------
// ciEnv::record_failure()
void ciEnv::record_failure(const char* reason) {
- if (_failure_reason == nullptr) {
+ if (_failure_reason.get() == nullptr) {
// Record the first failure reason.
- _failure_reason = reason;
+ _failure_reason.set(reason);
}
}
@@ -1265,7 +1263,7 @@ void ciEnv::record_method_not_compilable(const char* reason, bool all_tiers) {
_compilable = new_compilable;
// Reset failure reason; this one is more important.
- _failure_reason = nullptr;
+ _failure_reason.clear();
record_failure(reason);
}
}
diff --git a/src/hotspot/share/ci/ciEnv.hpp b/src/hotspot/share/ci/ciEnv.hpp
index fc12cc0259b71..5d3a61f809e7c 100644
--- a/src/hotspot/share/ci/ciEnv.hpp
+++ b/src/hotspot/share/ci/ciEnv.hpp
@@ -33,6 +33,7 @@
#include "code/exceptionHandlerTable.hpp"
#include "compiler/compiler_globals.hpp"
#include "compiler/compilerThread.hpp"
+#include "compiler/cHeapStringHolder.hpp"
#include "oops/methodData.hpp"
#include "runtime/javaThread.hpp"
@@ -57,7 +58,7 @@ class ciEnv : StackObj {
OopRecorder* _oop_recorder;
DebugInformationRecorder* _debug_info;
Dependencies* _dependencies;
- const char* _failure_reason;
+ CHeapStringHolder _failure_reason;
bool _inc_decompile_count_on_failure;
int _compilable;
bool _break_at_compile;
@@ -319,10 +320,10 @@ class ciEnv : StackObj {
// This is true if the compilation is not going to produce code.
// (It is reasonable to retry failed compilations.)
- bool failing() const { return _failure_reason != nullptr; }
+ bool failing() const { return _failure_reason.get() != nullptr; }
// Reason this compilation is failing, such as "too many basic blocks".
- const char* failure_reason() const { return _failure_reason; }
+ const char* failure_reason() const { return _failure_reason.get(); }
// Return state of appropriate compatibility
int compilable() { return _compilable; }
diff --git a/src/hotspot/share/ci/ciField.cpp b/src/hotspot/share/ci/ciField.cpp
index 0d4487fdf2fb0..0eddd87200ae0 100644
--- a/src/hotspot/share/ci/ciField.cpp
+++ b/src/hotspot/share/ci/ciField.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,7 +35,7 @@
#include "oops/oop.inline.hpp"
#include "runtime/fieldDescriptor.inline.hpp"
#include "runtime/handles.inline.hpp"
-#include "runtime/reflectionUtils.hpp"
+#include "runtime/reflection.hpp"
// ciField
//
diff --git a/src/hotspot/share/ci/ciMethodData.cpp b/src/hotspot/share/ci/ciMethodData.cpp
index dc7082c15ca9d..5abb342d03119 100644
--- a/src/hotspot/share/ci/ciMethodData.cpp
+++ b/src/hotspot/share/ci/ciMethodData.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,6 +31,7 @@
#include "memory/allocation.inline.hpp"
#include "memory/resourceArea.hpp"
#include "oops/klass.inline.hpp"
+#include "oops/methodData.inline.hpp"
#include "runtime/deoptimization.hpp"
#include "utilities/copy.hpp"
@@ -87,8 +88,18 @@ class PrepareExtraDataClosure : public CleanExtraDataClosure {
// Preparation finished iff all Methods* were already cached.
return true;
}
- // Holding locks through safepoints is bad practice.
- MutexUnlocker mu(_mdo->extra_data_lock());
+ // We are currently holding the extra_data_lock and ensuring
+ // no safepoint breaks the lock.
+ _mdo->check_extra_data_locked();
+
+ // We now want to cache some method data. This could cause a safepoint.
+ // We temporarily release the lock and allow safepoints, and revert that
+ // at the end of the scope. This is safe, since we currently do not hold
+ // any extra_method_data: finish is called only after clean_extra_data,
+ // and the outer scope that first aquired the lock should not hold any
+ // extra_method_data while cleaning is performed, as the offsets can change.
+ MutexUnlocker mu(_mdo->extra_data_lock(), Mutex::_no_safepoint_check_flag);
+
for (int i = 0; i < _uncached_methods.length(); ++i) {
if (has_safepointed()) {
// The metadata in the growable array might contain stale
@@ -123,7 +134,10 @@ void ciMethodData::prepare_metadata() {
void ciMethodData::load_remaining_extra_data() {
MethodData* mdo = get_MethodData();
- MutexLocker ml(mdo->extra_data_lock());
+
+ // Lock to read ProfileData, and ensure lock is not unintentionally broken by a safepoint
+ MutexLocker ml(mdo->extra_data_lock(), Mutex::_no_safepoint_check_flag);
+
// Deferred metadata cleaning due to concurrent class unloading.
prepare_metadata();
// After metadata preparation, there is no stale metadata,
@@ -562,6 +576,9 @@ void ciMethodData::set_argument_type(int bci, int i, ciKlass* k) {
VM_ENTRY_MARK;
MethodData* mdo = get_MethodData();
if (mdo != nullptr) {
+ // Lock to read ProfileData, and ensure lock is not broken by a safepoint
+ MutexLocker ml(mdo->extra_data_lock(), Mutex::_no_safepoint_check_flag);
+
ProfileData* data = mdo->bci_to_data(bci);
if (data != nullptr) {
if (data->is_CallTypeData()) {
@@ -586,6 +603,9 @@ void ciMethodData::set_return_type(int bci, ciKlass* k) {
VM_ENTRY_MARK;
MethodData* mdo = get_MethodData();
if (mdo != nullptr) {
+ // Lock to read ProfileData, and ensure lock is not broken by a safepoint
+ MutexLocker ml(mdo->extra_data_lock(), Mutex::_no_safepoint_check_flag);
+
ProfileData* data = mdo->bci_to_data(bci);
if (data != nullptr) {
if (data->is_CallTypeData()) {
diff --git a/src/hotspot/share/classfile/altHashing.cpp b/src/hotspot/share/classfile/altHashing.cpp
index 158a8a232a7b4..1d43d6ebf1ed0 100644
--- a/src/hotspot/share/classfile/altHashing.cpp
+++ b/src/hotspot/share/classfile/altHashing.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -120,7 +120,7 @@ static void halfsiphash_init64(uint32_t v[4], uint64_t seed) {
v[1] ^= 0xee;
}
-uint32_t halfsiphash_finish32(uint32_t v[4], int rounds) {
+static uint32_t halfsiphash_finish32(uint32_t v[4], int rounds) {
v[2] ^= 0xff;
halfsiphash_rounds(v, rounds);
return (v[1] ^ v[3]);
diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp
index 43aa82b67f85f..40a7d178f7561 100644
--- a/src/hotspot/share/classfile/classLoader.cpp
+++ b/src/hotspot/share/classfile/classLoader.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -134,7 +134,8 @@ ClassPathEntry* ClassLoader::_last_module_path_entry = nullptr;
#endif
// helper routines
-bool string_starts_with(const char* str, const char* str_to_find) {
+#if INCLUDE_CDS
+static bool string_starts_with(const char* str, const char* str_to_find) {
size_t str_len = strlen(str);
size_t str_to_find_len = strlen(str_to_find);
if (str_to_find_len > str_len) {
@@ -142,6 +143,7 @@ bool string_starts_with(const char* str, const char* str_to_find) {
}
return (strncmp(str, str_to_find, str_to_find_len) == 0);
}
+#endif
static const char* get_jimage_version_string() {
static char version_string[10] = "";
@@ -1009,8 +1011,8 @@ const char* ClassLoader::file_name_for_class_name(const char* class_name,
return file_name;
}
-ClassPathEntry* find_first_module_cpe(ModuleEntry* mod_entry,
- const GrowableArray* const module_list) {
+static ClassPathEntry* find_first_module_cpe(ModuleEntry* mod_entry,
+ const GrowableArray* const module_list) {
int num_of_entries = module_list->length();
const Symbol* class_module_name = mod_entry->name();
@@ -1028,17 +1030,14 @@ ClassPathEntry* find_first_module_cpe(ModuleEntry* mod_entry,
}
-// Search either the patch-module or exploded build entries for class.
+// Search the module list for the class file stream based on the file name and java package
ClassFileStream* ClassLoader::search_module_entries(JavaThread* current,
const GrowableArray* const module_list,
- const char* const class_name,
+ PackageEntry* pkg_entry, // Java package entry derived from the class name
const char* const file_name) {
ClassFileStream* stream = nullptr;
- // Find the class' defining module in the boot loader's module entry table
- TempNewSymbol class_name_symbol = SymbolTable::new_symbol(class_name);
- TempNewSymbol pkg_name = package_from_class_name(class_name_symbol);
- PackageEntry* pkg_entry = get_package_entry(pkg_name, ClassLoaderData::the_null_class_loader_data());
+ // Find the defining module in the boot loader's module entry table
ModuleEntry* mod_entry = (pkg_entry != nullptr) ? pkg_entry->module() : nullptr;
// If the module system has not defined java.base yet, then
@@ -1083,7 +1082,7 @@ ClassFileStream* ClassLoader::search_module_entries(JavaThread* current,
}
// Called by the boot classloader to load classes
-InstanceKlass* ClassLoader::load_class(Symbol* name, bool search_append_only, TRAPS) {
+InstanceKlass* ClassLoader::load_class(Symbol* name, PackageEntry* pkg_entry, bool search_append_only, TRAPS) {
assert(name != nullptr, "invariant");
ResourceMark rm(THREAD);
@@ -1130,7 +1129,7 @@ InstanceKlass* ClassLoader::load_class(Symbol* name, bool search_append_only, TR
// is not supported with UseSharedSpaces, we can never come here during dynamic dumping.
assert(!CDSConfig::is_dumping_dynamic_archive(), "sanity");
if (!CDSConfig::is_dumping_static_archive()) {
- stream = search_module_entries(THREAD, _patch_mod_entries, class_name, file_name);
+ stream = search_module_entries(THREAD, _patch_mod_entries, pkg_entry, file_name);
}
}
@@ -1142,7 +1141,7 @@ InstanceKlass* ClassLoader::load_class(Symbol* name, bool search_append_only, TR
} else {
// Exploded build - attempt to locate class in its defining module's location.
assert(_exploded_entries != nullptr, "No exploded build entries present");
- stream = search_module_entries(THREAD, _exploded_entries, class_name, file_name);
+ stream = search_module_entries(THREAD, _exploded_entries, pkg_entry, file_name);
}
}
@@ -1355,7 +1354,7 @@ void ClassLoader::initialize(TRAPS) {
setup_bootstrap_search_path(THREAD);
}
-char* lookup_vm_resource(JImageFile *jimage, const char *jimage_version, const char *path) {
+static char* lookup_vm_resource(JImageFile *jimage, const char *jimage_version, const char *path) {
jlong size;
JImageLocationRef location = (*JImageFindResource)(jimage, "java.base", jimage_version, path, &size);
if (location == 0)
diff --git a/src/hotspot/share/classfile/classLoader.hpp b/src/hotspot/share/classfile/classLoader.hpp
index 10373dbcf9f91..c8ea47435dd93 100644
--- a/src/hotspot/share/classfile/classLoader.hpp
+++ b/src/hotspot/share/classfile/classLoader.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -292,14 +292,14 @@ class ClassLoader: AllStatic {
// Add a module's exploded directory to the boot loader's exploded module build list
static void add_to_exploded_build_list(JavaThread* current, Symbol* module_name);
- // Attempt load of individual class from either the patched or exploded modules build lists
+ // Search the module list for the class file stream based on the file name and java package
static ClassFileStream* search_module_entries(JavaThread* current,
const GrowableArray* const module_list,
- const char* const class_name,
+ PackageEntry* pkg_entry, // Java package entry derived from the class name
const char* const file_name);
// Load individual .class file
- static InstanceKlass* load_class(Symbol* class_name, bool search_append_only, TRAPS);
+ static InstanceKlass* load_class(Symbol* class_name, PackageEntry* pkg_entry, bool search_append_only, TRAPS);
// If the specified package has been loaded by the system, then returns
// the name of the directory or ZIP file that the package was loaded from.
diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp
index d383fff3eb13e..00687d21a7e14 100644
--- a/src/hotspot/share/classfile/classLoaderData.cpp
+++ b/src/hotspot/share/classfile/classLoaderData.cpp
@@ -838,10 +838,10 @@ OopHandle ClassLoaderData::add_handle(Handle h) {
void ClassLoaderData::remove_handle(OopHandle h) {
assert(!is_unloading(), "Do not remove a handle for a CLD that is unloading");
- oop* ptr = h.ptr_raw();
- if (ptr != nullptr) {
- assert(_handles.owner_of(ptr), "Got unexpected handle " PTR_FORMAT, p2i(ptr));
- NativeAccess<>::oop_store(ptr, oop(nullptr));
+ if (!h.is_empty()) {
+ assert(_handles.owner_of(h.ptr_raw()),
+ "Got unexpected handle " PTR_FORMAT, p2i(h.ptr_raw()));
+ h.replace(oop(nullptr));
}
}
@@ -1003,7 +1003,11 @@ void ClassLoaderData::print_on(outputStream* out) const {
_holder.print_on(out);
out->print_cr("");
}
- out->print_cr(" - class loader " INTPTR_FORMAT, p2i(_class_loader.ptr_raw()));
+ if (!_unloading) {
+ out->print_cr(" - class loader " INTPTR_FORMAT, p2i(_class_loader.peek()));
+ } else {
+ out->print_cr(" - class loader ");
+ }
out->print_cr(" - metaspace " INTPTR_FORMAT, p2i(_metaspace));
out->print_cr(" - unloading %s", _unloading ? "true" : "false");
out->print_cr(" - class mirror holder %s", _has_class_mirror_holder ? "true" : "false");
diff --git a/src/hotspot/share/classfile/loaderConstraints.cpp b/src/hotspot/share/classfile/loaderConstraints.cpp
index 4206fc10d4f39..99d0c07ed42d4 100644
--- a/src/hotspot/share/classfile/loaderConstraints.cpp
+++ b/src/hotspot/share/classfile/loaderConstraints.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -296,8 +296,8 @@ void LoaderConstraintTable::purge_loader_constraints() {
_loader_constraint_table->unlink(&purge);
}
-void log_ldr_constraint_msg(Symbol* class_name, const char* reason,
- ClassLoaderData* loader1, ClassLoaderData* loader2) {
+static void log_ldr_constraint_msg(Symbol* class_name, const char* reason,
+ ClassLoaderData* loader1, ClassLoaderData* loader2) {
LogTarget(Info, class, loader, constraints) lt;
if (lt.is_enabled()) {
ResourceMark rm;
@@ -387,7 +387,7 @@ bool LoaderConstraintTable::add_entry(Symbol* class_name,
} else if (pp1 == nullptr) {
pp2->extend_loader_constraint(class_name, loader1, klass);
} else if (pp2 == nullptr) {
- pp1->extend_loader_constraint(class_name, loader1, klass);
+ pp1->extend_loader_constraint(class_name, loader2, klass);
} else {
merge_loader_constraints(class_name, pp1, pp2, klass);
}
diff --git a/src/hotspot/share/classfile/moduleEntry.cpp b/src/hotspot/share/classfile/moduleEntry.cpp
index 24beecdcaf72f..9d2a76e9e51b3 100644
--- a/src/hotspot/share/classfile/moduleEntry.cpp
+++ b/src/hotspot/share/classfile/moduleEntry.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -481,7 +481,7 @@ void ModuleEntry::init_as_archived_entry() {
if (_location != nullptr) {
_location = ArchiveBuilder::get_buffered_symbol(_location);
}
- JFR_ONLY(set_trace_id(0));// re-init at runtime
+ JFR_ONLY(set_trace_id(0);) // re-init at runtime
ArchivePtrMarker::mark_pointer((address*)&_reads);
ArchivePtrMarker::mark_pointer((address*)&_version);
diff --git a/src/hotspot/share/classfile/modules.cpp b/src/hotspot/share/classfile/modules.cpp
index bd4be93b86877..4664d9565d353 100644
--- a/src/hotspot/share/classfile/modules.cpp
+++ b/src/hotspot/share/classfile/modules.cpp
@@ -1,5 +1,5 @@
/*
-* Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+* Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -259,7 +259,7 @@ static void define_javabase_module(Handle module_handle, jstring version, jstrin
}
// Caller needs ResourceMark.
-void throw_dup_pkg_exception(const char* module_name, PackageEntry* package, TRAPS) {
+static void throw_dup_pkg_exception(const char* module_name, PackageEntry* package, TRAPS) {
const char* package_name = package->name()->as_C_string();
if (package->module()->is_named()) {
THROW_MSG(vmSymbols::java_lang_IllegalStateException(),
diff --git a/src/hotspot/share/classfile/packageEntry.cpp b/src/hotspot/share/classfile/packageEntry.cpp
index 1b315bc24be16..97cd041f3f103 100644
--- a/src/hotspot/share/classfile/packageEntry.cpp
+++ b/src/hotspot/share/classfile/packageEntry.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -252,7 +252,7 @@ void PackageEntry::init_as_archived_entry() {
_module = ModuleEntry::get_archived_entry(_module);
_qualified_exports = (GrowableArray*)archived_qualified_exports;
_defined_by_cds_in_class_path = 0;
- JFR_ONLY(set_trace_id(0)); // re-init at runtime
+ JFR_ONLY(set_trace_id(0);) // re-init at runtime
ArchivePtrMarker::mark_pointer((address*)&_name);
ArchivePtrMarker::mark_pointer((address*)&_module);
diff --git a/src/hotspot/share/classfile/placeholders.cpp b/src/hotspot/share/classfile/placeholders.cpp
index 1bb5f87870417..a6a86473ea794 100644
--- a/src/hotspot/share/classfile/placeholders.cpp
+++ b/src/hotspot/share/classfile/placeholders.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -198,8 +198,8 @@ void PlaceholderEntry::set_supername(Symbol* supername) {
// All threads examining the placeholder table must hold the
// SystemDictionary_lock, so we don't need special precautions
// on store ordering here.
-PlaceholderEntry* add_entry(Symbol* class_name, ClassLoaderData* loader_data,
- Symbol* supername){
+static PlaceholderEntry* add_entry(Symbol* class_name, ClassLoaderData* loader_data,
+ Symbol* supername){
assert_locked_or_safepoint(SystemDictionary_lock);
assert(class_name != nullptr, "adding nullptr obj");
@@ -213,7 +213,7 @@ PlaceholderEntry* add_entry(Symbol* class_name, ClassLoaderData* loader_data,
}
// Remove a placeholder object.
-void remove_entry(Symbol* class_name, ClassLoaderData* loader_data) {
+static void remove_entry(Symbol* class_name, ClassLoaderData* loader_data) {
assert_locked_or_safepoint(SystemDictionary_lock);
PlaceholderKey key(class_name, loader_data);
diff --git a/src/hotspot/share/classfile/stringTable.cpp b/src/hotspot/share/classfile/stringTable.cpp
index 9e96340d82b9c..be2971288ef12 100644
--- a/src/hotspot/share/classfile/stringTable.cpp
+++ b/src/hotspot/share/classfile/stringTable.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -122,7 +122,7 @@ volatile bool _alt_hash = false;
static bool _rehashed = false;
static uint64_t _alt_hash_seed = 0;
-unsigned int hash_string(const jchar* s, int len, bool useAlt) {
+static unsigned int hash_string(const jchar* s, int len, bool useAlt) {
return useAlt ?
AltHashing::halfsiphash_32(_alt_hash_seed, s, len) :
java_lang_String::hash_code(s, len);
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index e635c825633f6..30538926b4179 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -212,13 +212,13 @@ void SystemDictionary::set_platform_loader(ClassLoaderData *cld) {
// ----------------------------------------------------------------------------
// Parallel class loading check
-bool is_parallelCapable(Handle class_loader) {
+static bool is_parallelCapable(Handle class_loader) {
if (class_loader.is_null()) return true;
return java_lang_ClassLoader::parallelCapable(class_loader());
}
// ----------------------------------------------------------------------------
// ParallelDefineClass flag does not apply to bootclass loader
-bool is_parallelDefine(Handle class_loader) {
+static bool is_parallelDefine(Handle class_loader) {
if (class_loader.is_null()) return false;
if (AllowParallelDefineClass && java_lang_ClassLoader::parallelCapable(class_loader())) {
return true;
@@ -280,7 +280,7 @@ Symbol* SystemDictionary::class_name_symbol(const char* name, Symbol* exception,
#ifdef ASSERT
// Used to verify that class loading succeeded in adding k to the dictionary.
-void verify_dictionary_entry(Symbol* class_name, InstanceKlass* k) {
+static void verify_dictionary_entry(Symbol* class_name, InstanceKlass* k) {
MutexLocker mu(SystemDictionary_lock);
ClassLoaderData* loader_data = k->class_loader_data();
Dictionary* dictionary = loader_data->dictionary();
@@ -1111,14 +1111,13 @@ InstanceKlass* SystemDictionary::load_shared_lambda_proxy_class(InstanceKlass* i
if (loaded_ik != nullptr) {
assert(shared_nest_host->is_same_class_package(ik),
"lambda proxy class and its nest host must be in the same package");
+ // The lambda proxy class and its nest host have the same class loader and class loader data,
+ // as verified in SystemDictionaryShared::add_lambda_proxy_class()
+ assert(shared_nest_host->class_loader() == class_loader(), "mismatched class loader");
+ assert(shared_nest_host->class_loader_data() == class_loader_data(class_loader), "mismatched class loader data");
+ ik->set_nest_host(shared_nest_host);
}
- // The lambda proxy class and its nest host have the same class loader and class loader data,
- // as verified in SystemDictionaryShared::add_lambda_proxy_class()
- assert(shared_nest_host->class_loader() == class_loader(), "mismatched class loader");
- assert(shared_nest_host->class_loader_data() == class_loader_data(class_loader), "mismatched class loader data");
- ik->set_nest_host(shared_nest_host);
-
return loaded_ik;
}
@@ -1277,7 +1276,7 @@ InstanceKlass* SystemDictionary::load_instance_class_impl(Symbol* class_name, Ha
if (k == nullptr) {
// Use VM class loader
PerfTraceTime vmtimer(ClassLoader::perf_sys_classload_time());
- k = ClassLoader::load_class(class_name, search_only_bootloader_append, CHECK_NULL);
+ k = ClassLoader::load_class(class_name, pkg_entry, search_only_bootloader_append, CHECK_NULL);
}
// find_or_define_instance_class may return a different InstanceKlass
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp
index 297483526846c..44d7da5c4a4f1 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.cpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1348,7 +1348,7 @@ void SystemDictionaryShared::update_shared_entry(InstanceKlass* k, int id) {
info->_id = id;
}
-const char* class_loader_name_for_shared(Klass* k) {
+static const char* class_loader_name_for_shared(Klass* k) {
assert(k != nullptr, "Sanity");
assert(k->is_shared(), "Must be");
assert(k->is_instance_klass(), "Must be");
diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp
index 4aa95aff0e105..743bd9d06ba6c 100644
--- a/src/hotspot/share/classfile/verifier.cpp
+++ b/src/hotspot/share/classfile/verifier.cpp
@@ -2257,11 +2257,12 @@ void ClassVerifier::verify_switch(
"low must be less than or equal to high in tableswitch");
return;
}
- keys = high - low + 1;
- if (keys < 0) {
+ int64_t keys64 = ((int64_t)high - low) + 1;
+ if (keys64 > 65535) { // Max code length
verify_error(ErrorContext::bad_code(bci), "too many keys in tableswitch");
return;
}
+ keys = (int)keys64;
delta = 1;
} else {
keys = (int)Bytes::get_Java_u4(aligned_bcp + jintSize);
diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp
index 6c2eecf7ae7ee..6e72b4cbac31a 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.cpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.cpp
@@ -222,7 +222,6 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
case vmIntrinsics::_compareToLU:
case vmIntrinsics::_compareToUL:
case vmIntrinsics::_equalsL:
- case vmIntrinsics::_equalsU:
case vmIntrinsics::_equalsC:
case vmIntrinsics::_vectorizedHashCode:
case vmIntrinsics::_getCharStringU:
@@ -532,7 +531,6 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
if (!SpecialStringIndexOf) return true;
break;
case vmIntrinsics::_equalsL:
- case vmIntrinsics::_equalsU:
if (!SpecialStringEquals) return true;
break;
case vmIntrinsics::_vectorizedHashCode:
diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp
index c9bc4acbeff42..6104fb5683b3b 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.hpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -402,7 +402,6 @@ class methodHandle;
do_signature(indexOfI_signature, "([BI[BII)I") \
do_signature(indexOfChar_signature, "([BIII)I") \
do_intrinsic(_equalsL, java_lang_StringLatin1,equals_name, equalsB_signature, F_S) \
- do_intrinsic(_equalsU, java_lang_StringUTF16, equals_name, equalsB_signature, F_S) \
\
do_intrinsic(_isDigit, java_lang_CharacterDataLatin1, isDigit_name, int_bool_signature, F_R) \
do_name( isDigit_name, "isDigit") \
@@ -596,8 +595,8 @@ class methodHandle;
do_intrinsic(_notifyJvmtiVThreadEnd, java_lang_VirtualThread, notifyJvmtiEnd_name, void_method_signature, F_RN) \
do_intrinsic(_notifyJvmtiVThreadMount, java_lang_VirtualThread, notifyJvmtiMount_name, bool_void_signature, F_RN) \
do_intrinsic(_notifyJvmtiVThreadUnmount, java_lang_VirtualThread, notifyJvmtiUnmount_name, bool_void_signature, F_RN) \
- do_intrinsic(_notifyJvmtiVThreadHideFrames, java_lang_VirtualThread, notifyJvmtiHideFrames_name, bool_void_signature, F_RN) \
- do_intrinsic(_notifyJvmtiVThreadDisableSuspend, java_lang_VirtualThread, notifyJvmtiDisableSuspend_name, bool_void_signature, F_RN) \
+ do_intrinsic(_notifyJvmtiVThreadHideFrames, java_lang_VirtualThread, notifyJvmtiHideFrames_name, bool_void_signature, F_SN) \
+ do_intrinsic(_notifyJvmtiVThreadDisableSuspend, java_lang_VirtualThread, notifyJvmtiDisableSuspend_name, bool_void_signature, F_SN) \
\
/* support for UnsafeConstants */ \
do_class(jdk_internal_misc_UnsafeConstants, "jdk/internal/misc/UnsafeConstants") \
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index 5a2c9456468cf..d24e29c288d5c 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "code/codeBlob.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/relocInfo.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/disassembler.hpp"
@@ -236,9 +235,9 @@ const ImmutableOopMap* CodeBlob::oop_map_for_return_address(address return_addre
return _oop_maps->find_map_at_offset((intptr_t) return_address - (intptr_t) code_begin());
}
-void CodeBlob::print_code() {
+void CodeBlob::print_code_on(outputStream* st) {
ResourceMark m;
- Disassembler::decode(this, tty);
+ Disassembler::decode(this, st);
}
//----------------------------------------------------------------------------------------------------
@@ -649,11 +648,6 @@ void CodeBlob::dump_for_addr(address addr, outputStream* st, bool verbose) const
st->print_cr(INTPTR_FORMAT " is pointing to an (unnamed) stub routine", p2i(addr));
return;
}
- // the InlineCacheBuffer is using stubs generated into a buffer blob
- if (InlineCacheBuffer::contains(addr)) {
- st->print_cr(INTPTR_FORMAT " is pointing into InlineCacheBuffer", p2i(addr));
- return;
- }
VtableStub* v = VtableStubs::stub_containing(addr);
if (v != nullptr) {
st->print_cr(INTPTR_FORMAT " is at entry_point+%d in a vtable stub", p2i(addr), (int)(addr - v->entry_point()));
diff --git a/src/hotspot/share/code/codeBlob.hpp b/src/hotspot/share/code/codeBlob.hpp
index 6aae1acd84a35..56caa906ecb4b 100644
--- a/src/hotspot/share/code/codeBlob.hpp
+++ b/src/hotspot/share/code/codeBlob.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -240,7 +240,7 @@ class CodeBlob {
virtual void print_on(outputStream* st) const;
virtual void print_value_on(outputStream* st) const;
void dump_for_addr(address addr, outputStream* st, bool verbose) const;
- void print_code();
+ void print_code_on(outputStream* st);
// Print to stream, any comments associated with offset.
virtual void print_block_comment(outputStream* stream, address block_begin) const {
diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp
index bf8c1d84e71a0..d56de671a1352 100644
--- a/src/hotspot/share/code/codeCache.cpp
+++ b/src/hotspot/share/code/codeCache.cpp
@@ -29,7 +29,6 @@
#include "code/compiledIC.hpp"
#include "code/dependencies.hpp"
#include "code/dependencyContext.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "code/pcDesc.hpp"
#include "compiler/compilationPolicy.hpp"
@@ -913,23 +912,6 @@ void CodeCache::verify_clean_inline_caches() {
#endif
}
-void CodeCache::verify_icholder_relocations() {
-#ifdef ASSERT
- // make sure that we aren't leaking icholders
- int count = 0;
- FOR_ALL_HEAPS(heap) {
- FOR_ALL_BLOBS(cb, *heap) {
- CompiledMethod *nm = cb->as_compiled_method_or_null();
- if (nm != nullptr) {
- count += nm->verify_icholder_relocations();
- }
- }
- }
- assert(count + InlineCacheBuffer::pending_icholder_count() + CompiledICHolder::live_not_claimed_count() ==
- CompiledICHolder::live_count(), "must agree");
-#endif
-}
-
// Defer freeing of concurrently cleaned ExceptionCache entries until
// after a global handshake operation.
void CodeCache::release_exception_cache(ExceptionCache* entry) {
@@ -1748,6 +1730,10 @@ void CodeCache::print() {
void CodeCache::print_summary(outputStream* st, bool detailed) {
int full_count = 0;
+ julong total_used = 0;
+ julong total_max_used = 0;
+ julong total_free = 0;
+ julong total_size = 0;
FOR_ALL_HEAPS(heap_iterator) {
CodeHeap* heap = (*heap_iterator);
size_t total = (heap->high_boundary() - heap->low_boundary());
@@ -1756,10 +1742,17 @@ void CodeCache::print_summary(outputStream* st, bool detailed) {
} else {
st->print("CodeCache:");
}
+ size_t size = total/K;
+ size_t used = (total - heap->unallocated_capacity())/K;
+ size_t max_used = heap->max_allocated_capacity()/K;
+ size_t free = heap->unallocated_capacity()/K;
+ total_size += size;
+ total_used += used;
+ total_max_used += max_used;
+ total_free += free;
st->print_cr(" size=" SIZE_FORMAT "Kb used=" SIZE_FORMAT
"Kb max_used=" SIZE_FORMAT "Kb free=" SIZE_FORMAT "Kb",
- total/K, (total - heap->unallocated_capacity())/K,
- heap->max_allocated_capacity()/K, heap->unallocated_capacity()/K);
+ size, used, max_used, free);
if (detailed) {
st->print_cr(" bounds [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT "]",
@@ -1772,17 +1765,22 @@ void CodeCache::print_summary(outputStream* st, bool detailed) {
}
if (detailed) {
- st->print_cr(" total_blobs=" UINT32_FORMAT " nmethods=" UINT32_FORMAT
- " adapters=" UINT32_FORMAT,
- blob_count(), nmethod_count(), adapter_count());
- st->print_cr(" compilation: %s", CompileBroker::should_compile_new_jobs() ?
+ if (SegmentedCodeCache) {
+ st->print("CodeCache:");
+ st->print_cr(" size=" JULONG_FORMAT "Kb, used=" JULONG_FORMAT
+ "Kb, max_used=" JULONG_FORMAT "Kb, free=" JULONG_FORMAT "Kb",
+ total_size, total_used, total_max_used, total_free);
+ }
+ st->print_cr(" total_blobs=" UINT32_FORMAT ", nmethods=" UINT32_FORMAT
+ ", adapters=" UINT32_FORMAT ", full_count=" UINT32_FORMAT,
+ blob_count(), nmethod_count(), adapter_count(), full_count);
+ st->print_cr("Compilation: %s, stopped_count=%d, restarted_count=%d",
+ CompileBroker::should_compile_new_jobs() ?
"enabled" : Arguments::mode() == Arguments::_int ?
"disabled (interpreter mode)" :
- "disabled (not enough contiguous free space left)");
- st->print_cr(" stopped_count=%d, restarted_count=%d",
+ "disabled (not enough contiguous free space left)",
CompileBroker::get_total_compiler_stopped_count(),
CompileBroker::get_total_compiler_restarted_count());
- st->print_cr(" full_count=%d", full_count);
}
}
diff --git a/src/hotspot/share/code/codeCache.hpp b/src/hotspot/share/code/codeCache.hpp
index 103268c8ffcd1..d1c91727bf124 100644
--- a/src/hotspot/share/code/codeCache.hpp
+++ b/src/hotspot/share/code/codeCache.hpp
@@ -294,7 +294,6 @@ class CodeCache : AllStatic {
}
static void verify_clean_inline_caches();
- static void verify_icholder_relocations();
// Deoptimization
private:
diff --git a/src/hotspot/share/code/compiledIC.cpp b/src/hotspot/share/code/compiledIC.cpp
index 28c02d8578c55..250ef063a2a33 100644
--- a/src/hotspot/share/code/compiledIC.cpp
+++ b/src/hotspot/share/code/compiledIC.cpp
@@ -26,27 +26,19 @@
#include "code/codeBehaviours.hpp"
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "code/vtableStubs.hpp"
-#include "interpreter/interpreter.hpp"
-#include "interpreter/linkResolver.hpp"
-#include "memory/metadataFactory.hpp"
-#include "memory/oopFactory.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
+#include "oops/compressedKlass.hpp"
#include "oops/klass.inline.hpp"
#include "oops/method.inline.hpp"
-#include "oops/oop.inline.hpp"
-#include "oops/symbol.hpp"
+#include "runtime/atomic.hpp"
#include "runtime/continuationEntry.hpp"
#include "runtime/handles.inline.hpp"
-#include "runtime/icache.hpp"
-#include "runtime/safepoint.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/sharedRuntime.hpp"
-#include "runtime/stubRoutines.hpp"
#include "sanitizers/leak.hpp"
-#include "utilities/events.hpp"
// Every time a compiled IC is changed or its type is being accessed,
@@ -75,191 +67,175 @@ bool CompiledICLocker::is_safe(address code) {
return CompiledICProtectionBehaviour::current()->is_safe(cm);
}
-//-----------------------------------------------------------------------------
-// Low-level access to an inline cache. Private, since they might not be
-// MT-safe to use.
+CompiledICData::CompiledICData()
+ : _speculated_method(),
+ _speculated_klass(),
+ _itable_defc_klass(),
+ _itable_refc_klass(),
+ _is_initialized() {}
-void* CompiledIC::cached_value() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- assert (!is_optimized(), "an optimized virtual call does not have a cached metadata");
-
- if (!is_in_transition_state()) {
- void* data = get_data();
- // If we let the metadata value here be initialized to zero...
- assert(data != nullptr || Universe::non_oop_word() == nullptr,
- "no raw nulls in CompiledIC metadatas, because of patching races");
- return (data == (void*)Universe::non_oop_word()) ? nullptr : data;
+// Inline cache callsite info is initialized once the first time it is resolved
+void CompiledICData::initialize(CallInfo* call_info, Klass* receiver_klass) {
+ _speculated_method = call_info->selected_method();
+ if (UseCompressedClassPointers) {
+ _speculated_klass = (uintptr_t)CompressedKlassPointers::encode_not_null(receiver_klass);
} else {
- return InlineCacheBuffer::cached_value_for((CompiledIC *)this);
+ _speculated_klass = (uintptr_t)receiver_klass;
}
+ if (call_info->call_kind() == CallInfo::itable_call) {
+ _itable_defc_klass = call_info->resolved_method()->method_holder();
+ _itable_refc_klass = call_info->resolved_klass();
+ }
+ _is_initialized = true;
}
+bool CompiledICData::is_speculated_klass_unloaded() const {
+ return is_initialized() && _speculated_klass == 0;
+}
-void CompiledIC::internal_set_ic_destination(address entry_point, bool is_icstub, void* cache, bool is_icholder) {
- assert(entry_point != nullptr, "must set legal entry point");
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- assert (!is_optimized() || cache == nullptr, "an optimized virtual call does not have a cached metadata");
- assert (cache == nullptr || cache != (Metadata*)badOopVal, "invalid metadata");
-
- assert(!is_icholder || is_icholder_entry(entry_point), "must be");
-
- // Don't use ic_destination for this test since that forwards
- // through ICBuffer instead of returning the actual current state of
- // the CompiledIC.
- if (is_icholder_entry(_call->destination())) {
- // When patching for the ICStub case the cached value isn't
- // overwritten until the ICStub copied into the CompiledIC during
- // the next safepoint. Make sure that the CompiledICHolder* is
- // marked for release at this point since it won't be identifiable
- // once the entry point is overwritten.
- InlineCacheBuffer::queue_for_release((CompiledICHolder*)get_data());
+void CompiledICData::clean_metadata() {
+ if (!is_initialized() || is_speculated_klass_unloaded()) {
+ return;
}
- if (TraceCompiledIC) {
- tty->print(" ");
- print_compiled_ic();
- tty->print(" changing destination to " INTPTR_FORMAT, p2i(entry_point));
- if (!is_optimized()) {
- tty->print(" changing cached %s to " INTPTR_FORMAT, is_icholder ? "icholder" : "metadata", p2i((address)cache));
- }
- if (is_icstub) {
- tty->print(" (icstub)");
- }
- tty->cr();
+ // GC cleaning doesn't need to change the state of the inline cache,
+ // only nuke stale speculated metadata if it gets unloaded. If the
+ // inline cache is monomorphic, the unverified entries will miss, and
+ // subsequent miss handlers will upgrade the callsite to megamorphic,
+ // which makes sense as it obviously is megamorphic then.
+ if (!speculated_klass()->is_loader_alive()) {
+ Atomic::store(&_speculated_klass, (uintptr_t)0);
+ Atomic::store(&_speculated_method, (Method*)nullptr);
}
-#ifdef ASSERT
- {
- CodeBlob* cb = CodeCache::find_blob(_call->instruction_address());
- assert(cb != nullptr && cb->is_compiled(), "must be compiled");
- }
-#endif
- _call->set_destination_mt_safe(entry_point);
+ assert(_speculated_method == nullptr || _speculated_method->method_holder()->is_loader_alive(),
+ "Speculated method is not unloaded despite class being unloaded");
+}
- if (is_optimized() || is_icstub) {
- // Optimized call sites don't have a cache value and ICStub call
- // sites only change the entry point. Changing the value in that
- // case could lead to MT safety issues.
- assert(cache == nullptr, "must be null");
+void CompiledICData::metadata_do(MetadataClosure* cl) {
+ if (!is_initialized()) {
return;
}
- if (cache == nullptr) cache = Universe::non_oop_word();
-
- set_data((intptr_t)cache);
-}
-
-
-void CompiledIC::set_ic_destination(ICStub* stub) {
- internal_set_ic_destination(stub->code_begin(), true, nullptr, false);
+ if (!is_speculated_klass_unloaded()) {
+ cl->do_metadata(_speculated_method);
+ cl->do_metadata(speculated_klass());
+ }
+ if (_itable_refc_klass != nullptr) {
+ cl->do_metadata(_itable_refc_klass);
+ }
+ if (_itable_defc_klass != nullptr) {
+ cl->do_metadata(_itable_defc_klass);
+ }
}
+Klass* CompiledICData::speculated_klass() const {
+ if (is_speculated_klass_unloaded()) {
+ return nullptr;
+ }
-
-address CompiledIC::ic_destination() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- if (!is_in_transition_state()) {
- return _call->destination();
+ if (UseCompressedClassPointers) {
+ return CompressedKlassPointers::decode_not_null((narrowKlass)_speculated_klass);
} else {
- return InlineCacheBuffer::ic_destination_for((CompiledIC *)this);
+ return (Klass*)_speculated_klass;
}
}
+//-----------------------------------------------------------------------------
+// High-level access to an inline cache. Guaranteed to be MT-safe.
-bool CompiledIC::is_in_transition_state() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- return InlineCacheBuffer::contains(_call->destination());;
+CompiledICData* CompiledIC::data() const {
+ return _data;
}
+CompiledICData* data_from_reloc_iter(RelocIterator* iter) {
+ assert(iter->type() == relocInfo::virtual_call_type, "wrong reloc. info");
+
+ virtual_call_Relocation* r = iter->virtual_call_reloc();
+ NativeMovConstReg* value = nativeMovConstReg_at(r->cached_value());
+
+ return (CompiledICData*)value->data();
+}
-bool CompiledIC::is_icholder_call() const {
+CompiledIC::CompiledIC(RelocIterator* iter)
+ : _method(iter->code()),
+ _data(data_from_reloc_iter(iter)),
+ _call(nativeCall_at(iter->addr()))
+{
+ assert(_method != nullptr, "must pass compiled method");
+ assert(_method->contains(iter->addr()), "must be in compiled method");
assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- return !_is_optimized && is_icholder_entry(ic_destination());
}
-// Returns native address of 'call' instruction in inline-cache. Used by
-// the InlineCacheBuffer when it needs to find the stub.
-address CompiledIC::stub_address() const {
- assert(is_in_transition_state(), "should only be called when we are in a transition state");
- return _call->destination();
+CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr) {
+ address call_site = nativeCall_before(return_addr)->instruction_address();
+ return CompiledIC_at(nm, call_site);
}
-// Clears the IC stub if the compiled IC is in transition state
-void CompiledIC::clear_ic_stub() {
- if (is_in_transition_state()) {
- ICStub* stub = ICStub_from_destination_address(stub_address());
- stub->clear();
- }
+CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site) {
+ RelocIterator iter(nm, call_site, call_site + 1);
+ iter.next();
+ return CompiledIC_at(&iter);
}
-//-----------------------------------------------------------------------------
-// High-level access to an inline cache. Guaranteed to be MT-safe.
+CompiledIC* CompiledIC_at(Relocation* call_reloc) {
+ address call_site = call_reloc->addr();
+ CompiledMethod* cm = CodeCache::find_blob(call_reloc->addr())->as_compiled_method();
+ return CompiledIC_at(cm, call_site);
+}
-void CompiledIC::initialize_from_iter(RelocIterator* iter) {
- assert(iter->addr() == _call->instruction_address(), "must find ic_call");
+CompiledIC* CompiledIC_at(RelocIterator* reloc_iter) {
+ CompiledIC* c_ic = new CompiledIC(reloc_iter);
+ c_ic->verify();
+ return c_ic;
+}
- if (iter->type() == relocInfo::virtual_call_type) {
- virtual_call_Relocation* r = iter->virtual_call_reloc();
- _is_optimized = false;
- _value = _call->get_load_instruction(r);
- } else {
- assert(iter->type() == relocInfo::opt_virtual_call_type, "must be a virtual call");
- _is_optimized = true;
- _value = nullptr;
+void CompiledIC::ensure_initialized(CallInfo* call_info, Klass* receiver_klass) {
+ if (!_data->is_initialized()) {
+ _data->initialize(call_info, receiver_klass);
}
}
-CompiledIC::CompiledIC(CompiledMethod* cm, NativeCall* call)
- : _method(cm)
-{
- _call = _method->call_wrapper_at((address) call);
- address ic_call = _call->instruction_address();
-
- assert(ic_call != nullptr, "ic_call address must be set");
- assert(cm != nullptr, "must pass compiled method");
- assert(cm->contains(ic_call), "must be in compiled method");
-
- // Search for the ic_call at the given address.
- RelocIterator iter(cm, ic_call, ic_call+1);
- bool ret = iter.next();
- assert(ret == true, "relocInfo must exist at this address");
- assert(iter.addr() == ic_call, "must find ic_call");
-
- initialize_from_iter(&iter);
+void CompiledIC::set_to_clean() {
+ log_debug(inlinecache)("IC@" INTPTR_FORMAT ": set to clean", p2i(_call->instruction_address()));
+ _call->set_destination_mt_safe(SharedRuntime::get_resolve_virtual_call_stub());
}
-CompiledIC::CompiledIC(RelocIterator* iter)
- : _method(iter->code())
-{
- _call = _method->call_wrapper_at(iter->addr());
- address ic_call = _call->instruction_address();
+void CompiledIC::set_to_monomorphic() {
+ assert(data()->is_initialized(), "must be initialized");
+ Method* method = data()->speculated_method();
+ CompiledMethod* code = method->code();
+ address entry;
+ bool to_compiled = code != nullptr && code->is_in_use() && !code->is_unloading();
+
+ if (to_compiled) {
+ entry = code->entry_point();
+ } else {
+ entry = method->get_c2i_unverified_entry();
+ }
- CompiledMethod* nm = iter->code();
- assert(ic_call != nullptr, "ic_call address must be set");
- assert(nm != nullptr, "must pass compiled method");
- assert(nm->contains(ic_call), "must be in compiled method");
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to %s: %s",
+ p2i(_call->instruction_address()),
+ to_compiled ? "compiled" : "interpreter",
+ method->print_value_string());
- initialize_from_iter(iter);
+ _call->set_destination_mt_safe(entry);
}
-// This function may fail for two reasons: either due to running out of vtable
-// stubs, or due to running out of IC stubs in an attempted transition to a
-// transitional state. The needs_ic_stub_refill value will be set if the failure
-// was due to running out of IC stubs, in which case the caller will refill IC
-// stubs and retry.
-bool CompiledIC::set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecode,
- bool& needs_ic_stub_refill, TRAPS) {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- assert(!is_optimized(), "cannot set an optimized virtual call to megamorphic");
- assert(is_call_to_compiled() || is_call_to_interpreted(), "going directly to megamorphic?");
+void CompiledIC::set_to_megamorphic(CallInfo* call_info) {
+ assert(data()->is_initialized(), "must be initialized");
address entry;
- if (call_info->call_kind() == CallInfo::itable_call) {
- assert(bytecode == Bytecodes::_invokeinterface, "");
+ if (call_info->call_kind() == CallInfo::direct_call) {
+ // C1 sometimes compiles a callsite before the target method is loaded, resulting in
+ // dynamically bound callsites that should really be statically bound. However, the
+ // target method might not have a vtable or itable. We just wait for better code to arrive
+ return;
+ } else if (call_info->call_kind() == CallInfo::itable_call) {
int itable_index = call_info->itable_index();
entry = VtableStubs::find_itable_stub(itable_index);
if (entry == nullptr) {
- return false;
+ return;
}
#ifdef ASSERT
int index = call_info->resolved_method()->itable_index();
@@ -267,401 +243,151 @@ bool CompiledIC::set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecod
InstanceKlass* k = call_info->resolved_method()->method_holder();
assert(k->verify_itable_index(itable_index), "sanity check");
#endif //ASSERT
- CompiledICHolder* holder = new CompiledICHolder(call_info->resolved_method()->method_holder(),
- call_info->resolved_klass(), false);
- holder->claim();
- if (!InlineCacheBuffer::create_transition_stub(this, holder, entry)) {
- delete holder;
- needs_ic_stub_refill = true;
- return false;
- }
- // LSan appears unable to follow malloc-based memory consistently when embedded as an immediate
- // in generated machine code. So we have to ignore it.
- LSAN_IGNORE_OBJECT(holder);
} else {
- assert(call_info->call_kind() == CallInfo::vtable_call, "either itable or vtable");
+ assert(call_info->call_kind() == CallInfo::vtable_call, "what else?");
// Can be different than selected_method->vtable_index(), due to package-private etc.
int vtable_index = call_info->vtable_index();
assert(call_info->resolved_klass()->verify_vtable_index(vtable_index), "sanity check");
entry = VtableStubs::find_vtable_stub(vtable_index);
if (entry == nullptr) {
- return false;
+ return;
}
- if (!InlineCacheBuffer::create_transition_stub(this, nullptr, entry)) {
- needs_ic_stub_refill = true;
- return false;
- }
- }
-
- {
- ResourceMark rm;
- assert(call_info->selected_method() != nullptr, "Unexpected null selected method");
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": to megamorphic %s entry: " INTPTR_FORMAT,
- p2i(instruction_address()), call_info->selected_method()->print_value_string(), p2i(entry));
}
- // We can't check this anymore. With lazy deopt we could have already
- // cleaned this IC entry before we even return. This is possible if
- // we ran out of space in the inline cache buffer trying to do the
- // set_next and we safepointed to free up space. This is a benign
- // race because the IC entry was complete when we safepointed so
- // cleaning it immediately is harmless.
- // assert(is_megamorphic(), "sanity check");
- return true;
-}
-
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": to megamorphic %s entry: " INTPTR_FORMAT,
+ p2i(_call->instruction_address()), call_info->selected_method()->print_value_string(), p2i(entry));
-// true if destination is megamorphic stub
-bool CompiledIC::is_megamorphic() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- assert(!is_optimized(), "an optimized call cannot be megamorphic");
-
- // Cannot rely on cached_value. It is either an interface or a method.
- return VtableStubs::entry_point(ic_destination()) != nullptr;
+ _call->set_destination_mt_safe(entry);
+ assert(is_megamorphic(), "sanity check");
}
-bool CompiledIC::is_call_to_compiled() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
-
- CodeBlob* cb = CodeCache::find_blob(ic_destination());
- bool is_monomorphic = (cb != nullptr && cb->is_compiled());
- // Check that the cached_value is a klass for non-optimized monomorphic calls
- // This assertion is invalid for compiler1: a call that does not look optimized (no static stub) can be used
- // for calling directly to vep without using the inline cache (i.e., cached_value == nullptr).
- // For JVMCI this occurs because CHA is only used to improve inlining so call sites which could be optimized
- // virtuals because there are no currently loaded subclasses of a type are left as virtual call sites.
-#ifdef ASSERT
- CodeBlob* caller = CodeCache::find_blob(instruction_address());
- bool is_c1_or_jvmci_method = caller->is_compiled_by_c1() || caller->is_compiled_by_jvmci();
- assert( is_c1_or_jvmci_method ||
- !is_monomorphic ||
- is_optimized() ||
- (cached_metadata() != nullptr && cached_metadata()->is_klass()), "sanity check");
-#endif // ASSERT
- return is_monomorphic;
-}
+void CompiledIC::update(CallInfo* call_info, Klass* receiver_klass) {
+ // If this is the first time we fix the inline cache, we ensure it's initialized
+ ensure_initialized(call_info, receiver_klass);
+ if (is_megamorphic()) {
+ // Terminal state for the inline cache
+ return;
+ }
-bool CompiledIC::is_call_to_interpreted() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- // Call to interpreter if destination is either calling to a stub (if it
- // is optimized), or calling to an I2C blob
- bool is_call_to_interpreted = false;
- if (!is_optimized()) {
- CodeBlob* cb = CodeCache::find_blob(ic_destination());
- is_call_to_interpreted = (cb != nullptr && cb->is_adapter_blob());
- assert(!is_call_to_interpreted || (is_icholder_call() && cached_icholder() != nullptr), "sanity check");
+ if (is_speculated_klass(receiver_klass)) {
+ // If the speculated class matches the receiver klass, we can speculate that will
+ // continue to be the case with a monomorphic inline cache
+ set_to_monomorphic();
} else {
- // Check if we are calling into our own codeblob (i.e., to a stub)
- address dest = ic_destination();
-#ifdef ASSERT
- {
- _call->verify_resolve_call(dest);
- }
-#endif /* ASSERT */
- is_call_to_interpreted = _call->is_call_to_interpreted(dest);
+ // If the dynamic type speculation fails, we try to transform to a megamorphic state
+ // for the inline cache using stubs to dispatch in tables
+ set_to_megamorphic(call_info);
}
- return is_call_to_interpreted;
}
-bool CompiledIC::set_to_clean(bool in_use) {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- if (TraceInlineCacheClearing) {
- tty->print_cr("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address()));
- print();
- }
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address()));
-
- address entry = _call->get_resolve_call_stub(is_optimized());
-
- bool safe_transition = _call->is_safe_for_patching() || !in_use || is_optimized() || SafepointSynchronize::is_at_safepoint();
+bool CompiledIC::is_clean() const {
+ return destination() == SharedRuntime::get_resolve_virtual_call_stub();
+}
- if (safe_transition) {
- // Kill any leftover stub we might have too
- clear_ic_stub();
- if (is_optimized()) {
- set_ic_destination(entry);
- } else {
- set_ic_destination_and_value(entry, (void*)nullptr);
- }
- } else {
- // Unsafe transition - create stub.
- if (!InlineCacheBuffer::create_transition_stub(this, nullptr, entry)) {
- return false;
- }
- }
- // We can't check this anymore. With lazy deopt we could have already
- // cleaned this IC entry before we even return. This is possible if
- // we ran out of space in the inline cache buffer trying to do the
- // set_next and we safepointed to free up space. This is a benign
- // race because the IC entry was complete when we safepointed so
- // cleaning it immediately is harmless.
- // assert(is_clean(), "sanity check");
- return true;
+bool CompiledIC::is_monomorphic() const {
+ return !is_clean() && !is_megamorphic();
}
-bool CompiledIC::is_clean() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- bool is_clean = false;
- address dest = ic_destination();
- is_clean = dest == _call->get_resolve_call_stub(is_optimized());
- assert(!is_clean || is_optimized() || cached_value() == nullptr, "sanity check");
- return is_clean;
+bool CompiledIC::is_megamorphic() const {
+ return VtableStubs::entry_point(destination()) != nullptr;;
}
-bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- // Updating a cache to the wrong entry can cause bugs that are very hard
- // to track down - if cache entry gets invalid - we just clean it. In
- // this way it is always the same code path that is responsible for
- // updating and resolving an inline cache
- //
- // The above is no longer true. SharedRuntime::fixup_callers_callsite will change optimized
- // callsites. In addition ic_miss code will update a site to monomorphic if it determines
- // that an monomorphic call to the interpreter can now be monomorphic to compiled code.
- //
- // In both of these cases the only thing being modified is the jump/call target and these
- // transitions are mt_safe
-
- Thread *thread = Thread::current();
- if (info.to_interpreter()) {
- // Call to interpreter
- if (info.is_optimized() && is_optimized()) {
- assert(is_clean(), "unsafe IC path");
- // the call analysis (callee structure) specifies that the call is optimized
- // (either because of CHA or the static target is final)
- // At code generation time, this call has been emitted as static call
- // Call via stub
- assert(info.cached_metadata() != nullptr && info.cached_metadata()->is_method(), "sanity check");
- methodHandle method (thread, (Method*)info.cached_metadata());
- _call->set_to_interpreted(method, info);
-
- {
- ResourceMark rm(thread);
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to interpreter: %s",
- p2i(instruction_address()),
- method->print_value_string());
- }
- } else {
- // Call via method-klass-holder
- CompiledICHolder* holder = info.claim_cached_icholder();
- if (!InlineCacheBuffer::create_transition_stub(this, holder, info.entry())) {
- delete holder;
- return false;
- }
- // LSan appears unable to follow malloc-based memory consistently when embedded as an
- // immediate in generated machine code. So we have to ignore it.
- LSAN_IGNORE_OBJECT(holder);
- {
- ResourceMark rm(thread);
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to interpreter via icholder ", p2i(instruction_address()));
- }
- }
- } else {
- // Call to compiled code
- bool static_bound = info.is_optimized() || (info.cached_metadata() == nullptr);
-#ifdef ASSERT
- CodeBlob* cb = CodeCache::find_blob(info.entry());
- assert (cb != nullptr && cb->is_compiled(), "must be compiled!");
-#endif /* ASSERT */
-
- // This is MT safe if we come from a clean-cache and go through a
- // non-verified entry point
- bool safe = SafepointSynchronize::is_at_safepoint() ||
- (!is_in_transition_state() && (info.is_optimized() || static_bound || is_clean()));
-
- if (!safe) {
- if (!InlineCacheBuffer::create_transition_stub(this, info.cached_metadata(), info.entry())) {
- return false;
- }
- } else {
- if (is_optimized()) {
- set_ic_destination(info.entry());
- } else {
- set_ic_destination_and_value(info.entry(), info.cached_metadata());
- }
- }
+bool CompiledIC::is_speculated_klass(Klass* receiver_klass) {
+ return data()->speculated_klass() == receiver_klass;
+}
- {
- ResourceMark rm(thread);
- assert(info.cached_metadata() == nullptr || info.cached_metadata()->is_klass(), "must be");
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to compiled (rcvr klass = %s) %s",
- p2i(instruction_address()),
- (info.cached_metadata() != nullptr) ? ((Klass*)info.cached_metadata())->print_value_string() : "nullptr",
- (safe) ? "" : " via stub");
- }
- }
- // We can't check this anymore. With lazy deopt we could have already
- // cleaned this IC entry before we even return. This is possible if
- // we ran out of space in the inline cache buffer trying to do the
- // set_next and we safepointed to free up space. This is a benign
- // race because the IC entry was complete when we safepointed so
- // cleaning it immediately is harmless.
- // assert(is_call_to_compiled() || is_call_to_interpreted(), "sanity check");
- return true;
-}
-
-
-// is_optimized: Compiler has generated an optimized call (i.e. fixed, no inline cache)
-// static_bound: The call can be static bound. If it isn't also optimized, the property
-// wasn't provable at time of compilation. An optimized call will have any necessary
-// null check, while a static_bound won't. A static_bound (but not optimized) must
-// therefore use the unverified entry point.
-void CompiledIC::compute_monomorphic_entry(const methodHandle& method,
- Klass* receiver_klass,
- bool is_optimized,
- bool static_bound,
- bool caller_is_nmethod,
- CompiledICInfo& info,
- TRAPS) {
- CompiledMethod* method_code = method->code();
-
- address entry = nullptr;
- if (method_code != nullptr && method_code->is_in_use() && !method_code->is_unloading()) {
- assert(method_code->is_compiled(), "must be compiled");
- // Call to compiled code
- //
- // Note: the following problem exists with Compiler1:
- // - at compile time we may or may not know if the destination is final
- // - if we know that the destination is final (is_optimized), we will emit
- // an optimized virtual call (no inline cache), and need a Method* to make
- // a call to the interpreter
- // - if we don't know if the destination is final, we emit a standard
- // virtual call, and use CompiledICHolder to call interpreted code
- // (no static call stub has been generated)
- // - In the case that we here notice the call is static bound we
- // convert the call into what looks to be an optimized virtual call,
- // but we must use the unverified entry point (since there will be no
- // null check on a call when the target isn't loaded).
- // This causes problems when verifying the IC because
- // it looks vanilla but is optimized. Code in is_call_to_interpreted
- // is aware of this and weakens its asserts.
- if (is_optimized) {
- entry = method_code->verified_entry_point();
- } else {
- entry = method_code->entry_point();
- }
- }
- if (entry != nullptr) {
- // Call to near compiled code.
- info.set_compiled_entry(entry, is_optimized ? nullptr : receiver_klass, is_optimized);
- } else {
- if (is_optimized) {
- // Use stub entry
- info.set_interpreter_entry(method()->get_c2i_entry(), method());
- } else {
- // Use icholder entry
- assert(method_code == nullptr || method_code->is_compiled(), "must be compiled");
- CompiledICHolder* holder = new CompiledICHolder(method(), receiver_klass);
- info.set_icholder_entry(method()->get_c2i_unverified_entry(), holder);
- }
- }
- assert(info.is_optimized() == is_optimized, "must agree");
+// GC support
+void CompiledIC::clean_metadata() {
+ data()->clean_metadata();
}
+void CompiledIC::metadata_do(MetadataClosure* cl) {
+ data()->metadata_do(cl);
+}
-bool CompiledIC::is_icholder_entry(address entry) {
- CodeBlob* cb = CodeCache::find_blob(entry);
- if (cb == nullptr) {
- return false;
- }
- if (cb->is_adapter_blob()) {
- return true;
- } else if (cb->is_vtable_blob()) {
- return VtableStubs::is_icholder_entry(entry);
- }
- return false;
+#ifndef PRODUCT
+void CompiledIC::print() {
+ tty->print("Inline cache at " INTPTR_FORMAT ", calling " INTPTR_FORMAT " cached_value " INTPTR_FORMAT,
+ p2i(instruction_address()), p2i(destination()), p2i(data()));
+ tty->cr();
}
-bool CompiledIC::is_icholder_call_site(virtual_call_Relocation* call_site, const CompiledMethod* cm) {
- // This call site might have become stale so inspect it carefully.
- address dest = cm->call_wrapper_at(call_site->addr())->destination();
- return is_icholder_entry(dest);
+void CompiledIC::verify() {
+ _call->verify();
}
+#endif
// ----------------------------------------------------------------------------
-bool CompiledStaticCall::set_to_clean(bool in_use) {
+void CompiledDirectCall::set_to_clean() {
// in_use is unused but needed to match template function in CompiledMethod
assert(CompiledICLocker::is_safe(instruction_address()), "mt unsafe call");
// Reset call site
- set_destination_mt_safe(resolve_call_stub());
+ RelocIterator iter((nmethod*)nullptr, instruction_address(), instruction_address() + 1);
+ while (iter.next()) {
+ switch(iter.type()) {
+ case relocInfo::static_call_type:
+ _call->set_destination_mt_safe(SharedRuntime::get_resolve_static_call_stub());
+ break;
+ case relocInfo::opt_virtual_call_type:
+ _call->set_destination_mt_safe(SharedRuntime::get_resolve_opt_virtual_call_stub());
+ break;
+ default:
+ ShouldNotReachHere();
+ }
+ }
+ assert(is_clean(), "should be clean after cleaning");
- // Do not reset stub here: It is too expensive to call find_stub.
- // Instead, rely on caller (nmethod::clear_inline_caches) to clear
- // both the call and its stub.
- return true;
+ log_debug(inlinecache)("DC@" INTPTR_FORMAT ": set to clean", p2i(_call->instruction_address()));
}
-bool CompiledStaticCall::is_clean() const {
- return destination() == resolve_call_stub();
-}
+void CompiledDirectCall::set(const methodHandle& callee_method) {
+ CompiledMethod* code = callee_method->code();
+ CompiledMethod* caller = CodeCache::find_compiled(instruction_address());
-bool CompiledStaticCall::is_call_to_compiled() const {
- return CodeCache::contains(destination());
-}
+ bool to_interp_cont_enter = caller->method()->is_continuation_enter_intrinsic() &&
+ ContinuationEntry::is_interpreted_call(instruction_address());
-bool CompiledDirectStaticCall::is_call_to_interpreted() const {
- // It is a call to interpreted, if it calls to a stub. Hence, the destination
- // must be in the stub part of the nmethod that contains the call
- CompiledMethod* cm = CodeCache::find_compiled(instruction_address());
- return cm->stub_contains(destination());
-}
+ bool to_compiled = !to_interp_cont_enter && code != nullptr && code->is_in_use() && !code->is_unloading();
-void CompiledStaticCall::set_to_compiled(address entry) {
- {
- ResourceMark rm;
- log_trace(inlinecache)("%s@" INTPTR_FORMAT ": set_to_compiled " INTPTR_FORMAT,
- name(),
- p2i(instruction_address()),
- p2i(entry));
+ if (to_compiled) {
+ _call->set_destination_mt_safe(code->verified_entry_point());
+ assert(is_call_to_compiled(), "should be compiled after set to compiled");
+ } else {
+ // Patch call site to C2I adapter if code is deoptimized or unloaded.
+ // We also need to patch the static call stub to set the rmethod register
+ // to the callee_method so the c2i adapter knows how to build the frame
+ set_to_interpreted(callee_method, callee_method->get_c2i_entry());
+ assert(is_call_to_interpreted(), "should be interpreted after set to interpreted");
}
- // Call to compiled code
- assert(CodeCache::contains(entry), "wrong entry point");
- set_destination_mt_safe(entry);
+
+ log_trace(inlinecache)("DC@" INTPTR_FORMAT ": set to %s: %s: " INTPTR_FORMAT,
+ p2i(_call->instruction_address()),
+ to_compiled ? "compiled" : "interpreter",
+ callee_method->print_value_string(),
+ p2i(_call->destination()));
}
-void CompiledStaticCall::set(const StaticCallInfo& info) {
- assert(CompiledICLocker::is_safe(instruction_address()), "mt unsafe call");
- // Updating a cache to the wrong entry can cause bugs that are very hard
- // to track down - if cache entry gets invalid - we just clean it. In
- // this way it is always the same code path that is responsible for
- // updating and resolving an inline cache
- assert(is_clean(), "do not update a call entry - use clean");
-
- if (info._to_interpreter) {
- // Call to interpreted code
- set_to_interpreted(info.callee(), info.entry());
- } else {
- set_to_compiled(info.entry());
- }
+bool CompiledDirectCall::is_clean() const {
+ return destination() == SharedRuntime::get_resolve_static_call_stub() ||
+ destination() == SharedRuntime::get_resolve_opt_virtual_call_stub();
}
-// Compute settings for a CompiledStaticCall. Since we might have to set
-// the stub when calling to the interpreter, we need to return arguments.
-void CompiledStaticCall::compute_entry(const methodHandle& m, bool caller_is_nmethod, StaticCallInfo& info) {
- CompiledMethod* m_code = m->code();
- info._callee = m;
- if (m_code != nullptr && m_code->is_in_use() && !m_code->is_unloading()) {
- info._to_interpreter = false;
- info._entry = m_code->verified_entry_point();
- } else {
- // Callee is interpreted code. In any case entering the interpreter
- // puts a converter-frame on the stack to save arguments.
- assert(!m->is_method_handle_intrinsic(), "Compiled code should never call interpreter MH intrinsics");
- info._to_interpreter = true;
- info._entry = m()->get_c2i_entry();
- }
+bool CompiledDirectCall::is_call_to_interpreted() const {
+ // It is a call to interpreted, if it calls to a stub. Hence, the destination
+ // must be in the stub part of the nmethod that contains the call
+ CompiledMethod* cm = CodeCache::find_compiled(instruction_address());
+ return cm->stub_contains(destination());
}
-void CompiledStaticCall::compute_entry_for_continuation_entry(const methodHandle& m, StaticCallInfo& info) {
- if (ContinuationEntry::is_interpreted_call(instruction_address())) {
- info._to_interpreter = true;
- info._entry = m()->get_c2i_entry();
- }
+bool CompiledDirectCall::is_call_to_compiled() const {
+ CompiledMethod* caller = CodeCache::find_compiled(instruction_address());
+ CodeBlob* dest_cb = CodeCache::find_blob(destination());
+ return !caller->stub_contains(destination()) && dest_cb->is_compiled();
}
-address CompiledDirectStaticCall::find_stub_for(address instruction) {
+address CompiledDirectCall::find_stub_for(address instruction) {
// Find reloc. information containing this call-site
RelocIterator iter((nmethod*)nullptr, instruction);
while (iter.next()) {
@@ -673,8 +399,6 @@ address CompiledDirectStaticCall::find_stub_for(address instruction) {
// from the CompiledIC implementation
case relocInfo::opt_virtual_call_type:
return iter.opt_virtual_call_reloc()->static_stub();
- case relocInfo::poll_type:
- case relocInfo::poll_return_type: // A safepoint can't overlap a call.
default:
ShouldNotReachHere();
}
@@ -683,36 +407,13 @@ address CompiledDirectStaticCall::find_stub_for(address instruction) {
return nullptr;
}
-address CompiledDirectStaticCall::find_stub() {
- return CompiledDirectStaticCall::find_stub_for(instruction_address());
+address CompiledDirectCall::find_stub() {
+ return find_stub_for(instruction_address());
}
-address CompiledDirectStaticCall::resolve_call_stub() const {
- return SharedRuntime::get_resolve_static_call_stub();
-}
-
-//-----------------------------------------------------------------------------
-// Non-product mode code
#ifndef PRODUCT
-
-void CompiledIC::verify() {
- _call->verify();
- assert(is_clean() || is_call_to_compiled() || is_call_to_interpreted()
- || is_optimized() || is_megamorphic(), "sanity check");
-}
-
-void CompiledIC::print() {
- print_compiled_ic();
- tty->cr();
-}
-
-void CompiledIC::print_compiled_ic() {
- tty->print("Inline cache at " INTPTR_FORMAT ", calling %s " INTPTR_FORMAT " cached_value " INTPTR_FORMAT,
- p2i(instruction_address()), is_call_to_interpreted() ? "interpreted " : "", p2i(ic_destination()), p2i(is_optimized() ? nullptr : cached_value()));
-}
-
-void CompiledDirectStaticCall::print() {
- tty->print("static call at " INTPTR_FORMAT " -> ", p2i(instruction_address()));
+void CompiledDirectCall::print() {
+ tty->print("direct call at " INTPTR_FORMAT " to " INTPTR_FORMAT " -> ", p2i(instruction_address()), p2i(destination()));
if (is_clean()) {
tty->print("clean");
} else if (is_call_to_compiled()) {
@@ -723,9 +424,10 @@ void CompiledDirectStaticCall::print() {
tty->cr();
}
-void CompiledDirectStaticCall::verify_mt_safe(const methodHandle& callee, address entry,
- NativeMovConstReg* method_holder,
- NativeJump* jump) {
+void CompiledDirectCall::verify_mt_safe(const methodHandle& callee, address entry,
+ NativeMovConstReg* method_holder,
+ NativeJump* jump) {
+ _call->verify();
// A generated lambda form might be deleted from the Lambdaform
// cache in MethodTypeForm. If a jit compiled lambdaform method
// becomes not entrant and the cache access returns null, the new
@@ -743,4 +445,4 @@ void CompiledDirectStaticCall::verify_mt_safe(const methodHandle& callee, addres
|| old_method->is_old(), // may be race patching deoptimized nmethod due to redefinition.
"b) MT-unsafe modification of inline cache");
}
-#endif // !PRODUCT
+#endif
diff --git a/src/hotspot/share/code/compiledIC.hpp b/src/hotspot/share/code/compiledIC.hpp
index 17586fc57a05f..321bf280ed40a 100644
--- a/src/hotspot/share/code/compiledIC.hpp
+++ b/src/hotspot/share/code/compiledIC.hpp
@@ -27,42 +27,19 @@
#include "code/nativeInst.hpp"
#include "interpreter/linkResolver.hpp"
-#include "oops/compiledICHolder.hpp"
#include "runtime/safepointVerifiers.hpp"
//-----------------------------------------------------------------------------
// The CompiledIC represents a compiled inline cache.
//
-// In order to make patching of the inline cache MT-safe, we only allow the following
-// transitions (when not at a safepoint):
-//
-//
-// [1] --<-- Clean -->--- [1]
-// / (null) \
-// / \ /-<-\
-// / [2] \ / \
-// Interpreted ---------> Monomorphic | [3]
-// (CompiledICHolder*) (Klass*) |
-// \ / \ /
-// [4] \ / [4] \->-/
-// \->- Megamorphic -<-/
-// (CompiledICHolder*)
-//
-// The text in parentheses () refers to the value of the inline cache receiver (mov instruction)
-//
-// The numbers in square brackets refer to the kind of transition:
-// [1]: Initial fixup. Receiver it found from debug information
-// [2]: Compilation of a method
-// [3]: Recompilation of a method (note: only entry is changed. The Klass* must stay the same)
-// [4]: Inline cache miss. We go directly to megamorphic call.
-//
-// The class automatically inserts transition stubs (using the InlineCacheBuffer) when an MT-unsafe
-// transition is made to a stub.
+// It's safe to transition from any state to any state. Typically an inline cache starts
+// in the clean state, meaning it will resolve the call when called. Then it typically
+// transitions to monomorphic, assuming the first dynamic receiver will be the only one
+// observed. If that speculation fails, we transition to megamorphic.
//
class CompiledIC;
class CompiledICProtectionBehaviour;
class CompiledMethod;
-class ICStub;
class CompiledICLocker: public StackObj {
CompiledMethod* _method;
@@ -77,237 +54,105 @@ class CompiledICLocker: public StackObj {
static bool is_safe(address code);
};
-class CompiledICInfo : public StackObj {
- private:
- address _entry; // entry point for call
- void* _cached_value; // Value of cached_value (either in stub or inline cache)
- bool _is_icholder; // Is the cached value a CompiledICHolder*
- bool _is_optimized; // it is an optimized virtual call (i.e., can be statically bound)
- bool _to_interpreter; // Call it to interpreter
- bool _release_icholder;
+// A CompiledICData is a helper object for the inline cache implementation.
+// It comprises:
+// (1) The first receiver klass and its selected method
+// (2) Itable call metadata
+
+class CompiledICData : public CHeapObj {
+ friend class VMStructs;
+ friend class JVMCIVMStructs;
+
+ Method* volatile _speculated_method;
+ uintptr_t volatile _speculated_klass;
+ Klass* _itable_defc_klass;
+ Klass* _itable_refc_klass;
+ bool _is_initialized;
+
+ bool is_speculated_klass_unloaded() const;
+
public:
- address entry() const { return _entry; }
- Metadata* cached_metadata() const { assert(!_is_icholder, ""); return (Metadata*)_cached_value; }
- CompiledICHolder* claim_cached_icholder() {
- assert(_is_icholder, "");
- assert(_cached_value != nullptr, "must be non-null");
- _release_icholder = false;
- CompiledICHolder* icholder = (CompiledICHolder*)_cached_value;
- icholder->claim();
- return icholder;
- }
- bool is_optimized() const { return _is_optimized; }
- bool to_interpreter() const { return _to_interpreter; }
-
- void set_compiled_entry(address entry, Klass* klass, bool is_optimized) {
- _entry = entry;
- _cached_value = (void*)klass;
- _to_interpreter = false;
- _is_icholder = false;
- _is_optimized = is_optimized;
- _release_icholder = false;
- }
+ // Constructor
+ CompiledICData();
- void set_interpreter_entry(address entry, Method* method) {
- _entry = entry;
- _cached_value = (void*)method;
- _to_interpreter = true;
- _is_icholder = false;
- _is_optimized = true;
- _release_icholder = false;
- }
+ // accessors
+ Klass* speculated_klass() const;
+ Method* speculated_method() const { return _speculated_method; }
+ Klass* itable_defc_klass() const { return _itable_defc_klass; }
+ Klass* itable_refc_klass() const { return _itable_refc_klass; }
- void set_icholder_entry(address entry, CompiledICHolder* icholder) {
- _entry = entry;
- _cached_value = (void*)icholder;
- _to_interpreter = true;
- _is_icholder = true;
- _is_optimized = false;
- _release_icholder = true;
- }
+ static ByteSize speculated_method_offset() { return byte_offset_of(CompiledICData, _speculated_method); }
+ static ByteSize speculated_klass_offset() { return byte_offset_of(CompiledICData, _speculated_klass); }
- CompiledICInfo(): _entry(nullptr), _cached_value(nullptr), _is_icholder(false),
- _is_optimized(false), _to_interpreter(false), _release_icholder(false) {
- }
- ~CompiledICInfo() {
- // In rare cases the info is computed but not used, so release any
- // CompiledICHolder* that was created
- if (_release_icholder) {
- assert(_is_icholder, "must be");
- CompiledICHolder* icholder = (CompiledICHolder*)_cached_value;
- icholder->claim();
- delete icholder;
- }
- }
-};
+ static ByteSize itable_defc_klass_offset() { return byte_offset_of(CompiledICData, _itable_defc_klass); }
+ static ByteSize itable_refc_klass_offset() { return byte_offset_of(CompiledICData, _itable_refc_klass); }
-class NativeCallWrapper: public ResourceObj {
-public:
- virtual address destination() const = 0;
- virtual address instruction_address() const = 0;
- virtual address next_instruction_address() const = 0;
- virtual address return_address() const = 0;
- virtual address get_resolve_call_stub(bool is_optimized) const = 0;
- virtual void set_destination_mt_safe(address dest) = 0;
- virtual void set_to_interpreted(const methodHandle& method, CompiledICInfo& info) = 0;
- virtual void verify() const = 0;
- virtual void verify_resolve_call(address dest) const = 0;
-
- virtual bool is_call_to_interpreted(address dest) const = 0;
- virtual bool is_safe_for_patching() const = 0;
-
- virtual NativeInstruction* get_load_instruction(virtual_call_Relocation* r) const = 0;
-
- virtual void *get_data(NativeInstruction* instruction) const = 0;
- virtual void set_data(NativeInstruction* instruction, intptr_t data) = 0;
+ void initialize(CallInfo* call_info, Klass* receiver_klass);
+
+ bool is_initialized() const { return _is_initialized; }
+
+ // GC Support
+ void clean_metadata();
+ void metadata_do(MetadataClosure* cl);
};
class CompiledIC: public ResourceObj {
- friend class InlineCacheBuffer;
- friend class ICStub;
-
- private:
- NativeCallWrapper* _call;
- NativeInstruction* _value; // patchable value cell for this IC
- bool _is_optimized; // an optimized virtual call (i.e., no compiled IC)
+private:
CompiledMethod* _method;
+ CompiledICData* _data;
+ NativeCall* _call;
- CompiledIC(CompiledMethod* cm, NativeCall* ic_call);
CompiledIC(RelocIterator* iter);
- void initialize_from_iter(RelocIterator* iter);
+ // CompiledICData wrappers
+ void ensure_initialized(CallInfo* call_info, Klass* receiver_klass);
+ bool is_speculated_klass(Klass* receiver_klass);
- static bool is_icholder_entry(address entry);
+ // Inline cache states
+ void set_to_monomorphic();
+ void set_to_megamorphic(CallInfo* call_info);
- // low-level inline-cache manipulation. Cannot be accessed directly, since it might not be MT-safe
- // to change an inline-cache. These changes the underlying inline-cache directly. They *newer* make
- // changes to a transition stub.
- void internal_set_ic_destination(address entry_point, bool is_icstub, void* cache, bool is_icholder);
- void set_ic_destination(ICStub* stub);
- void set_ic_destination(address entry_point) {
- assert(_is_optimized, "use set_ic_destination_and_value instead");
- internal_set_ic_destination(entry_point, false, nullptr, false);
- }
- // This only for use by ICStubs where the type of the value isn't known
- void set_ic_destination_and_value(address entry_point, void* value) {
- internal_set_ic_destination(entry_point, false, value, is_icholder_entry(entry_point));
- }
- void set_ic_destination_and_value(address entry_point, Metadata* value) {
- internal_set_ic_destination(entry_point, false, value, false);
- }
- void set_ic_destination_and_value(address entry_point, CompiledICHolder* value) {
- internal_set_ic_destination(entry_point, false, value, true);
- }
-
- // Reads the location of the transition stub. This will fail with an assertion, if no transition stub is
- // associated with the inline cache.
- address stub_address() const;
- bool is_in_transition_state() const; // Use InlineCacheBuffer
-
- public:
+public:
// conversion (machine PC to CompiledIC*)
friend CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr);
friend CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site);
friend CompiledIC* CompiledIC_at(Relocation* call_site);
friend CompiledIC* CompiledIC_at(RelocIterator* reloc_iter);
- static bool is_icholder_call_site(virtual_call_Relocation* call_site, const CompiledMethod* cm);
-
- // Return the cached_metadata/destination associated with this inline cache. If the cache currently points
- // to a transition stub, it will read the values from the transition stub.
- void* cached_value() const;
- CompiledICHolder* cached_icholder() const {
- assert(is_icholder_call(), "must be");
- return (CompiledICHolder*) cached_value();
- }
- Metadata* cached_metadata() const {
- assert(!is_icholder_call(), "must be");
- return (Metadata*) cached_value();
- }
-
- void* get_data() const {
- return _call->get_data(_value);
- }
-
- void set_data(intptr_t data) {
- _call->set_data(_value, data);
- }
-
- address ic_destination() const;
-
- bool is_optimized() const { return _is_optimized; }
+ CompiledICData* data() const;
// State
- bool is_clean() const;
+ bool is_clean() const;
+ bool is_monomorphic() const;
bool is_megamorphic() const;
- bool is_call_to_compiled() const;
- bool is_call_to_interpreted() const;
-
- bool is_icholder_call() const;
- address end_of_call() const { return _call->return_address(); }
+ address end_of_call() const { return _call->return_address(); }
- // MT-safe patching of inline caches. Note: Only safe to call is_xxx when holding the CompiledIC_ock
+ // MT-safe patching of inline caches. Note: Only safe to call is_xxx when holding the CompiledICLocker
// so you are guaranteed that no patching takes place. The same goes for verify.
- //
- // Note: We do not provide any direct access to the stub code, to prevent parts of the code
- // to manipulate the inline cache in MT-unsafe ways.
- //
- // They all takes a TRAP argument, since they can cause a GC if the inline-cache buffer is full.
- //
- bool set_to_clean(bool in_use = true);
- bool set_to_monomorphic(CompiledICInfo& info);
- void clear_ic_stub();
-
- // Returns true if successful and false otherwise. The call can fail if memory
- // allocation in the code cache fails, or ic stub refill is required.
- bool set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecode, bool& needs_ic_stub_refill, TRAPS);
-
- static void compute_monomorphic_entry(const methodHandle& method, Klass* receiver_klass,
- bool is_optimized, bool static_bound, bool caller_is_nmethod,
- CompiledICInfo& info, TRAPS);
+ void set_to_clean();
+ void update(CallInfo* call_info, Klass* receiver_klass);
+
+ // GC support
+ void clean_metadata();
+ void metadata_do(MetadataClosure* cl);
// Location
address instruction_address() const { return _call->instruction_address(); }
+ address destination() const { return _call->destination(); }
// Misc
void print() PRODUCT_RETURN;
- void print_compiled_ic() PRODUCT_RETURN;
void verify() PRODUCT_RETURN;
};
-inline CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr) {
- CompiledIC* c_ic = new CompiledIC(nm, nativeCall_before(return_addr));
- c_ic->verify();
- return c_ic;
-}
-
-inline CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site) {
- CompiledIC* c_ic = new CompiledIC(nm, nativeCall_at(call_site));
- c_ic->verify();
- return c_ic;
-}
-
-inline CompiledIC* CompiledIC_at(Relocation* call_site) {
- assert(call_site->type() == relocInfo::virtual_call_type ||
- call_site->type() == relocInfo::opt_virtual_call_type, "wrong reloc. info");
- CompiledIC* c_ic = new CompiledIC(call_site->code(), nativeCall_at(call_site->addr()));
- c_ic->verify();
- return c_ic;
-}
-
-inline CompiledIC* CompiledIC_at(RelocIterator* reloc_iter) {
- assert(reloc_iter->type() == relocInfo::virtual_call_type ||
- reloc_iter->type() == relocInfo::opt_virtual_call_type, "wrong reloc. info");
- CompiledIC* c_ic = new CompiledIC(reloc_iter);
- c_ic->verify();
- return c_ic;
-}
+CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr);
+CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site);
+CompiledIC* CompiledIC_at(Relocation* call_site);
+CompiledIC* CompiledIC_at(RelocIterator* reloc_iter);
//-----------------------------------------------------------------------------
-// The CompiledStaticCall represents a call to a static method in the compiled
-//
-// Transition diagram of a static call site is somewhat simpler than for an inlined cache:
+// The CompiledDirectCall represents a call to a method in the compiled code
//
//
// -----<----- Clean ----->-----
@@ -321,63 +166,7 @@ inline CompiledIC* CompiledIC_at(RelocIterator* reloc_iter) {
//
//
-class StaticCallInfo {
- private:
- address _entry; // Entrypoint
- methodHandle _callee; // Callee (used when calling interpreter)
- bool _to_interpreter; // call to interpreted method (otherwise compiled)
-
- friend class CompiledStaticCall;
- friend class CompiledDirectStaticCall;
- friend class CompiledPltStaticCall;
- public:
- address entry() const { return _entry; }
- methodHandle callee() const { return _callee; }
-};
-
-class CompiledStaticCall : public ResourceObj {
- public:
- // Code
-
- // Returns null if CodeBuffer::expand fails
- static address emit_to_interp_stub(CodeBuffer &cbuf, address mark = nullptr);
- static int to_interp_stub_size();
- static int to_trampoline_stub_size();
- static int reloc_to_interp_stub();
-
- // Compute entry point given a method
- static void compute_entry(const methodHandle& m, bool caller_is_nmethod, StaticCallInfo& info);
- void compute_entry_for_continuation_entry(const methodHandle& m, StaticCallInfo& info);
-
-public:
- // Clean static call (will force resolving on next use)
- virtual address destination() const = 0;
-
- // Clean static call (will force resolving on next use)
- bool set_to_clean(bool in_use = true);
-
- // Set state. The entry must be the same, as computed by compute_entry.
- // Computation and setting is split up, since the actions are separate during
- // a OptoRuntime::resolve_xxx.
- void set(const StaticCallInfo& info);
-
- // State
- bool is_clean() const;
- bool is_call_to_compiled() const;
- virtual bool is_call_to_interpreted() const = 0;
-
- virtual address instruction_address() const = 0;
- virtual address end_of_call() const = 0;
-protected:
- virtual address resolve_call_stub() const = 0;
- virtual void set_destination_mt_safe(address dest) = 0;
- virtual void set_to_interpreted(const methodHandle& callee, address entry) = 0;
- virtual const char* name() const = 0;
-
- void set_to_compiled(address entry);
-};
-
-class CompiledDirectStaticCall : public CompiledStaticCall {
+class CompiledDirectCall : public ResourceObj {
private:
friend class CompiledIC;
friend class DirectNativeCallWrapper;
@@ -392,22 +181,28 @@ class CompiledDirectStaticCall : public CompiledStaticCall {
NativeCall* _call;
- CompiledDirectStaticCall(NativeCall* call) : _call(call) {}
+ CompiledDirectCall(NativeCall* call) : _call(call) {}
public:
- static inline CompiledDirectStaticCall* before(address return_addr) {
- CompiledDirectStaticCall* st = new CompiledDirectStaticCall(nativeCall_before(return_addr));
+ // Returns null if CodeBuffer::expand fails
+ static address emit_to_interp_stub(CodeBuffer &cbuf, address mark = nullptr);
+ static int to_interp_stub_size();
+ static int to_trampoline_stub_size();
+ static int reloc_to_interp_stub();
+
+ static inline CompiledDirectCall* before(address return_addr) {
+ CompiledDirectCall* st = new CompiledDirectCall(nativeCall_before(return_addr));
st->verify();
return st;
}
- static inline CompiledDirectStaticCall* at(address native_call) {
- CompiledDirectStaticCall* st = new CompiledDirectStaticCall(nativeCall_at(native_call));
+ static inline CompiledDirectCall* at(address native_call) {
+ CompiledDirectCall* st = new CompiledDirectCall(nativeCall_at(native_call));
st->verify();
return st;
}
- static inline CompiledDirectStaticCall* at(Relocation* call_site) {
+ static inline CompiledDirectCall* at(Relocation* call_site) {
return at(call_site->addr());
}
@@ -415,8 +210,15 @@ class CompiledDirectStaticCall : public CompiledStaticCall {
address destination() const { return _call->destination(); }
address end_of_call() const { return _call->return_address(); }
+ // Clean static call (will force resolving on next use)
+ void set_to_clean();
+
+ void set(const methodHandle& callee_method);
+
// State
- virtual bool is_call_to_interpreted() const;
+ bool is_clean() const;
+ bool is_call_to_interpreted() const;
+ bool is_call_to_compiled() const;
// Stub support
static address find_stub_for(address instruction);
@@ -426,10 +228,6 @@ class CompiledDirectStaticCall : public CompiledStaticCall {
// Misc.
void print() PRODUCT_RETURN;
void verify() PRODUCT_RETURN;
-
- protected:
- virtual address resolve_call_stub() const;
- virtual const char* name() const { return "CompiledDirectStaticCall"; }
};
#endif // SHARE_CODE_COMPILEDIC_HPP
diff --git a/src/hotspot/share/code/compiledMethod.cpp b/src/hotspot/share/code/compiledMethod.cpp
index 30b4e3617d68c..6553d6f79344f 100644
--- a/src/hotspot/share/code/compiledMethod.cpp
+++ b/src/hotspot/share/code/compiledMethod.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,6 @@
#include "code/exceptionHandlerTable.hpp"
#include "code/scopeDesc.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shared/gcBehaviours.hpp"
@@ -36,7 +35,6 @@
#include "logging/log.hpp"
#include "logging/logTag.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.inline.hpp"
#include "oops/klass.inline.hpp"
#include "oops/methodData.hpp"
#include "oops/method.inline.hpp"
@@ -335,28 +333,6 @@ address CompiledMethod::oops_reloc_begin() const {
return low_boundary;
}
-int CompiledMethod::verify_icholder_relocations() {
- ResourceMark rm;
- int count = 0;
-
- RelocIterator iter(this);
- while(iter.next()) {
- if (iter.type() == relocInfo::virtual_call_type) {
- if (CompiledIC::is_icholder_call_site(iter.virtual_call_reloc(), this)) {
- CompiledIC *ic = CompiledIC_at(&iter);
- if (TraceCompiledIC) {
- tty->print("noticed icholder " INTPTR_FORMAT " ", p2i(ic->cached_icholder()));
- ic->print();
- }
- assert(ic->cached_icholder() != nullptr, "must be non-nullptr");
- count++;
- }
- }
- }
-
- return count;
-}
-
// Method that knows how to preserve outgoing arguments at call. This method must be
// called with a frame corresponding to a Java invoke
void CompiledMethod::preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map, OopClosure* f) {
@@ -431,20 +407,6 @@ void CompiledMethod::clear_inline_caches() {
}
}
-// Clear IC callsites, releasing ICStubs of all compiled ICs
-// as well as any associated CompiledICHolders.
-void CompiledMethod::clear_ic_callsites() {
- assert(CompiledICLocker::is_safe(this), "mt unsafe call");
- ResourceMark rm;
- RelocIterator iter(this);
- while(iter.next()) {
- if (iter.type() == relocInfo::virtual_call_type) {
- CompiledIC* ic = CompiledIC_at(&iter);
- ic->set_to_clean(false);
- }
- }
-}
-
#ifdef ASSERT
// Check class_loader is alive for this bit of metadata.
class CheckClass : public MetadataClosure {
@@ -466,70 +428,22 @@ class CheckClass : public MetadataClosure {
#endif // ASSERT
-bool CompiledMethod::clean_ic_if_metadata_is_dead(CompiledIC *ic) {
- if (ic->is_clean()) {
- return true;
- }
- if (ic->is_icholder_call()) {
- // The only exception is compiledICHolder metadata which may
- // yet be marked below. (We check this further below).
- CompiledICHolder* cichk_metdata = ic->cached_icholder();
-
- if (cichk_metdata->is_loader_alive()) {
- return true;
- }
- } else {
- Metadata* ic_metdata = ic->cached_metadata();
- if (ic_metdata != nullptr) {
- if (ic_metdata->is_klass()) {
- if (((Klass*)ic_metdata)->is_loader_alive()) {
- return true;
- }
- } else if (ic_metdata->is_method()) {
- Method* method = (Method*)ic_metdata;
- assert(!method->is_old(), "old method should have been cleaned");
- if (method->method_holder()->is_loader_alive()) {
- return true;
- }
- } else {
- ShouldNotReachHere();
- }
- } else {
- // This inline cache is a megamorphic vtable call. Those ICs never hold
- // any Metadata and should therefore never be cleaned by this function.
- return true;
- }
- }
-
- return ic->set_to_clean();
+static void clean_ic_if_metadata_is_dead(CompiledIC *ic) {
+ ic->clean_metadata();
}
// Clean references to unloaded nmethods at addr from this one, which is not unloaded.
-template
-static bool clean_if_nmethod_is_unloaded(CompiledICorStaticCall *ic, address addr, CompiledMethod* from,
+template
+static void clean_if_nmethod_is_unloaded(CallsiteT* callsite, CompiledMethod* from,
bool clean_all) {
- CodeBlob *cb = CodeCache::find_blob(addr);
- CompiledMethod* nm = (cb != nullptr) ? cb->as_compiled_method_or_null() : nullptr;
- if (nm != nullptr) {
- // Clean inline caches pointing to bad nmethods
- if (clean_all || !nm->is_in_use() || nm->is_unloading() || (nm->method()->code() != nm)) {
- if (!ic->set_to_clean(!from->is_unloading())) {
- return false;
- }
- assert(ic->is_clean(), "nmethod " PTR_FORMAT "not clean %s", p2i(from), from->method()->name_and_sig_as_C_string());
- }
+ CodeBlob* cb = CodeCache::find_blob(callsite->destination());
+ if (!cb->is_compiled()) {
+ return;
+ }
+ CompiledMethod* cm = cb->as_compiled_method();
+ if (clean_all || !cm->is_in_use() || cm->is_unloading() || cm->method()->code() != cm) {
+ callsite->set_to_clean();
}
- return true;
-}
-
-static bool clean_if_nmethod_is_unloaded(CompiledIC *ic, CompiledMethod* from,
- bool clean_all) {
- return clean_if_nmethod_is_unloaded(ic, ic->ic_destination(), from, clean_all);
-}
-
-static bool clean_if_nmethod_is_unloaded(CompiledStaticCall *csc, CompiledMethod* from,
- bool clean_all) {
- return clean_if_nmethod_is_unloaded(csc, csc->destination(), from, clean_all);
}
// Cleans caches in nmethods that point to either classes that are unloaded
@@ -539,7 +453,7 @@ static bool clean_if_nmethod_is_unloaded(CompiledStaticCall *csc, CompiledMethod
// nmethods are unloaded. Return postponed=true in the parallel case for
// inline caches found that point to nmethods that are not yet visited during
// the do_unloading walk.
-bool CompiledMethod::unload_nmethod_caches(bool unloading_occurred) {
+void CompiledMethod::unload_nmethod_caches(bool unloading_occurred) {
ResourceMark rm;
// Exception cache only needs to be called if unloading occurred
@@ -547,16 +461,13 @@ bool CompiledMethod::unload_nmethod_caches(bool unloading_occurred) {
clean_exception_cache();
}
- if (!cleanup_inline_caches_impl(unloading_occurred, false)) {
- return false;
- }
+ cleanup_inline_caches_impl(unloading_occurred, false);
#ifdef ASSERT
// Check that the metadata embedded in the nmethod is alive
CheckClass check_class;
metadata_do(&check_class);
#endif
- return true;
}
void CompiledMethod::run_nmethod_entry_barrier() {
@@ -578,8 +489,7 @@ void CompiledMethod::run_nmethod_entry_barrier() {
void CompiledMethod::cleanup_inline_caches_whitebox() {
assert_locked_or_safepoint(CodeCache_lock);
CompiledICLocker ic_locker(this);
- guarantee(cleanup_inline_caches_impl(false /* unloading_occurred */, true /* clean_all */),
- "Inline cache cleaning in a safepoint can't fail");
+ cleanup_inline_caches_impl(false /* unloading_occurred */, true /* clean_all */);
}
address* CompiledMethod::orig_pc_addr(const frame* fr) {
@@ -587,7 +497,7 @@ address* CompiledMethod::orig_pc_addr(const frame* fr) {
}
// Called to clean up after class unloading for live nmethods
-bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all) {
+void CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all) {
assert(CompiledICLocker::is_safe(this), "mt unsafe call");
ResourceMark rm;
@@ -602,26 +512,15 @@ bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool cl
if (unloading_occurred) {
// If class unloading occurred we first clear ICs where the cached metadata
// is referring to an unloaded klass or method.
- if (!clean_ic_if_metadata_is_dead(CompiledIC_at(&iter))) {
- return false;
- }
+ clean_ic_if_metadata_is_dead(CompiledIC_at(&iter));
}
- if (!clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all)) {
- return false;
- }
+ clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all);
break;
case relocInfo::opt_virtual_call_type:
- if (!clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all)) {
- return false;
- }
- break;
-
case relocInfo::static_call_type:
- if (!clean_if_nmethod_is_unloaded(compiledStaticCall_at(iter.reloc()), this, clean_all)) {
- return false;
- }
+ clean_if_nmethod_is_unloaded(CompiledDirectCall::at(iter.reloc()), this, clean_all);
break;
case relocInfo::static_stub_type: {
@@ -672,8 +571,6 @@ bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool cl
break;
}
}
-
- return true;
}
address CompiledMethod::continuation_for_implicit_exception(address pc, bool for_div0_check) {
@@ -687,12 +584,15 @@ address CompiledMethod::continuation_for_implicit_exception(address pc, bool for
ResourceMark rm(thread);
CodeBlob* cb = CodeCache::find_blob(pc);
assert(cb != nullptr && cb == this, "");
- ttyLocker ttyl;
- tty->print_cr("implicit exception happened at " INTPTR_FORMAT, p2i(pc));
- print();
- method()->print_codes();
- print_code();
- print_pcs();
+
+ // Keep tty output consistent. To avoid ttyLocker, we buffer in stream, and print all at once.
+ stringStream ss;
+ ss.print_cr("implicit exception happened at " INTPTR_FORMAT, p2i(pc));
+ print_on(&ss);
+ method()->print_codes_on(&ss);
+ print_code_on(&ss);
+ print_pcs_on(&ss);
+ tty->print("%s", ss.as_string()); // print all at once
}
#endif
if (cont_offset == 0) {
diff --git a/src/hotspot/share/code/compiledMethod.hpp b/src/hotspot/share/code/compiledMethod.hpp
index 6973911c06328..42d68bda55472 100644
--- a/src/hotspot/share/code/compiledMethod.hpp
+++ b/src/hotspot/share/code/compiledMethod.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,7 +35,7 @@ class ExceptionHandlerTable;
class ImplicitExceptionTable;
class AbstractCompiler;
class xmlStream;
-class CompiledStaticCall;
+class CompiledDirectCall;
class NativeCallWrapper;
class ScopeDesc;
class CompiledIC;
@@ -227,7 +227,7 @@ class CompiledMethod : public CodeBlob {
virtual bool is_osr_method() const = 0;
virtual int osr_entry_bci() const = 0;
Method* method() const { return _method; }
- virtual void print_pcs() = 0;
+ virtual void print_pcs_on(outputStream* st) = 0;
bool is_native_method() const { return _method != nullptr && _method->is_native(); }
bool is_java_method() const { return _method != nullptr && !_method->is_native(); }
@@ -364,7 +364,7 @@ class CompiledMethod : public CodeBlob {
// Inline cache support for class unloading and nmethod unloading
private:
- bool cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all);
+ void cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all);
address continuation_for_implicit_exception(address pc, bool for_div0_check);
@@ -373,13 +373,10 @@ class CompiledMethod : public CodeBlob {
void cleanup_inline_caches_whitebox();
virtual void clear_inline_caches();
- void clear_ic_callsites();
// Execute nmethod barrier code, as if entering through nmethod call.
void run_nmethod_entry_barrier();
- // Verify and count cached icholder relocations.
- int verify_icholder_relocations();
void verify_oop_relocations();
bool has_evol_metadata();
@@ -389,14 +386,8 @@ class CompiledMethod : public CodeBlob {
// corresponds to the given method as well.
virtual bool is_dependent_on_method(Method* dependee) = 0;
- virtual NativeCallWrapper* call_wrapper_at(address call) const = 0;
- virtual NativeCallWrapper* call_wrapper_before(address return_pc) const = 0;
virtual address call_instruction_address(address pc) const = 0;
- virtual CompiledStaticCall* compiledStaticCall_at(Relocation* call_site) const = 0;
- virtual CompiledStaticCall* compiledStaticCall_at(address addr) const = 0;
- virtual CompiledStaticCall* compiledStaticCall_before(address addr) const = 0;
-
Method* attached_method(address call_pc);
Method* attached_method_before_pc(address pc);
@@ -406,16 +397,13 @@ class CompiledMethod : public CodeBlob {
protected:
address oops_reloc_begin() const;
- private:
- bool static clean_ic_if_metadata_is_dead(CompiledIC *ic);
-
public:
// GC unloading support
// Cleans unloaded klasses and unloaded nmethods in inline caches
virtual bool is_unloading() = 0;
- bool unload_nmethod_caches(bool class_unloading_occurred);
+ void unload_nmethod_caches(bool class_unloading_occurred);
virtual void do_unloading(bool unloading_occurred) = 0;
private:
diff --git a/src/hotspot/share/code/icBuffer.cpp b/src/hotspot/share/code/icBuffer.cpp
deleted file mode 100644
index dbceefb7c8efc..0000000000000
--- a/src/hotspot/share/code/icBuffer.cpp
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "code/codeCache.hpp"
-#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
-#include "code/nmethod.hpp"
-#include "code/scopeDesc.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/interpreter.hpp"
-#include "interpreter/linkResolver.hpp"
-#include "memory/resourceArea.hpp"
-#include "oops/method.hpp"
-#include "oops/oop.inline.hpp"
-#include "runtime/atomic.hpp"
-#include "runtime/handles.inline.hpp"
-#include "runtime/javaThread.hpp"
-#include "runtime/mutexLocker.hpp"
-#include "runtime/stubRoutines.hpp"
-#include "runtime/vmOperations.hpp"
-
-DEF_STUB_INTERFACE(ICStub);
-
-StubQueue* InlineCacheBuffer::_buffer = nullptr;
-
-CompiledICHolder* volatile InlineCacheBuffer::_pending_released = nullptr;
-volatile int InlineCacheBuffer::_pending_count = 0;
-
-#ifdef ASSERT
-ICRefillVerifier::ICRefillVerifier()
- : _refill_requested(false),
- _refill_remembered(false)
-{
- Thread* thread = Thread::current();
- assert(thread->missed_ic_stub_refill_verifier() == nullptr, "nesting not supported");
- thread->set_missed_ic_stub_refill_verifier(this);
-}
-
-ICRefillVerifier::~ICRefillVerifier() {
- assert(!_refill_requested || _refill_remembered,
- "Forgot to refill IC stubs after failed IC transition");
- Thread::current()->set_missed_ic_stub_refill_verifier(nullptr);
-}
-
-ICRefillVerifierMark::ICRefillVerifierMark(ICRefillVerifier* verifier) {
- Thread* thread = Thread::current();
- assert(thread->missed_ic_stub_refill_verifier() == nullptr, "nesting not supported");
- thread->set_missed_ic_stub_refill_verifier(verifier);
-}
-
-ICRefillVerifierMark::~ICRefillVerifierMark() {
- Thread::current()->set_missed_ic_stub_refill_verifier(nullptr);
-}
-
-static ICRefillVerifier* current_ic_refill_verifier() {
- Thread* current = Thread::current();
- ICRefillVerifier* verifier = current->missed_ic_stub_refill_verifier();
- assert(verifier != nullptr, "need a verifier for safety");
- return verifier;
-}
-#endif
-
-void ICStub::finalize() {
- if (!is_empty()) {
- ResourceMark rm;
- CompiledIC *ic = CompiledIC_at(CodeCache::find_compiled(ic_site()), ic_site());
- assert(CodeCache::find_compiled(ic->instruction_address()) != nullptr, "inline cache in non-compiled?");
-
- assert(this == ICStub_from_destination_address(ic->stub_address()), "wrong owner of ic buffer");
- ic->set_ic_destination_and_value(destination(), cached_value());
- }
-}
-
-
-address ICStub::destination() const {
- return InlineCacheBuffer::ic_buffer_entry_point(code_begin());
-}
-
-void* ICStub::cached_value() const {
- return InlineCacheBuffer::ic_buffer_cached_value(code_begin());
-}
-
-
-void ICStub::set_stub(CompiledIC *ic, void* cached_val, address dest_addr) {
- // We cannot store a pointer to the 'ic' object, since it is resource allocated. Instead we
- // store the location of the inline cache. Then we have enough information recreate the CompiledIC
- // object when we need to remove the stub.
- _ic_site = ic->instruction_address();
-
- // Assemble new stub
- InlineCacheBuffer::assemble_ic_buffer_code(code_begin(), cached_val, dest_addr);
- assert(destination() == dest_addr, "can recover destination");
- assert(cached_value() == cached_val, "can recover destination");
-}
-
-
-void ICStub::clear() {
- if (CompiledIC::is_icholder_entry(destination())) {
- InlineCacheBuffer::queue_for_release((CompiledICHolder*)cached_value());
- }
- _ic_site = nullptr;
-}
-
-
-#ifndef PRODUCT
-// anybody calling to this stub will trap
-
-void ICStub::verify() {
-}
-
-void ICStub::print() {
- tty->print_cr("ICStub: site: " INTPTR_FORMAT, p2i(_ic_site));
-}
-#endif
-
-//-----------------------------------------------------------------------------------------------
-// Implementation of InlineCacheBuffer
-
-
-void InlineCacheBuffer::initialize() {
- if (_buffer != nullptr) return; // already initialized
- _buffer = new StubQueue(new ICStubInterface, checked_cast(InlineCacheBufferSize), InlineCacheBuffer_lock, "InlineCacheBuffer");
- assert (_buffer != nullptr, "cannot allocate InlineCacheBuffer");
-}
-
-
-ICStub* InlineCacheBuffer::new_ic_stub() {
- return (ICStub*)buffer()->request_committed(ic_stub_code_size());
-}
-
-
-void InlineCacheBuffer::refill_ic_stubs() {
-#ifdef ASSERT
- ICRefillVerifier* verifier = current_ic_refill_verifier();
- verifier->request_remembered();
-#endif
- // we ran out of inline cache buffer space; must enter safepoint.
- // We do this by forcing a safepoint
- VM_ICBufferFull ibf;
- VMThread::execute(&ibf);
-}
-
-bool InlineCacheBuffer::needs_update_inline_caches() {
- // Stub removal
- if (buffer()->number_of_stubs() > 0) {
- return true;
- }
-
- // Release pending CompiledICHolder
- if (pending_icholder_count() > 0) {
- return true;
- }
-
- return false;
-}
-
-void InlineCacheBuffer::update_inline_caches() {
- if (buffer()->number_of_stubs() > 0) {
- if (TraceICBuffer) {
- tty->print_cr("[updating inline caches with %d stubs]", buffer()->number_of_stubs());
- }
- buffer()->remove_all();
- }
- release_pending_icholders();
-}
-
-
-bool InlineCacheBuffer::contains(address instruction_address) {
- return buffer()->contains(instruction_address);
-}
-
-
-bool InlineCacheBuffer::is_empty() {
- return buffer()->number_of_stubs() == 0;
-}
-
-
-void InlineCacheBuffer_init() {
- InlineCacheBuffer::initialize();
-}
-
-bool InlineCacheBuffer::create_transition_stub(CompiledIC *ic, void* cached_value, address entry) {
- assert(!SafepointSynchronize::is_at_safepoint(), "should not be called during a safepoint");
- assert(CompiledICLocker::is_safe(ic->instruction_address()), "mt unsafe call");
- if (TraceICBuffer) {
- tty->print_cr(" create transition stub for " INTPTR_FORMAT " destination " INTPTR_FORMAT " cached value " INTPTR_FORMAT,
- p2i(ic->instruction_address()), p2i(entry), p2i(cached_value));
- }
-
- // allocate and initialize new "out-of-line" inline-cache
- ICStub* ic_stub = new_ic_stub();
- if (ic_stub == nullptr) {
-#ifdef ASSERT
- ICRefillVerifier* verifier = current_ic_refill_verifier();
- verifier->request_refill();
-#endif
- return false;
- }
-
- // If an transition stub is already associate with the inline cache, then we remove the association.
- if (ic->is_in_transition_state()) {
- ICStub* old_stub = ICStub_from_destination_address(ic->stub_address());
- old_stub->clear();
- }
-
- ic_stub->set_stub(ic, cached_value, entry);
-
- // Update inline cache in nmethod to point to new "out-of-line" allocated inline cache
- ic->set_ic_destination(ic_stub);
- return true;
-}
-
-
-address InlineCacheBuffer::ic_destination_for(CompiledIC *ic) {
- ICStub* stub = ICStub_from_destination_address(ic->stub_address());
- return stub->destination();
-}
-
-
-void* InlineCacheBuffer::cached_value_for(CompiledIC *ic) {
- ICStub* stub = ICStub_from_destination_address(ic->stub_address());
- return stub->cached_value();
-}
-
-
-// Free CompiledICHolder*s that are no longer in use
-void InlineCacheBuffer::release_pending_icholders() {
- assert(SafepointSynchronize::is_at_safepoint(), "should only be called during a safepoint");
- CompiledICHolder* holder = Atomic::load(&_pending_released);
- _pending_released = nullptr;
- int count = 0;
- while (holder != nullptr) {
- CompiledICHolder* next = holder->next();
- delete holder;
- holder = next;
- count++;
- }
- assert(pending_icholder_count() == count, "wrong count");
- Atomic::store(&_pending_count, 0);
-}
-
-// Enqueue this icholder for release during the next safepoint. It's
-// not safe to free them until then since they might be visible to
-// another thread.
-void InlineCacheBuffer::queue_for_release(CompiledICHolder* icholder) {
- assert(icholder->next() == nullptr, "multiple enqueue?");
-
- CompiledICHolder* old = Atomic::load(&_pending_released);
- for (;;) {
- icholder->set_next(old);
- // The only reader runs at a safepoint serially so there is no need for a more strict atomic.
- CompiledICHolder* cur = Atomic::cmpxchg(&_pending_released, old, icholder, memory_order_relaxed);
- if (cur == old) {
- break;
- }
- old = cur;
- }
- Atomic::inc(&_pending_count, memory_order_relaxed);
-
- if (TraceICBuffer) {
- tty->print_cr("enqueueing icholder " INTPTR_FORMAT " to be freed", p2i(icholder));
- }
-}
-
-int InlineCacheBuffer::pending_icholder_count() {
- return Atomic::load(&_pending_count);
-}
diff --git a/src/hotspot/share/code/icBuffer.hpp b/src/hotspot/share/code/icBuffer.hpp
deleted file mode 100644
index c2da3abdca302..0000000000000
--- a/src/hotspot/share/code/icBuffer.hpp
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_CODE_ICBUFFER_HPP
-#define SHARE_CODE_ICBUFFER_HPP
-
-#include "asm/codeBuffer.hpp"
-#include "code/stubs.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/allocation.hpp"
-#include "runtime/safepointVerifiers.hpp"
-#include "utilities/align.hpp"
-#include "utilities/debug.hpp"
-#include "utilities/macros.hpp"
-
-class CompiledIC;
-class CompiledICHolder;
-
-//
-// For CompiledIC's:
-//
-// In cases where we do not have MT-safe state transformation,
-// we go to a transition state, using ICStubs. At a safepoint,
-// the inline caches are transferred from the transitional code:
-//
-// instruction_address --> 01 set xxx_oop, Ginline_cache_klass
-// 23 jump_to Gtemp, yyyy
-// 4 nop
-
-class ICStub: public Stub {
- private:
- int _size; // total size of the stub incl. code
- address _ic_site; // points at call instruction of owning ic-buffer
- /* stub code follows here */
- protected:
- friend class ICStubInterface;
- // This will be called only by ICStubInterface
- void initialize(int size) { _size = size; _ic_site = nullptr; }
- void finalize(); // called when a method is removed
-
- // General info
- int size() const { return _size; }
-
- // ICStub_from_destination_address looks up Stub* address from code entry address,
- // which unfortunately means the stub head should be at the same alignment as the code.
- static int alignment() { return CodeEntryAlignment; }
-
- public:
- // Creation
- void set_stub(CompiledIC *ic, void* cached_value, address dest_addr);
-
- // Code info
- address code_begin() const { return align_up((address)this + sizeof(ICStub), CodeEntryAlignment); }
- address code_end() const { return (address)this + size(); }
-
- // Call site info
- address ic_site() const { return _ic_site; }
- void clear();
- bool is_empty() const { return _ic_site == nullptr; }
-
- // stub info
- address destination() const; // destination of jump instruction
- void* cached_value() const; // cached_value for stub
-
- // Debugging
- void verify() PRODUCT_RETURN;
- void print() PRODUCT_RETURN;
-
- // Creation
- friend ICStub* ICStub_from_destination_address(address destination_address);
-};
-
-// ICStub Creation
-inline ICStub* ICStub_from_destination_address(address destination_address) {
- ICStub* stub = (ICStub*) (destination_address - align_up(sizeof(ICStub), CodeEntryAlignment));
- #ifdef ASSERT
- stub->verify();
- #endif
- return stub;
-}
-
-#ifdef ASSERT
-// The ICRefillVerifier class is a stack allocated RAII object used to
-// detect if a failed IC transition that required IC stub refilling has
-// been accidentally missed. It is up to the caller to in that case
-// refill IC stubs.
-class ICRefillVerifier: StackObj {
- bool _refill_requested;
- bool _refill_remembered;
-
- public:
- ICRefillVerifier();
- ~ICRefillVerifier();
-
- void request_refill() { _refill_requested = true; }
- void request_remembered() { _refill_remembered = true; }
-};
-
-// The ICRefillVerifierMark is used to set the thread's current
-// ICRefillVerifier to a provided one. This is useful in particular
-// when transitioning IC stubs in parallel and refilling from the
-// master thread invoking the IC stub transitioning code.
-class ICRefillVerifierMark: StackObj {
- public:
- ICRefillVerifierMark(ICRefillVerifier* verifier);
- ~ICRefillVerifierMark();
-};
-#else
-class ICRefillVerifier: StackObj {
- public:
- ICRefillVerifier() {}
-};
-class ICRefillVerifierMark: StackObj {
- public:
- ICRefillVerifierMark(ICRefillVerifier* verifier) {}
-};
-#endif
-
-class InlineCacheBuffer: public AllStatic {
- private:
- // friends
- friend class ICStub;
-
- static int ic_stub_code_size();
-
- static StubQueue* _buffer;
-
- static CompiledICHolder* volatile _pending_released;
- static volatile int _pending_count;
-
- static StubQueue* buffer() { return _buffer; }
-
- static ICStub* new_ic_stub();
-
- // Machine-dependent implementation of ICBuffer
- static void assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point);
- static address ic_buffer_entry_point (address code_begin);
- static void* ic_buffer_cached_value (address code_begin);
-
- public:
-
- // Initialization; must be called before first usage
- static void initialize();
-
- // Access
- static bool contains(address instruction_address);
-
- // removes the ICStubs after backpatching
- static bool needs_update_inline_caches();
- static void update_inline_caches();
- static void refill_ic_stubs();
-
- // for debugging
- static bool is_empty();
-
- static void release_pending_icholders();
- static void queue_for_release(CompiledICHolder* icholder);
- static int pending_icholder_count();
-
- // New interface
- static bool create_transition_stub(CompiledIC *ic, void* cached_value, address entry);
- static address ic_destination_for(CompiledIC *ic);
- static void* cached_value_for(CompiledIC *ic);
-};
-
-#endif // SHARE_CODE_ICBUFFER_HPP
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index b18077fddfdef..2755df3251396 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -640,6 +640,7 @@ nmethod::nmethod(
ByteSize basic_lock_sp_offset,
OopMapSet* oop_maps )
: CompiledMethod(method, "native nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true),
+ _compiled_ic_data(nullptr),
_is_unlinked(false),
_native_receiver_sp_offset(basic_lock_owner_sp_offset),
_native_basic_lock_sp_offset(basic_lock_sp_offset),
@@ -697,12 +698,12 @@ nmethod::nmethod(
clear_unloading_state();
+ finalize_relocations();
+
Universe::heap()->register_nmethod(this);
debug_only(Universe::heap()->verify_nmethod(this));
CodeCache::commit(this);
-
- finalize_relocations();
}
if (PrintNativeNMethods || PrintDebugInfo || PrintRelocations || PrintDependencies) {
@@ -784,6 +785,7 @@ nmethod::nmethod(
#endif
)
: CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true),
+ _compiled_ic_data(nullptr),
_is_unlinked(false),
_native_receiver_sp_offset(in_ByteSize(-1)),
_native_basic_lock_sp_offset(in_ByteSize(-1)),
@@ -887,13 +889,13 @@ nmethod::nmethod(
}
#endif
+ finalize_relocations();
+
Universe::heap()->register_nmethod(this);
debug_only(Universe::heap()->verify_nmethod(this));
CodeCache::commit(this);
- finalize_relocations();
-
// Copy contents of ExceptionHandlerTable to nmethod
handler_table->copy_to(this);
nul_chk_table->copy_to(this);
@@ -1021,7 +1023,7 @@ void nmethod::print_nmethod(bool printmethod) {
tty->print_cr("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ");
print_metadata(tty);
tty->print_cr("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ");
- print_pcs();
+ print_pcs_on(tty);
tty->print_cr("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ");
if (oop_maps() != nullptr) {
tty->print("oop maps:"); // oop_maps()->print_on(tty) outputs a cr() at the beginning
@@ -1145,16 +1147,33 @@ static void install_post_call_nop_displacement(nmethod* nm, address pc) {
void nmethod::finalize_relocations() {
NoSafepointVerifier nsv;
+ GrowableArray virtual_call_data;
+
// Make sure that post call nops fill in nmethod offsets eagerly so
// we don't have to race with deoptimization
RelocIterator iter(this);
while (iter.next()) {
- if (iter.type() == relocInfo::post_call_nop_type) {
+ if (iter.type() == relocInfo::virtual_call_type) {
+ virtual_call_Relocation* r = iter.virtual_call_reloc();
+ NativeMovConstReg* value = nativeMovConstReg_at(r->cached_value());
+ virtual_call_data.append(value);
+ } else if (iter.type() == relocInfo::post_call_nop_type) {
post_call_nop_Relocation* const reloc = iter.post_call_nop_reloc();
address pc = reloc->addr();
install_post_call_nop_displacement(this, pc);
}
}
+
+ if (virtual_call_data.length() > 0) {
+ // We allocate a block of CompiledICData per nmethod so the GC can purge this faster.
+ _compiled_ic_data = new CompiledICData[virtual_call_data.length()];
+ CompiledICData* next_data = _compiled_ic_data;
+
+ for (NativeMovConstReg* value : virtual_call_data) {
+ value->set_data((intptr_t)next_data);
+ next_data++;
+ }
+ }
}
void nmethod::make_deoptimized() {
@@ -1180,8 +1199,7 @@ void nmethod::make_deoptimized() {
while (iter.next()) {
switch (iter.type()) {
- case relocInfo::virtual_call_type:
- case relocInfo::opt_virtual_call_type: {
+ case relocInfo::virtual_call_type: {
CompiledIC *ic = CompiledIC_at(&iter);
address pc = ic->end_of_call();
NativePostCallNop* nop = nativePostCallNop_at(pc);
@@ -1191,8 +1209,9 @@ void nmethod::make_deoptimized() {
assert(NativeDeoptInstruction::is_deopt_at(pc), "check");
break;
}
- case relocInfo::static_call_type: {
- CompiledStaticCall *csc = compiledStaticCall_at(iter.reloc());
+ case relocInfo::static_call_type:
+ case relocInfo::opt_virtual_call_type: {
+ CompiledDirectCall *csc = CompiledDirectCall::at(iter.reloc());
address pc = csc->end_of_call();
NativePostCallNop* nop = nativePostCallNop_at(pc);
//tty->print_cr(" - static pc %p", pc);
@@ -1219,29 +1238,29 @@ void nmethod::verify_clean_inline_caches() {
RelocIterator iter(this, oops_reloc_begin());
while(iter.next()) {
switch(iter.type()) {
- case relocInfo::virtual_call_type:
- case relocInfo::opt_virtual_call_type: {
+ case relocInfo::virtual_call_type: {
CompiledIC *ic = CompiledIC_at(&iter);
- CodeBlob *cb = CodeCache::find_blob(ic->ic_destination());
+ CodeBlob *cb = CodeCache::find_blob(ic->destination());
assert(cb != nullptr, "destination not in CodeBlob?");
nmethod* nm = cb->as_nmethod_or_null();
- if( nm != nullptr ) {
+ if (nm != nullptr) {
// Verify that inline caches pointing to bad nmethods are clean
- if (!nm->is_in_use() || (nm->method()->code() != nm)) {
+ if (!nm->is_in_use() || nm->is_unloading()) {
assert(ic->is_clean(), "IC should be clean");
}
}
break;
}
- case relocInfo::static_call_type: {
- CompiledStaticCall *csc = compiledStaticCall_at(iter.reloc());
- CodeBlob *cb = CodeCache::find_blob(csc->destination());
+ case relocInfo::static_call_type:
+ case relocInfo::opt_virtual_call_type: {
+ CompiledDirectCall *cdc = CompiledDirectCall::at(iter.reloc());
+ CodeBlob *cb = CodeCache::find_blob(cdc->destination());
assert(cb != nullptr, "destination not in CodeBlob?");
nmethod* nm = cb->as_nmethod_or_null();
- if( nm != nullptr ) {
+ if (nm != nullptr) {
// Verify that inline caches pointing to bad nmethods are clean
- if (!nm->is_in_use() || (nm->method()->code() != nm)) {
- assert(csc->is_clean(), "IC should be clean");
+ if (!nm->is_in_use() || nm->is_unloading() || nm->method()->code() != nm) {
+ assert(cdc->is_clean(), "IC should be clean");
}
}
break;
@@ -1405,9 +1424,7 @@ bool nmethod::make_not_entrant() {
// For concurrent GCs, there must be a handshake between unlink and flush
void nmethod::unlink() {
if (_is_unlinked) {
- // Already unlinked. It can be invoked twice because concurrent code cache
- // unloading might need to restart when inline cache cleaning fails due to
- // running out of ICStubs, which can only be refilled at safepoints
+ // Already unlinked.
return;
}
@@ -1418,7 +1435,6 @@ void nmethod::unlink() {
// the Method, because it is only concurrently unlinked by
// the entry barrier, which acquires the per nmethod lock.
unlink_from_method();
- clear_ic_callsites();
if (is_osr_method()) {
invalidate_osr_method();
@@ -1463,10 +1479,11 @@ void nmethod::purge(bool free_code_cache_data, bool unregister_nmethod) {
ec = next;
}
+ delete[] _compiled_ic_data;
+
if (unregister_nmethod) {
Universe::heap()->unregister_nmethod(this);
}
-
CodeCache::unregister_old_nmethod(this);
CodeBlob::purge(free_code_cache_data, unregister_nmethod);
@@ -1604,16 +1621,7 @@ void nmethod::metadata_do(MetadataClosure* f) {
// Check compiledIC holders associated with this nmethod
ResourceMark rm;
CompiledIC *ic = CompiledIC_at(&iter);
- if (ic->is_icholder_call()) {
- CompiledICHolder* cichk = ic->cached_icholder();
- f->do_metadata(cichk->holder_metadata());
- f->do_metadata(cichk->holder_klass());
- } else {
- Metadata* ic_oop = ic->cached_metadata();
- if (ic_oop != nullptr) {
- f->do_metadata(ic_oop);
- }
- }
+ ic->metadata_do(f);
}
}
}
@@ -1750,8 +1758,7 @@ void nmethod::do_unloading(bool unloading_occurred) {
if (is_unloading()) {
unlink();
} else {
- guarantee(unload_nmethod_caches(unloading_occurred),
- "Should not need transition stubs");
+ unload_nmethod_caches(unloading_occurred);
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
if (bs_nm != nullptr) {
bs_nm->disarm(this);
@@ -2284,15 +2291,23 @@ void nmethod::verify() {
}
-void nmethod::verify_interrupt_point(address call_site) {
+void nmethod::verify_interrupt_point(address call_site, bool is_inline_cache) {
// Verify IC only when nmethod installation is finished.
if (!is_not_installed()) {
if (CompiledICLocker::is_safe(this)) {
- CompiledIC_at(this, call_site);
+ if (is_inline_cache) {
+ CompiledIC_at(this, call_site);
+ } else {
+ CompiledDirectCall::at(call_site);
+ }
} else {
CompiledICLocker ml_verify(this);
- CompiledIC_at(this, call_site);
+ if (is_inline_cache) {
+ CompiledIC_at(this, call_site);
+ } else {
+ CompiledDirectCall::at(call_site);
+ }
}
}
@@ -2316,15 +2331,15 @@ void nmethod::verify_scopes() {
address stub = nullptr;
switch (iter.type()) {
case relocInfo::virtual_call_type:
- verify_interrupt_point(iter.addr());
+ verify_interrupt_point(iter.addr(), true /* is_inline_cache */);
break;
case relocInfo::opt_virtual_call_type:
stub = iter.opt_virtual_call_reloc()->static_stub();
- verify_interrupt_point(iter.addr());
+ verify_interrupt_point(iter.addr(), false /* is_inline_cache */);
break;
case relocInfo::static_call_type:
stub = iter.static_call_reloc()->static_stub();
- //verify_interrupt_point(iter.addr());
+ verify_interrupt_point(iter.addr(), false /* is_inline_cache */);
break;
case relocInfo::runtime_call_type:
case relocInfo::runtime_call_w_cp_type: {
@@ -3239,75 +3254,6 @@ void nmethod::print_code_comment_on(outputStream* st, int column, address begin,
#endif
-class DirectNativeCallWrapper: public NativeCallWrapper {
-private:
- NativeCall* _call;
-
-public:
- DirectNativeCallWrapper(NativeCall* call) : _call(call) {}
-
- virtual address destination() const { return _call->destination(); }
- virtual address instruction_address() const { return _call->instruction_address(); }
- virtual address next_instruction_address() const { return _call->next_instruction_address(); }
- virtual address return_address() const { return _call->return_address(); }
-
- virtual address get_resolve_call_stub(bool is_optimized) const {
- if (is_optimized) {
- return SharedRuntime::get_resolve_opt_virtual_call_stub();
- }
- return SharedRuntime::get_resolve_virtual_call_stub();
- }
-
- virtual void set_destination_mt_safe(address dest) {
- _call->set_destination_mt_safe(dest);
- }
-
- virtual void set_to_interpreted(const methodHandle& method, CompiledICInfo& info) {
- CompiledDirectStaticCall* csc = CompiledDirectStaticCall::at(instruction_address());
- {
- csc->set_to_interpreted(method, info.entry());
- }
- }
-
- virtual void verify() const {
- // make sure code pattern is actually a call imm32 instruction
- _call->verify();
- _call->verify_alignment();
- }
-
- virtual void verify_resolve_call(address dest) const {
- CodeBlob* db = CodeCache::find_blob(dest);
- assert(db != nullptr && !db->is_adapter_blob(), "must use stub!");
- }
-
- virtual bool is_call_to_interpreted(address dest) const {
- CodeBlob* cb = CodeCache::find_blob(_call->instruction_address());
- return cb->contains(dest);
- }
-
- virtual bool is_safe_for_patching() const { return false; }
-
- virtual NativeInstruction* get_load_instruction(virtual_call_Relocation* r) const {
- return nativeMovConstReg_at(r->cached_value());
- }
-
- virtual void *get_data(NativeInstruction* instruction) const {
- return (void*)((NativeMovConstReg*) instruction)->data();
- }
-
- virtual void set_data(NativeInstruction* instruction, intptr_t data) {
- ((NativeMovConstReg*) instruction)->set_data(data);
- }
-};
-
-NativeCallWrapper* nmethod::call_wrapper_at(address call) const {
- return new DirectNativeCallWrapper((NativeCall*) call);
-}
-
-NativeCallWrapper* nmethod::call_wrapper_before(address return_pc) const {
- return new DirectNativeCallWrapper(nativeCall_before(return_pc));
-}
-
address nmethod::call_instruction_address(address pc) const {
if (NativeCall::is_call_before(pc)) {
NativeCall *ncall = nativeCall_before(pc);
@@ -3316,18 +3262,6 @@ address nmethod::call_instruction_address(address pc) const {
return nullptr;
}
-CompiledStaticCall* nmethod::compiledStaticCall_at(Relocation* call_site) const {
- return CompiledDirectStaticCall::at(call_site);
-}
-
-CompiledStaticCall* nmethod::compiledStaticCall_at(address call_site) const {
- return CompiledDirectStaticCall::at(call_site);
-}
-
-CompiledStaticCall* nmethod::compiledStaticCall_before(address return_addr) const {
- return CompiledDirectStaticCall::before(return_addr);
-}
-
#if defined(SUPPORT_DATA_STRUCTS)
void nmethod::print_value_on(outputStream* st) const {
st->print("nmethod");
@@ -3341,15 +3275,15 @@ void nmethod::print_calls(outputStream* st) {
RelocIterator iter(this);
while (iter.next()) {
switch (iter.type()) {
- case relocInfo::virtual_call_type:
- case relocInfo::opt_virtual_call_type: {
+ case relocInfo::virtual_call_type: {
CompiledICLocker ml_verify(this);
CompiledIC_at(&iter)->print();
break;
}
case relocInfo::static_call_type:
- st->print_cr("Static call at " INTPTR_FORMAT, p2i(iter.reloc()->addr()));
- CompiledDirectStaticCall::at(iter.reloc())->print();
+ case relocInfo::opt_virtual_call_type:
+ st->print_cr("Direct call at " INTPTR_FORMAT, p2i(iter.reloc()->addr()));
+ CompiledDirectCall::at(iter.reloc())->print();
break;
default:
break;
diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp
index 656dae4171b9a..2993db21305ee 100644
--- a/src/hotspot/share/code/nmethod.hpp
+++ b/src/hotspot/share/code/nmethod.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
#include "code/compiledMethod.hpp"
+class CompiledICData;
class CompileTask;
class DepChange;
class DirectiveSet;
@@ -196,6 +197,7 @@ class nmethod : public CompiledMethod {
address _verified_entry_point; // entry point without class check
address _osr_entry_point; // entry point for on stack replacement
+ CompiledICData* _compiled_ic_data;
bool _is_unlinked;
// Shared fields for all nmethod's
@@ -604,7 +606,7 @@ class nmethod : public CompiledMethod {
// verify operations
void verify();
void verify_scopes();
- void verify_interrupt_point(address interrupt_point);
+ void verify_interrupt_point(address interrupt_point, bool is_inline_cache);
// Disassemble this nmethod with additional debug information, e.g. information about blocks.
void decode2(outputStream* st) const;
@@ -621,7 +623,6 @@ class nmethod : public CompiledMethod {
#if defined(SUPPORT_DATA_STRUCTS)
// print output in opt build for disassembler library
void print_relocations() PRODUCT_RETURN;
- void print_pcs() { print_pcs_on(tty); }
void print_pcs_on(outputStream* st);
void print_scopes() { print_scopes_on(tty); }
void print_scopes_on(outputStream* st) PRODUCT_RETURN;
@@ -635,8 +636,7 @@ class nmethod : public CompiledMethod {
void print_oops(outputStream* st); // oops from the underlying CodeBlob.
void print_metadata(outputStream* st); // metadata in metadata pool.
#else
- // void print_pcs() PRODUCT_RETURN;
- void print_pcs() { return; }
+ void print_pcs_on(outputStream* st) { return; }
#endif
void print_calls(outputStream* st) PRODUCT_RETURN;
@@ -701,14 +701,8 @@ class nmethod : public CompiledMethod {
virtual void metadata_do(MetadataClosure* f);
- NativeCallWrapper* call_wrapper_at(address call) const;
- NativeCallWrapper* call_wrapper_before(address return_pc) const;
address call_instruction_address(address pc) const;
- virtual CompiledStaticCall* compiledStaticCall_at(Relocation* call_site) const;
- virtual CompiledStaticCall* compiledStaticCall_at(address addr) const;
- virtual CompiledStaticCall* compiledStaticCall_before(address addr) const;
-
virtual void make_deoptimized();
void finalize_relocations();
};
diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp
index bfb3db72d7265..ef90875767503 100644
--- a/src/hotspot/share/code/relocInfo.cpp
+++ b/src/hotspot/share/code/relocInfo.cpp
@@ -641,12 +641,10 @@ Method* virtual_call_Relocation::method_value() {
return (Method*)m;
}
-bool virtual_call_Relocation::clear_inline_cache() {
- // No stubs for ICs
- // Clean IC
+void virtual_call_Relocation::clear_inline_cache() {
ResourceMark rm;
CompiledIC* icache = CompiledIC_at(this);
- return icache->set_to_clean();
+ icache->set_to_clean();
}
@@ -669,18 +667,10 @@ Method* opt_virtual_call_Relocation::method_value() {
return (Method*)m;
}
-template
-static bool set_to_clean_no_ic_refill(CompiledICorStaticCall* ic) {
- guarantee(ic->set_to_clean(), "Should not need transition stubs");
- return true;
-}
-
-bool opt_virtual_call_Relocation::clear_inline_cache() {
- // No stubs for ICs
- // Clean IC
+void opt_virtual_call_Relocation::clear_inline_cache() {
ResourceMark rm;
- CompiledIC* icache = CompiledIC_at(this);
- return set_to_clean_no_ic_refill(icache);
+ CompiledDirectCall* callsite = CompiledDirectCall::at(this);
+ callsite->set_to_clean();
}
address opt_virtual_call_Relocation::static_stub() {
@@ -717,10 +707,10 @@ void static_call_Relocation::unpack_data() {
_method_index = unpack_1_int();
}
-bool static_call_Relocation::clear_inline_cache() {
- // Safe call site info
- CompiledStaticCall* handler = this->code()->compiledStaticCall_at(this);
- return set_to_clean_no_ic_refill(handler);
+void static_call_Relocation::clear_inline_cache() {
+ ResourceMark rm;
+ CompiledDirectCall* callsite = CompiledDirectCall::at(this);
+ callsite->set_to_clean();
}
@@ -759,11 +749,10 @@ address trampoline_stub_Relocation::get_trampoline_for(address call, nmethod* co
return nullptr;
}
-bool static_stub_Relocation::clear_inline_cache() {
+void static_stub_Relocation::clear_inline_cache() {
// Call stub is only used when calling the interpreted code.
// It does not really need to be cleared, except that we want to clean out the methodoop.
- CompiledDirectStaticCall::set_stub_to_clean(this);
- return true;
+ CompiledDirectCall::set_stub_to_clean(this);
}
diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp
index 77e24708bb143..5f67f94bdadc5 100644
--- a/src/hotspot/share/code/relocInfo.hpp
+++ b/src/hotspot/share/code/relocInfo.hpp
@@ -862,7 +862,7 @@ class Relocation {
// all relocations are able to reassert their values
virtual void set_value(address x);
- virtual bool clear_inline_cache() { return true; }
+ virtual void clear_inline_cache() {}
// This method assumes that all virtual/static (inline) caches are cleared (since for static_call_type and
// ic_call_type is not always position dependent (depending on the state of the cache)). However, this is
@@ -1141,7 +1141,7 @@ class virtual_call_Relocation : public CallRelocation {
void pack_data_to(CodeSection* dest) override;
void unpack_data() override;
- bool clear_inline_cache() override;
+ void clear_inline_cache() override;
};
@@ -1170,7 +1170,7 @@ class opt_virtual_call_Relocation : public CallRelocation {
void pack_data_to(CodeSection* dest) override;
void unpack_data() override;
- bool clear_inline_cache() override;
+ void clear_inline_cache() override;
// find the matching static_stub
address static_stub();
@@ -1202,7 +1202,7 @@ class static_call_Relocation : public CallRelocation {
void pack_data_to(CodeSection* dest) override;
void unpack_data() override;
- bool clear_inline_cache() override;
+ void clear_inline_cache() override;
// find the matching static_stub
address static_stub();
@@ -1227,7 +1227,7 @@ class static_stub_Relocation : public Relocation {
static_stub_Relocation() : Relocation(relocInfo::static_stub_type) { }
public:
- bool clear_inline_cache() override;
+ void clear_inline_cache() override;
address static_call() { return _static_call; }
diff --git a/src/hotspot/share/code/stubs.cpp b/src/hotspot/share/code/stubs.cpp
index a24b3e6ce019a..69195c3ecb71f 100644
--- a/src/hotspot/share/code/stubs.cpp
+++ b/src/hotspot/share/code/stubs.cpp
@@ -73,9 +73,15 @@ StubQueue::StubQueue(StubInterface* stub_interface, int buffer_size,
vm_exit_out_of_memory(size, OOM_MALLOC_ERROR, "CodeCache: no room for %s", name);
}
_stub_interface = stub_interface;
- _buffer_size = blob->content_size();
- _buffer_limit = blob->content_size();
- _stub_buffer = blob->content_begin();
+
+ // The code blob alignment can be smaller than the requested stub alignment.
+ // Make sure we put the stubs at their requested alignment by aligning the buffer base and limits.
+ address aligned_start = align_up(blob->content_begin(), stub_alignment());
+ address aligned_end = align_down(blob->content_end(), stub_alignment());
+ int aligned_size = aligned_end - aligned_start;
+ _buffer_size = aligned_size;
+ _buffer_limit = aligned_size;
+ _stub_buffer = aligned_start;
_queue_begin = 0;
_queue_end = 0;
_number_of_stubs = 0;
@@ -94,8 +100,11 @@ void StubQueue::deallocate_unused_tail() {
CodeBlob* blob = CodeCache::find_blob((void*)_stub_buffer);
CodeCache::free_unused_tail(blob, used_space());
// Update the limits to the new, trimmed CodeBlob size
- _buffer_size = blob->content_size();
- _buffer_limit = blob->content_size();
+ address aligned_start = align_up(blob->content_begin(), stub_alignment());
+ address aligned_end = align_down(blob->content_end(), stub_alignment());
+ int aligned_size = aligned_end - aligned_start;
+ _buffer_size = aligned_size;
+ _buffer_limit = aligned_size;
}
Stub* StubQueue::stub_containing(address pc) const {
diff --git a/src/hotspot/share/code/stubs.hpp b/src/hotspot/share/code/stubs.hpp
index 7b8ee922101d8..c448e0bddfaad 100644
--- a/src/hotspot/share/code/stubs.hpp
+++ b/src/hotspot/share/code/stubs.hpp
@@ -49,7 +49,7 @@ class Mutex;
// | | |
// | data | |
// | | |
-// code_begin -->|--------| | <--- aligned by CodeEntryAlignment
+// code_begin -->|--------| | <--- aligned by code_alignment()
// | | |
// | | |
// | code | | size
@@ -99,6 +99,7 @@ class StubInterface: public CHeapObj {
// General info/converters
virtual int size(Stub* self) const = 0; // the total size of the stub in bytes (must be a multiple of HeapWordSize)
virtual int alignment() const = 0; // computes the alignment
+ virtual int code_alignment() const = 0; // computes the code alignment
// Code info
virtual address code_begin(Stub* self) const = 0; // points to the first code byte
@@ -127,6 +128,7 @@ class StubInterface: public CHeapObj {
/* General info */ \
virtual int size(Stub* self) const { return cast(self)->size(); } \
virtual int alignment() const { return stub::alignment(); } \
+ virtual int code_alignment() const { return stub::code_alignment(); } \
\
/* Code info */ \
virtual address code_begin(Stub* self) const { return cast(self)->code_begin(); } \
@@ -154,9 +156,10 @@ class StubQueue: public CHeapObj {
Mutex* const _mutex; // the lock used for a (request, commit) transaction
void check_index(int i) const { assert(0 <= i && i < _buffer_limit && i % stub_alignment() == 0, "illegal index"); }
+ void check_stub_align(Stub* s) const { assert(((intptr_t)s) % stub_alignment() == 0, "incorrect stub alignment"); }
bool is_contiguous() const { return _queue_begin <= _queue_end; }
int index_of(Stub* s) const { int i = (int)((address)s - _stub_buffer); check_index(i); return i; }
- Stub* stub_at(int i) const { check_index(i); return (Stub*)(_stub_buffer + i); }
+ Stub* stub_at(int i) const { check_index(i); Stub* s = (Stub*)(_stub_buffer + i); check_stub_align(s); return s; }
Stub* current_stub() const { return stub_at(_queue_end); }
// Stub functionality accessed via interface
diff --git a/src/hotspot/share/code/vtableStubs.cpp b/src/hotspot/share/code/vtableStubs.cpp
index eed3dc8e7876a..5a54426d6a420 100644
--- a/src/hotspot/share/code/vtableStubs.cpp
+++ b/src/hotspot/share/code/vtableStubs.cpp
@@ -283,13 +283,6 @@ VtableStub* VtableStubs::entry_point(address pc) {
return (s == stub) ? s : nullptr;
}
-bool VtableStubs::is_icholder_entry(address pc) {
- assert(contains(pc), "must contain all vtable blobs");
- VtableStub* stub = (VtableStub*)(pc - VtableStub::entry_offset());
- // itable stubs use CompiledICHolder.
- return stub->is_itable_stub();
-}
-
bool VtableStubs::contains(address pc) {
// simple solution for now - we may want to use
// a faster way if this function is called often
diff --git a/src/hotspot/share/code/vtableStubs.hpp b/src/hotspot/share/code/vtableStubs.hpp
index 7076e50f3e3d8..3993e1e72d5cf 100644
--- a/src/hotspot/share/code/vtableStubs.hpp
+++ b/src/hotspot/share/code/vtableStubs.hpp
@@ -107,7 +107,6 @@ class VtableStubs : AllStatic {
static address find_itable_stub(int itable_index) { return find_stub(false, itable_index); }
static VtableStub* entry_point(address pc); // vtable stub entry point for a pc
- static bool is_icholder_entry(address pc); // is the blob containing pc (which must be a vtable blob) an icholder?
static bool contains(address pc); // is pc within any stub?
static VtableStub* stub_containing(address pc); // stub containing pc or nullptr
static void initialize();
diff --git a/src/hotspot/share/compiler/cHeapStringHolder.cpp b/src/hotspot/share/compiler/cHeapStringHolder.cpp
new file mode 100644
index 0000000000000..0383e738a47c9
--- /dev/null
+++ b/src/hotspot/share/compiler/cHeapStringHolder.cpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "compiler/cHeapStringHolder.hpp"
+
+void CHeapStringHolder::set(const char* string) {
+ clear();
+ if (string != nullptr) {
+ size_t len = strlen(string);
+ _string = NEW_C_HEAP_ARRAY(char, len + 1, mtCompiler);
+ ::memcpy(_string, string, len);
+ _string[len] = 0; // terminating null
+ }
+}
+
+void CHeapStringHolder::clear() {
+ if (_string != nullptr) {
+ FREE_C_HEAP_ARRAY(char, _string);
+ _string = nullptr;
+ }
+}
diff --git a/src/hotspot/share/oops/compiledICHolder.inline.hpp b/src/hotspot/share/compiler/cHeapStringHolder.hpp
similarity index 58%
rename from src/hotspot/share/oops/compiledICHolder.inline.hpp
rename to src/hotspot/share/compiler/cHeapStringHolder.hpp
index efbfcec064762..46659c2a521be 100644
--- a/src/hotspot/share/oops/compiledICHolder.inline.hpp
+++ b/src/hotspot/share/compiler/cHeapStringHolder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,22 +22,29 @@
*
*/
-#ifndef SHARE_OOPS_COMPILEDICHOLDER_INLINE_HPP
-#define SHARE_OOPS_COMPILEDICHOLDER_INLINE_HPP
+#ifndef SHARE_COMPILER_CHEAPSTRINGHOLDER_HPP
+#define SHARE_COMPILER_CHEAPSTRINGHOLDER_HPP
-#include "oops/compiledICHolder.hpp"
+#include "memory/allocation.hpp"
-#include "oops/klass.inline.hpp"
+// Holder for a C-Heap allocated String
+// The user must ensure that the destructor is called, or at least clear.
+class CHeapStringHolder : public StackObj {
+private:
+ char* _string;
-inline bool CompiledICHolder::is_loader_alive() {
- Klass* k = _is_metadata_method ? ((Method*)_holder_metadata)->method_holder() : (Klass*)_holder_metadata;
- if (!k->is_loader_alive()) {
- return false;
- }
- if (!_holder_klass->is_loader_alive()) {
- return false;
- }
- return true;
-}
+public:
+ CHeapStringHolder() : _string(nullptr) {}
+ ~CHeapStringHolder() { clear(); };
+ NONCOPYABLE(CHeapStringHolder);
-#endif // SHARE_OOPS_COMPILEDICHOLDER_INLINE_HPP
+ // Allocate memory to hold a copy of string
+ void set(const char* string);
+
+ // Release allocated memory
+ void clear();
+
+ const char* get() const { return _string; };
+};
+
+#endif // SHARE_COMPILER_CHEAPSTRINGHOLDER_HPP
diff --git a/src/hotspot/share/compiler/compilationFailureInfo.cpp b/src/hotspot/share/compiler/compilationFailureInfo.cpp
index e3f3353589e54..fb94102ef1654 100644
--- a/src/hotspot/share/compiler/compilationFailureInfo.cpp
+++ b/src/hotspot/share/compiler/compilationFailureInfo.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2023, Red Hat, Inc. and/or its affiliates.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Red Hat, Inc. and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,11 +42,16 @@
#include "utilities/ostream.hpp"
#include "utilities/nativeCallStack.hpp"
+int CompilationFailureInfo::current_compile_id_or_0() {
+ ciEnv* env = ciEnv::current();
+ return (env != nullptr) ? env->compile_id() : 0;
+}
+
CompilationFailureInfo::CompilationFailureInfo(const char* failure_reason) :
_stack(2),
_failure_reason(os::strdup(failure_reason)),
_elapsed_seconds(os::elapsedTime()),
- _compile_id(ciEnv::current()->task()->compile_id())
+ _compile_id(current_compile_id_or_0())
{}
CompilationFailureInfo::~CompilationFailureInfo() {
diff --git a/src/hotspot/share/compiler/compilationFailureInfo.hpp b/src/hotspot/share/compiler/compilationFailureInfo.hpp
index 3de62eb69da5a..470865a2f6608 100644
--- a/src/hotspot/share/compiler/compilationFailureInfo.hpp
+++ b/src/hotspot/share/compiler/compilationFailureInfo.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2023, Red Hat, Inc. and/or its affiliates.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Red Hat, Inc. and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,7 @@ class CompilationFailureInfo : public CHeapObj {
char* const _failure_reason;
const double _elapsed_seconds;
const int _compile_id;
+ static int current_compile_id_or_0();
public:
CompilationFailureInfo(const char* failure_reason);
~CompilationFailureInfo();
diff --git a/src/hotspot/share/compiler/compilationMemoryStatistic.cpp b/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
index 33e797d235395..7c39024a82a79 100644
--- a/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
+++ b/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
@@ -405,7 +405,7 @@ void CompilationMemoryStatistic::on_end_compilation() {
if (env) {
const char* const failure_reason = env->failure_reason();
if (failure_reason != nullptr) {
- result = (failure_reason == failure_reason_memlimit()) ? "oom" : "err";
+ result = (strcmp(failure_reason, failure_reason_memlimit()) == 0) ? "oom" : "err";
}
}
diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp
index ff5fa44050c16..112a15233bde3 100644
--- a/src/hotspot/share/compiler/compileBroker.cpp
+++ b/src/hotspot/share/compiler/compileBroker.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -582,7 +582,7 @@ CompilerCounters::CompilerCounters() {
// c2 uses explicit CompilerPhaseType idToPhase mapping in opto/phasetype.hpp,
// so if c2 is used, it should be always registered first.
// This function is called during vm initialization.
-void register_jfr_phasetype_serializer(CompilerType compiler_type) {
+static void register_jfr_phasetype_serializer(CompilerType compiler_type) {
ResourceMark rm;
static bool first_registration = true;
if (compiler_type == compiler_jvmci) {
@@ -841,8 +841,15 @@ void DeoptimizeObjectsALotThread::deoptimize_objects_alot_loop_all() {
JavaThread* CompileBroker::make_thread(ThreadType type, jobject thread_handle, CompileQueue* queue, AbstractCompiler* comp, JavaThread* THREAD) {
- JavaThread* new_thread = nullptr;
+ Handle thread_oop(THREAD, JNIHandles::resolve_non_null(thread_handle));
+ if (java_lang_Thread::thread(thread_oop()) != nullptr) {
+ assert(type == compiler_t, "should only happen with reused compiler threads");
+ // The compiler thread hasn't actually exited yet so don't try to reuse it
+ return nullptr;
+ }
+
+ JavaThread* new_thread = nullptr;
switch (type) {
case compiler_t:
assert(comp != nullptr, "Compiler instance missing.");
@@ -871,7 +878,6 @@ JavaThread* CompileBroker::make_thread(ThreadType type, jobject thread_handle, C
// JavaThread due to lack of resources. We will handle that failure below.
// Also check new_thread so that static analysis is happy.
if (new_thread != nullptr && new_thread->osthread() != nullptr) {
- Handle thread_oop(THREAD, JNIHandles::resolve_non_null(thread_handle));
if (type == compiler_t) {
CompilerThread::cast(new_thread)->set_compiler(comp);
@@ -2322,7 +2328,9 @@ void CompileBroker::invoke_compiler_on_method(CompileTask* task) {
compilable = ci_env.compilable();
if (ci_env.failing()) {
- failure_reason = ci_env.failure_reason();
+ // Duplicate the failure reason string, so that it outlives ciEnv
+ failure_reason = os::strdup(ci_env.failure_reason(), mtCompiler);
+ failure_reason_on_C_heap = true;
retry_message = ci_env.retry_message();
ci_env.report_failure(failure_reason);
}
diff --git a/src/hotspot/share/compiler/compilerDirectives.cpp b/src/hotspot/share/compiler/compilerDirectives.cpp
index d7d4c5b60ed7f..a16e5f23ed954 100644
--- a/src/hotspot/share/compiler/compilerDirectives.cpp
+++ b/src/hotspot/share/compiler/compilerDirectives.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "memory/allocation.inline.hpp"
#include "memory/resourceArea.hpp"
#include "opto/phasetype.hpp"
+#include "opto/traceAutoVectorizationTag.hpp"
#include "runtime/globals_extension.hpp"
CompilerDirectives::CompilerDirectives() : _next(nullptr), _match(nullptr), _ref_count(0) {
@@ -300,7 +301,8 @@ void DirectiveSet::init_control_intrinsic() {
DirectiveSet::DirectiveSet(CompilerDirectives* d) :
_inlinematchers(nullptr),
_directive(d),
- _ideal_phase_name_set(PHASE_NUM_TYPES, mtCompiler)
+ _ideal_phase_name_set(PHASE_NUM_TYPES, mtCompiler),
+ _trace_auto_vectorization_tags(TRACE_AUTO_VECTORIZATION_TAG_NUM, mtCompiler)
{
#define init_defaults_definition(name, type, dvalue, compiler) this->name##Option = dvalue;
compilerdirectives_common_flags(init_defaults_definition)
@@ -433,6 +435,16 @@ DirectiveSet* DirectiveSet::compilecommand_compatibility_init(const methodHandle
// Parse PrintIdealPhaseName and create a lookup set
#ifndef PRODUCT
#ifdef COMPILER2
+ if (!_modified[TraceAutoVectorizationIndex]) {
+ // Parse ccstr and create mask
+ ccstrlist option;
+ if (CompilerOracle::has_option_value(method, CompileCommand::TraceAutoVectorization, option)) {
+ TraceAutoVectorizationTagValidator validator(option, false);
+ if (validator.is_valid()) {
+ set.cloned()->set_trace_auto_vectorization_tags(validator.tags());
+ }
+ }
+ }
if (!_modified[PrintIdealPhaseIndex]) {
// Parse ccstr and create set
ccstrlist option;
diff --git a/src/hotspot/share/compiler/compilerDirectives.hpp b/src/hotspot/share/compiler/compilerDirectives.hpp
index 4c9b51724f9a3..747c878181751 100644
--- a/src/hotspot/share/compiler/compilerDirectives.hpp
+++ b/src/hotspot/share/compiler/compilerDirectives.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -87,10 +87,10 @@ NOT_PRODUCT(cflags(PrintIdeal, bool, PrintIdeal, PrintIdeal)) \
cflags(Vectorize, bool, false, Vectorize) \
cflags(CloneMapDebug, bool, false, CloneMapDebug) \
NOT_PRODUCT(cflags(IGVPrintLevel, intx, PrintIdealGraphLevel, IGVPrintLevel)) \
- cflags(VectorizeDebug, uintx, 0, VectorizeDebug) \
cflags(IncrementalInlineForceCleanup, bool, IncrementalInlineForceCleanup, IncrementalInlineForceCleanup) \
cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit)
#define compilerdirectives_c2_string_flags(cflags) \
+NOT_PRODUCT(cflags(TraceAutoVectorization, ccstrlist, "", TraceAutoVectorization)) \
NOT_PRODUCT(cflags(PrintIdealPhase, ccstrlist, "", PrintIdealPhase))
#else
#define compilerdirectives_c2_other_flags(cflags)
@@ -131,6 +131,7 @@ class DirectiveSet : public CHeapObj {
CompilerDirectives* _directive;
TriBoolArray<(size_t)vmIntrinsics::number_of_intrinsics(), int> _intrinsic_control_words;
CHeapBitMap _ideal_phase_name_set;
+ CHeapBitMap _trace_auto_vectorization_tags;
public:
DirectiveSet(CompilerDirectives* directive);
@@ -205,6 +206,12 @@ void set_##name(void* value) { \
bool should_print_phase(const CompilerPhaseType cpt) const {
return _ideal_phase_name_set.at(cpt);
};
+ void set_trace_auto_vectorization_tags(const CHeapBitMap& tags) {
+ _trace_auto_vectorization_tags.set_from(tags);
+ };
+ const CHeapBitMap& trace_auto_vectorization_tags() {
+ return _trace_auto_vectorization_tags;
+ };
void print_intx(outputStream* st, ccstr n, intx v, bool mod) { if (mod) { st->print("%s:" INTX_FORMAT " ", n, v); } }
void print_uintx(outputStream* st, ccstr n, intx v, bool mod) { if (mod) { st->print("%s:" UINTX_FORMAT " ", n, v); } }
diff --git a/src/hotspot/share/compiler/compilerOracle.cpp b/src/hotspot/share/compiler/compilerOracle.cpp
index 66eea29fcc136..a8eee10fac539 100644
--- a/src/hotspot/share/compiler/compilerOracle.cpp
+++ b/src/hotspot/share/compiler/compilerOracle.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,6 +35,7 @@
#include "oops/method.inline.hpp"
#include "oops/symbol.hpp"
#include "opto/phasetype.hpp"
+#include "opto/traceAutoVectorizationTag.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/jniHandles.hpp"
@@ -47,7 +48,7 @@ static const char* optiontype_names[] = {
#undef enum_of_types
};
-const char* optiontype2name(enum OptionType type) {
+static const char* optiontype2name(enum OptionType type) {
return optiontype_names[static_cast(type)];
}
@@ -57,7 +58,7 @@ static enum OptionType option_types[] = {
#undef enum_of_options
};
-enum OptionType option2type(enum CompileCommand option) {
+static enum OptionType option2type(enum CompileCommand option) {
return option_types[static_cast(option)];
}
@@ -67,7 +68,7 @@ static const char* option_names[] = {
#undef enum_of_options
};
-const char* option2name(enum CompileCommand option) {
+static const char* option2name(enum CompileCommand option) {
return option_names[static_cast(option)];
}
@@ -107,7 +108,7 @@ static bool print_final_memstat_report = false;
// A filter for quick lookup if an option is set
static bool option_filter[static_cast(CompileCommand::Unknown) + 1] = { 0 };
-void command_set_in_filter(enum CompileCommand option) {
+static void command_set_in_filter(enum CompileCommand option) {
assert(option != CompileCommand::Unknown, "sanity");
assert(option2type(option) != OptionType::Unknown, "sanity");
@@ -119,7 +120,7 @@ void command_set_in_filter(enum CompileCommand option) {
option_filter[static_cast(option)] = true;
}
-bool has_command(enum CompileCommand option) {
+static bool has_command(enum CompileCommand option) {
return option_filter[static_cast(option)];
}
@@ -546,7 +547,7 @@ enum OptionType CompilerOracle::parse_option_type(const char* type_str) {
return OptionType::Unknown;
}
-void print_tip() { // CMH Update info
+static void print_tip() { // CMH Update info
tty->cr();
tty->print_cr("Usage: '-XX:CompileCommand=,' - to set boolean option to true");
tty->print_cr("Usage: '-XX:CompileCommand=,,'");
@@ -554,13 +555,13 @@ void print_tip() { // CMH Update info
tty->cr();
}
-void print_option(enum CompileCommand option, const char* name, enum OptionType type) {
+static void print_option(enum CompileCommand option, const char* name, enum OptionType type) {
if (type != OptionType::Unknown) {
tty->print_cr(" %s (%s)", name, optiontype2name(type));
}
}
-void print_commands() {
+static void print_commands() {
tty->cr();
tty->print_cr("All available options:");
#define enum_of_options(option, name, ctype) print_option(CompileCommand::option, name, OptionType::ctype);
@@ -620,7 +621,7 @@ static void usage() {
tty->cr();
};
-int skip_whitespace(char* &line) {
+static int skip_whitespace(char* &line) {
// Skip any leading spaces
int whitespace_read = 0;
sscanf(line, "%*[ \t]%n", &whitespace_read);
@@ -628,7 +629,7 @@ int skip_whitespace(char* &line) {
return whitespace_read;
}
-void skip_comma(char* &line) {
+static void skip_comma(char* &line) {
// Skip any leading spaces
if (*line == ',') {
line++;
@@ -775,8 +776,14 @@ static void scan_value(enum OptionType type, char* line, int& total_bytes_read,
jio_snprintf(errorbuf, buf_size, "Unrecognized intrinsic detected in %s: %s", option2name(option), validator.what());
}
}
-#ifndef PRODUCT
- else if (option == CompileCommand::PrintIdealPhase) {
+#if !defined(PRODUCT) && defined(COMPILER2)
+ else if (option == CompileCommand::TraceAutoVectorization) {
+ TraceAutoVectorizationTagValidator validator(value, true);
+
+ if (!validator.is_valid()) {
+ jio_snprintf(errorbuf, buf_size, "Unrecognized tag name in %s: %s", option2name(option), validator.what());
+ }
+ } else if (option == CompileCommand::PrintIdealPhase) {
PhaseNameValidator validator(value);
if (!validator.is_valid()) {
diff --git a/src/hotspot/share/compiler/compilerOracle.hpp b/src/hotspot/share/compiler/compilerOracle.hpp
index be1a270f3c957..192e292d35a2b 100644
--- a/src/hotspot/share/compiler/compilerOracle.hpp
+++ b/src/hotspot/share/compiler/compilerOracle.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -86,8 +86,8 @@ NOT_PRODUCT(option(TraceEscapeAnalysis, "TraceEscapeAnalysis", Bool)) \
NOT_PRODUCT(option(PrintIdeal, "PrintIdeal", Bool)) \
NOT_PRODUCT(option(PrintIdealPhase, "PrintIdealPhase", Ccstrlist)) \
NOT_PRODUCT(option(IGVPrintLevel, "IGVPrintLevel", Intx)) \
+NOT_PRODUCT(option(TraceAutoVectorization, "TraceAutoVectorization", Ccstrlist)) \
option(Vectorize, "Vectorize", Bool) \
- option(VectorizeDebug, "VectorizeDebug", Uintx) \
option(CloneMapDebug, "CloneMapDebug", Bool) \
option(IncrementalInlineForceCleanup, "IncrementalInlineForceCleanup", Bool) \
option(MaxNodeLimit, "MaxNodeLimit", Intx) \
diff --git a/src/hotspot/share/compiler/directivesParser.cpp b/src/hotspot/share/compiler/directivesParser.cpp
index 60955615133e2..5501490028349 100644
--- a/src/hotspot/share/compiler/directivesParser.cpp
+++ b/src/hotspot/share/compiler/directivesParser.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
#include "memory/allocation.inline.hpp"
#include "memory/resourceArea.hpp"
#include "opto/phasetype.hpp"
+#include "opto/traceAutoVectorizationTag.hpp"
#include "runtime/os.hpp"
#include
@@ -335,6 +336,17 @@ bool DirectivesParser::set_option_flag(JSON_TYPE t, JSON_VAL* v, const key* opti
if (!valid) {
error(VALUE_ERROR, "Unrecognized intrinsic detected in DisableIntrinsic: %s", validator.what());
}
+ }
+#if !defined(PRODUCT) && defined(COMPILER2)
+ else if (strncmp(option_key->name, "TraceAutoVectorization", 22) == 0) {
+ TraceAutoVectorizationTagValidator validator(s, false);
+
+ valid = validator.is_valid();
+ if (valid) {
+ set->set_trace_auto_vectorization_tags(validator.tags());
+ } else {
+ error(VALUE_ERROR, "Unrecognized tag name detected in TraceAutoVectorization: %s", validator.what());
+ }
} else if (strncmp(option_key->name, "PrintIdealPhase", 15) == 0) {
PhaseNameValidator validator(s);
@@ -345,6 +357,7 @@ bool DirectivesParser::set_option_flag(JSON_TYPE t, JSON_VAL* v, const key* opti
error(VALUE_ERROR, "Unrecognized phase name detected in PrintIdealPhase: %s", validator.what());
}
}
+#endif
if (!valid) {
FREE_C_HEAP_ARRAY(char, s);
diff --git a/src/hotspot/share/compiler/methodMatcher.cpp b/src/hotspot/share/compiler/methodMatcher.cpp
index 3e2ee288c80ed..de2e4e82bba3f 100644
--- a/src/hotspot/share/compiler/methodMatcher.cpp
+++ b/src/hotspot/share/compiler/methodMatcher.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -246,7 +246,7 @@ static MethodMatcher::Mode check_mode(char name[], const char*& error_msg) {
}
// Skip any leading spaces
-void skip_leading_spaces(char*& line, int* total_bytes_read ) {
+static void skip_leading_spaces(char*& line, int* total_bytes_read ) {
int bytes_read = 0;
sscanf(line, "%*[ \t]%n", &bytes_read);
if (bytes_read > 0) {
diff --git a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp
index 741f1d656036d..f22d1c22ac8a7 100644
--- a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp
+++ b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp
@@ -37,7 +37,6 @@
class EpsilonHeap : public CollectedHeap {
friend class VMStructs;
private:
- SoftRefPolicy _soft_ref_policy;
EpsilonMonitoringSupport* _monitoring_support;
MemoryPool* _pool;
GCMemoryManager _memory_manager;
@@ -65,10 +64,6 @@ class EpsilonHeap : public CollectedHeap {
return "Epsilon";
}
- SoftRefPolicy* soft_ref_policy() override {
- return &_soft_ref_policy;
- }
-
jint initialize() override;
void initialize_serviceability() override;
diff --git a/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp b/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp
index ef2f23633b9ad..53d4aa221f214 100644
--- a/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp
+++ b/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp
@@ -28,8 +28,8 @@
#include "gc/g1/c1/g1BarrierSetC1.hpp"
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1BarrierSetAssembler.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "utilities/macros.hpp"
#ifdef ASSERT
diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
index 4d45c227f8c07..807e9aa9c5938 100644
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
@@ -29,7 +29,7 @@
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "opto/arraycopynode.hpp"
#include "opto/compile.hpp"
#include "opto/escape.hpp"
@@ -389,7 +389,7 @@ void G1BarrierSetC2::post_barrier(GraphKit* kit,
if (use_ReduceInitialCardMarks() && obj == kit->just_allocated_object(kit->control())) {
// We can skip marks on a freshly-allocated object in Eden.
- // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
+ // Keep this code in sync with CardTableBarrierSet::on_slowpath_allocation_exit.
// That routine informs GC to take appropriate compensating steps,
// upon a slow-path allocation, so as to make this card-mark
// elision safe.
diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.hpp b/src/hotspot/share/gc/g1/g1AllocRegion.hpp
index 600d155125936..b39a717d3b708 100644
--- a/src/hotspot/share/gc/g1/g1AllocRegion.hpp
+++ b/src/hotspot/share/gc/g1/g1AllocRegion.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_GC_G1_G1ALLOCREGION_HPP
#define SHARE_GC_G1_G1ALLOCREGION_HPP
-#include "gc/g1/heapRegion.hpp"
#include "gc/g1/g1EvacStats.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegionAttr.hpp"
#include "gc/g1/g1NUMA.hpp"
diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp b/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp
index 33c402b00871e..00ef254e84c37 100644
--- a/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp
@@ -27,7 +27,7 @@
#include "gc/g1/g1AllocRegion.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#define assert_alloc_region(p, message) \
do { \
diff --git a/src/hotspot/share/gc/g1/g1Allocator.cpp b/src/hotspot/share/gc/g1/g1Allocator.cpp
index ffd2b65f555e7..3964ad8c8af3b 100644
--- a/src/hotspot/share/gc/g1/g1Allocator.cpp
+++ b/src/hotspot/share/gc/g1/g1Allocator.cpp
@@ -28,11 +28,11 @@
#include "gc/g1/g1EvacInfo.hpp"
#include "gc/g1/g1EvacStats.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
+#include "gc/g1/g1HeapRegionType.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1Policy.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
-#include "gc/g1/heapRegionType.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "runtime/mutexLocker.hpp"
#include "utilities/align.hpp"
diff --git a/src/hotspot/share/gc/g1/g1Arguments.cpp b/src/hotspot/share/gc/g1/g1Arguments.cpp
index 79b50cd0ea9ee..d0d9cd3129709 100644
--- a/src/hotspot/share/gc/g1/g1Arguments.cpp
+++ b/src/hotspot/share/gc/g1/g1Arguments.cpp
@@ -29,10 +29,10 @@
#include "gc/g1/g1CardSet.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionBounds.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionBounds.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/shared/cardTable.hpp"
#include "gc/shared/gcArguments.hpp"
#include "gc/shared/workerPolicy.hpp"
@@ -143,11 +143,6 @@ void G1Arguments::initialize_card_set_configuration() {
G1RemSetArrayOfCardsEntriesBase << region_size_log_mb));
}
- // Round to next 8 byte boundary for array to maximize space usage.
- size_t const cur_size = G1CardSetArray::size_in_bytes(G1RemSetArrayOfCardsEntries);
- FLAG_SET_ERGO(G1RemSetArrayOfCardsEntries,
- G1RemSetArrayOfCardsEntries + (uint)(align_up(cur_size, G1CardSetAllocOptions::SlotAlignment) - cur_size) / sizeof(G1CardSetArray::EntryDataType));
-
// Howl card set container globals.
if (FLAG_IS_DEFAULT(G1RemSetHowlNumBuckets)) {
FLAG_SET_ERGO(G1RemSetHowlNumBuckets, G1CardSetHowl::num_buckets(HeapRegion::CardsPerRegion,
diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.cpp b/src/hotspot/share/gc/g1/g1BarrierSet.cpp
index 3f2b27c2d063c..448f2f5fccf04 100644
--- a/src/hotspot/share/gc/g1/g1BarrierSet.cpp
+++ b/src/hotspot/share/gc/g1/g1BarrierSet.cpp
@@ -27,9 +27,10 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1CardTable.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1RegionPinCache.inline.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/satbMarkQueue.hpp"
#include "logging/log.hpp"
#include "oops/access.inline.hpp"
@@ -171,4 +172,8 @@ void G1BarrierSet::on_thread_detach(Thread* thread) {
qset.flush_queue(queue);
qset.record_detached_refinement_stats(queue.refinement_stats());
}
+ {
+ G1RegionPinCache& cache = G1ThreadLocalData::pin_count_cache(thread);
+ cache.flush();
+ }
}
diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
index 71107387564a2..af47ce92dc511 100644
--- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
+++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "logging/log.hpp"
#include "nmt/memTracker.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
index a29050c04b34f..80697d38f7d4b 100644
--- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
@@ -27,7 +27,7 @@
#include "gc/g1/g1BlockOffsetTable.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/shared/memset_with_concurrent_readers.hpp"
#include "runtime/atomic.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CardSet.cpp b/src/hotspot/share/gc/g1/g1CardSet.cpp
index 58ae441f51ff1..101822dbc4428 100644
--- a/src/hotspot/share/gc/g1/g1CardSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CardSet.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
#include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CardSetMemory.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gcLogPrecious.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "memory/allocation.inline.hpp"
@@ -286,7 +286,10 @@ class G1CardSetHashTable : public CHeapObj {
size_t initial_log_table_size = InitialLogTableSize) :
_inserted_card(false),
_mm(mm),
- _table(mm, initial_log_table_size, false),
+ _table(Mutex::service-1,
+ mm,
+ initial_log_table_size,
+ false /* enable_statistics */),
_table_scanner(&_table, BucketClaimSize) {
}
diff --git a/src/hotspot/share/gc/g1/g1CardTable.inline.hpp b/src/hotspot/share/gc/g1/g1CardTable.inline.hpp
index d48fce6cb9ed1..0585b68e33644 100644
--- a/src/hotspot/share/gc/g1/g1CardTable.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1CardTable.inline.hpp
@@ -27,7 +27,7 @@
#include "gc/g1/g1CardTable.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
inline uint G1CardTable::region_idx_for(CardValue* p) {
size_t const card_idx = pointer_delta(p, _byte_map, sizeof(CardValue));
diff --git a/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp b/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp
index 77211af436444..5e99b91e76a50 100644
--- a/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp
+++ b/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp
@@ -27,8 +27,8 @@
#include "gc/g1/g1CodeBlobClosure.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CodeRootSet.cpp b/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
index 3b933dfa7f903..135c1bca2e305 100644
--- a/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,7 @@
#include "code/codeCache.hpp"
#include "code/nmethod.hpp"
#include "gc/g1/g1CodeRootSet.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomic.hpp"
@@ -84,8 +84,10 @@ class G1CodeRootSetHashTable : public CHeapObj {
public:
G1CodeRootSetHashTable() :
- _table(Log2DefaultNumBuckets,
- HashTable::DEFAULT_MAX_SIZE_LOG2),
+ _table(Mutex::service-1,
+ nullptr,
+ Log2DefaultNumBuckets,
+ false /* enable_statistics */),
_table_scanner(&_table, BucketClaimSize), _num_entries(0) {
clear();
}
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index 7a630c27f7963..ec3d5fa4ecc44 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,6 @@
#include "classfile/metadataOnStackMark.hpp"
#include "classfile/systemDictionary.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "compiler/oopMap.hpp"
#include "gc/g1/g1Allocator.inline.hpp"
#include "gc/g1/g1Arguments.hpp"
@@ -47,6 +46,9 @@
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "gc/g1/g1GCPauseType.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1HeapSizingPolicy.hpp"
#include "gc/g1/g1HeapTransition.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
@@ -59,6 +61,7 @@
#include "gc/g1/g1PeriodicGCTask.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RedirtyCardsQueue.hpp"
+#include "gc/g1/g1RegionPinCache.inline.hpp"
#include "gc/g1/g1RegionToSpaceMapper.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1RootClosures.hpp"
@@ -71,9 +74,6 @@
#include "gc/g1/g1VMOperations.hpp"
#include "gc/g1/g1YoungCollector.hpp"
#include "gc/g1/g1YoungGCAllocationFailureInjector.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/concurrentGCBreakpoints.hpp"
#include "gc/shared/gcBehaviours.hpp"
@@ -1136,7 +1136,6 @@ G1CollectedHeap::G1CollectedHeap() :
_workers(nullptr),
_card_table(nullptr),
_collection_pause_end(Ticks::now()),
- _soft_ref_policy(),
_old_set("Old Region Set", new OldRegionSetChecker()),
_humongous_set("Humongous Region Set", new HumongousRegionSetChecker()),
_bot(nullptr),
@@ -1525,10 +1524,6 @@ void G1CollectedHeap::ref_processing_init() {
&_is_alive_closure_stw); // is alive closure
}
-SoftRefPolicy* G1CollectedHeap::soft_ref_policy() {
- return &_soft_ref_policy;
-}
-
size_t G1CollectedHeap::capacity() const {
return _hrm.length() * HeapRegion::GrainBytes;
}
@@ -2202,8 +2197,6 @@ void G1CollectedHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) {
}
void G1CollectedHeap::gc_prologue(bool full) {
- assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
-
// Update common counters.
increment_total_collections(full /* full gc */);
if (full || collector_state()->in_concurrent_start_gc()) {
@@ -2471,6 +2464,12 @@ void G1CollectedHeap::retire_tlabs() {
ensure_parsability(true);
}
+void G1CollectedHeap::flush_region_pin_cache() {
+ for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
+ G1ThreadLocalData::pin_count_cache(thread).flush();
+ }
+}
+
void G1CollectedHeap::do_collection_pause_at_safepoint_helper() {
ResourceMark rm;
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
index 58ac36a734f7f..b4157f16493b5 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,6 +36,8 @@
#include "gc/g1/g1EvacStats.hpp"
#include "gc/g1/g1GCPauseType.hpp"
#include "gc/g1/g1HeapRegionAttr.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1HeapTransition.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1HRPrinter.hpp"
@@ -45,8 +47,6 @@
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1SurvivorRegions.hpp"
#include "gc/g1/g1YoungGCAllocationFailureInjector.hpp"
-#include "gc/g1/heapRegionManager.hpp"
-#include "gc/g1/heapRegionSet.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/gcHeapSummary.hpp"
@@ -178,8 +178,6 @@ class G1CollectedHeap : public CollectedHeap {
Ticks _collection_pause_end;
- SoftRefPolicy _soft_ref_policy;
-
static size_t _humongous_object_threshold_in_words;
// These sets keep track of old and humongous regions respectively.
@@ -775,6 +773,10 @@ class G1CollectedHeap : public CollectedHeap {
void retire_tlabs();
+ // Update all region's pin counts from the per-thread caches and resets them.
+ // Must be called before any decision based on pin counts.
+ void flush_region_pin_cache();
+
void expand_heap_after_young_collection();
// Update object copying statistics.
void record_obj_copy_mem_stats();
@@ -923,8 +925,6 @@ class G1CollectedHeap : public CollectedHeap {
inline bool is_collection_set_candidate(const HeapRegion* r) const;
- SoftRefPolicy* soft_ref_policy() override;
-
void initialize_serviceability() override;
MemoryUsage memory_usage() override;
GrowableArray memory_managers() override;
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
index 7b81a93abe177..513b0a4505ba7 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,12 +31,13 @@
#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1EvacFailureRegions.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1Policy.hpp"
+#include "gc/g1/g1RegionPinCache.inline.hpp"
#include "gc/g1/g1RemSet.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
#include "gc/shared/markBitMap.inline.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "oops/stackChunkOop.hpp"
@@ -266,15 +267,17 @@ inline void G1CollectedHeap::pin_object(JavaThread* thread, oop obj) {
assert(obj != nullptr, "obj must not be null");
assert(!is_gc_active(), "must not pin objects during a GC");
assert(obj->is_typeArray(), "must be typeArray");
- HeapRegion *r = heap_region_containing(obj);
- r->increment_pinned_object_count();
+
+ uint obj_region_idx = heap_region_containing(obj)->hrm_index();
+ G1ThreadLocalData::pin_count_cache(thread).inc_count(obj_region_idx);
}
inline void G1CollectedHeap::unpin_object(JavaThread* thread, oop obj) {
assert(obj != nullptr, "obj must not be null");
assert(!is_gc_active(), "must not unpin objects during a GC");
- HeapRegion *r = heap_region_containing(obj);
- r->decrement_pinned_object_count();
+
+ uint obj_region_idx = heap_region_containing(obj)->hrm_index();
+ G1ThreadLocalData::pin_count_cache(thread).dec_count(obj_region_idx);
}
inline bool G1CollectedHeap::is_obj_dead(const oop obj) const {
diff --git a/src/hotspot/share/gc/g1/g1CollectionSet.cpp b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
index 30787adbe543b..3ccd8b69644b4 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
@@ -28,11 +28,11 @@
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
#include "gc/g1/g1CollectorState.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1ParScanThreadState.hpp"
#include "gc/g1/g1Policy.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionSet.hpp"
#include "logging/logStream.hpp"
#include "runtime/orderAccess.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
index 9f903bc924f1e..b4649414b10e6 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "utilities/bitMap.inline.hpp"
#include "utilities/growableArray.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp b/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
index f19488a49a8ae..ca7f2b373985b 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
@@ -26,7 +26,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/shared/space.inline.hpp"
#include "runtime/atomic.hpp"
#include "utilities/quickSort.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp b/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp
index 5b005205551b3..62f9b2a5cef02 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP
#define SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/shared/gc_globals.hpp"
#include "memory/allStatic.hpp"
#include "runtime/globals.hpp"
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
index 533817f97246c..5d81f49546c32 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,16 +34,16 @@
#include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
#include "gc/g1/g1ConcurrentRebuildAndScrub.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
#include "gc/g1/g1Trace.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionManager.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
#include "gc/shared/gcId.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
@@ -152,7 +152,8 @@ G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_
MutexLocker x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
if (Atomic::load_acquire(&_buckets[bucket]) == nullptr) {
- if (!expand()) {
+ size_t desired_capacity = bucket_size(bucket) * 2;
+ if (!try_expand_to(desired_capacity)) {
return nullptr;
}
}
@@ -196,23 +197,28 @@ bool G1CMMarkStack::ChunkAllocator::initialize(size_t initial_capacity, size_t m
return true;
}
-bool G1CMMarkStack::ChunkAllocator::expand() {
+bool G1CMMarkStack::ChunkAllocator::try_expand_to(size_t desired_capacity) {
if (_capacity == _max_capacity) {
log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _capacity);
return false;
}
+
size_t old_capacity = _capacity;
- // Double capacity if possible.
- size_t new_capacity = MIN2(old_capacity * 2, _max_capacity);
+ desired_capacity = MIN2(desired_capacity, _max_capacity);
- if (reserve(new_capacity)) {
+ if (reserve(desired_capacity)) {
log_debug(gc)("Expanded the mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
- old_capacity, new_capacity);
+ old_capacity, desired_capacity);
return true;
}
return false;
}
+bool G1CMMarkStack::ChunkAllocator::try_expand() {
+ size_t new_capacity = _capacity * 2;
+ return try_expand_to(new_capacity);
+}
+
G1CMMarkStack::ChunkAllocator::~ChunkAllocator() {
if (_buckets == nullptr) {
return;
@@ -234,6 +240,9 @@ bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
size_t highest_bucket = get_bucket(new_capacity - 1);
size_t i = get_bucket(_capacity);
+ // Allocate all buckets associated with indexes between the current capacity (_capacity)
+ // and the new capacity (new_capacity). This step ensures that there are no gaps in the
+ // array and that the capacity accurately reflects the reserved memory.
for (; i <= highest_bucket; i++) {
if (Atomic::load_acquire(&_buckets[i]) != nullptr) {
continue; // Skip over already allocated buckets.
@@ -261,7 +270,7 @@ bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
}
void G1CMMarkStack::expand() {
- _chunk_allocator.expand();
+ _chunk_allocator.try_expand();
}
void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) {
@@ -480,12 +489,10 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
// _verbose_level set below
- _init_times(),
_remark_times(),
_remark_mark_times(),
_remark_weak_ref_times(),
_cleanup_times(),
- _total_cleanup_time(0.0),
_accum_task_vtime(nullptr),
@@ -1549,9 +1556,7 @@ void G1ConcurrentMark::cleanup() {
verify_during_pause(G1HeapVerifier::G1VerifyCleanup, VerifyLocation::CleanupAfter);
// Local statistics
- double recent_cleanup_time = (os::elapsedTime() - start);
- _total_cleanup_time += recent_cleanup_time;
- _cleanup_times.add(recent_cleanup_time);
+ _cleanup_times.add((os::elapsedTime() - start) * 1000.0);
{
GCTraceTime(Debug, gc, phases) debug("Finalize Concurrent Mark Cleanup", _gc_timer_cm);
@@ -2117,7 +2122,6 @@ void G1ConcurrentMark::print_summary_info() {
}
log.trace(" Concurrent marking:");
- print_ms_time_info(" ", "init marks", _init_times);
print_ms_time_info(" ", "remarks", _remark_times);
{
print_ms_time_info(" ", "final marks", _remark_mark_times);
@@ -2126,9 +2130,9 @@ void G1ConcurrentMark::print_summary_info() {
}
print_ms_time_info(" ", "cleanups", _cleanup_times);
log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
- _total_cleanup_time, (_cleanup_times.num() > 0 ? _total_cleanup_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
+ _cleanup_times.sum() / 1000.0, _cleanup_times.avg());
log.trace(" Total stop_world time = %8.2f s.",
- (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
+ (_remark_times.sum() + _cleanup_times.sum())/1000.0);
log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
cm_thread()->vtime_accum(), cm_thread()->vtime_mark_accum());
}
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
index a4315ec5f6c48..3d9693bed100b 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
@@ -27,9 +27,9 @@
#include "gc/g1/g1ConcurrentMarkBitMap.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1RegionMarkStatsCache.hpp"
-#include "gc/g1/heapRegionSet.hpp"
#include "gc/shared/gcCause.hpp"
#include "gc/shared/taskTerminator.hpp"
#include "gc/shared/taskqueue.hpp"
@@ -145,7 +145,7 @@ class G1CMMarkStack {
// within the bucket. Additionally, each new bucket added to the growable array doubles the capacity of
// the growable array.
//
- // Illustration of the Growable Array data structure.
+ // Illustration of the growable array data structure.
//
// +----+ +----+----+
// | |------->| | |
@@ -174,7 +174,7 @@ class G1CMMarkStack {
size_t bucket_size(size_t bucket) {
return (bucket == 0) ?
_min_capacity :
- _min_capacity * ( 1ULL << (bucket -1));
+ _min_capacity * ( 1ULL << (bucket - 1));
}
static unsigned int find_highest_bit(uintptr_t mask) {
@@ -225,7 +225,9 @@ class G1CMMarkStack {
size_t capacity() const { return _capacity; }
- bool expand();
+ // Expand the mark stack doubling its size.
+ bool try_expand();
+ bool try_expand_to(size_t desired_capacity);
TaskQueueEntryChunk* allocate_new_chunk();
};
@@ -435,12 +437,10 @@ class G1ConcurrentMark : public CHeapObj {
G1OldTracer* _gc_tracer_cm;
// Timing statistics. All of them are in ms
- NumberSeq _init_times;
NumberSeq _remark_times;
NumberSeq _remark_mark_times;
NumberSeq _remark_weak_ref_times;
NumberSeq _cleanup_times;
- double _total_cleanup_time;
double* _accum_task_vtime; // Accumulated task vtime
@@ -561,7 +561,7 @@ class G1ConcurrentMark : public CHeapObj {
// Sets the internal top_at_region_start for the given region to current top of the region.
inline void update_top_at_rebuild_start(HeapRegion* r);
// TARS for the given region during remembered set rebuilding.
- inline HeapWord* top_at_rebuild_start(uint region) const;
+ inline HeapWord* top_at_rebuild_start(HeapRegion* r) const;
// Clear statistics gathered during the concurrent cycle for the given region after
// it has been reclaimed.
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
index 3601f4529a336..4a05b86ae5a7c 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
@@ -30,12 +30,12 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
#include "gc/g1/g1RemSetTrackingPolicy.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "utilities/bitMap.inline.hpp"
@@ -184,9 +184,8 @@ inline size_t G1CMTask::scan_objArray(objArrayOop obj, MemRegion mr) {
return mr.word_size();
}
-inline HeapWord* G1ConcurrentMark::top_at_rebuild_start(uint region) const {
- assert(region < _g1h->max_reserved_regions(), "Tried to access TARS for region %u out of bounds", region);
- return _top_at_rebuild_starts[region];
+inline HeapWord* G1ConcurrentMark::top_at_rebuild_start(HeapRegion* r) const {
+ return _top_at_rebuild_starts[r->hrm_index()];
}
inline void G1ConcurrentMark::update_top_at_rebuild_start(HeapRegion* r) {
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkBitMap.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkBitMap.cpp
index e2ace6c8c0a97..438b58c2c02c7 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkBitMap.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkBitMap.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/virtualspace.hpp"
G1CMBitMap::G1CMBitMap() : MarkBitMap(), _listener() {
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp
index 80acada04726f..84823f8bf830d 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp
@@ -26,7 +26,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gc_globals.hpp"
#include "memory/memRegion.hpp"
#include "utilities/globalDefinitions.hpp"
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
index 8cf7601d65a25..74e242a0f3fa6 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
@@ -28,9 +28,9 @@
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
#include "gc/g1/g1_globals.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/workerThread.hpp"
#include "logging/log.hpp"
@@ -105,7 +105,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
// - been allocated after rebuild start, or
// - been reclaimed by a collection.
bool should_rebuild_or_scrub(HeapRegion* hr) const {
- return _cm->top_at_rebuild_start(hr->hrm_index()) != nullptr;
+ return _cm->top_at_rebuild_start(hr) != nullptr;
}
// Helper used by both humongous objects and when chunking an object larger than the
@@ -229,7 +229,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
assert(should_rebuild_or_scrub(hr), "must be");
log_trace(gc, marking)("Scrub and rebuild region: " HR_FORMAT " pb: " PTR_FORMAT " TARS: " PTR_FORMAT " TAMS: " PTR_FORMAT,
- HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr->hrm_index())), p2i(hr->top_at_mark_start()));
+ HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr)), p2i(hr->top_at_mark_start()));
if (scan_and_scrub_to_pb(hr, hr->bottom(), pb)) {
log_trace(gc, marking)("Scan and scrub aborted for region: %u", hr->hrm_index());
@@ -246,7 +246,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
hr->note_end_of_scrubbing();
// Rebuild from TAMS (= parsable_bottom) to TARS.
- if (scan_from_pb_to_tars(hr, pb, _cm->top_at_rebuild_start(hr->hrm_index()))) {
+ if (scan_from_pb_to_tars(hr, pb, _cm->top_at_rebuild_start(hr))) {
log_trace(gc, marking)("Rebuild aborted for region: %u (%s)", hr->hrm_index(), hr->get_short_type_str());
return true;
}
@@ -272,7 +272,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
"Humongous object not live");
log_trace(gc, marking)("Rebuild for humongous region: " HR_FORMAT " pb: " PTR_FORMAT " TARS: " PTR_FORMAT,
- HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr->hrm_index())));
+ HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr)));
// Scan the humongous object in chunks from bottom to top to rebuild remembered sets.
HeapWord* humongous_end = hr->humongous_start_region()->bottom() + humongous->size();
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
index b2ac4b25e9065..9ae9ad793c424 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
@@ -28,9 +28,9 @@
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineThread.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1Policy.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/gc_globals.hpp"
#include "logging/log.hpp"
#include "memory/allocation.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp
index 8767f8e1da99b..33943eb5113ab 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1Analytics.hpp"
#include "gc/g1/g1ConcurrentRefineThreadsNeeded.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1Policy.hpp"
#include "utilities/globalDefinitions.hpp"
#include
diff --git a/src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp b/src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp
index 93ff09134bfa6..67059b6ec1c47 100644
--- a/src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp
+++ b/src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp
@@ -30,10 +30,10 @@
#include "gc/g1/g1ConcurrentRefineThread.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
#include "gc/g1/g1FreeIdSet.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1RedirtyCardsQueue.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/bufferNodeList.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
diff --git a/src/hotspot/share/gc/g1/g1EdenRegions.hpp b/src/hotspot/share/gc/g1/g1EdenRegions.hpp
index 23b97785ad358..c76099c284a19 100644
--- a/src/hotspot/share/gc/g1/g1EdenRegions.hpp
+++ b/src/hotspot/share/gc/g1/g1EdenRegions.hpp
@@ -26,7 +26,7 @@
#define SHARE_GC_G1_G1EDENREGIONS_HPP
#include "gc/g1/g1RegionsOnNodes.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "runtime/globals.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp b/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
index a308fe7a3934a..845868c3e2415 100644
--- a/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
+++ b/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
@@ -27,7 +27,7 @@
#include "gc/g1/g1BatchedTask.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
#include "utilities/bitMap.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp
index 6ae43c770cc4e..81a54a9c31f32 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -120,8 +120,8 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap,
_oop_queue_set(_num_workers),
_array_queue_set(_num_workers),
_preserved_marks_set(true),
- _serial_compaction_point(this),
- _humongous_compaction_point(this),
+ _serial_compaction_point(this, nullptr),
+ _humongous_compaction_point(this, nullptr),
_is_alive(this, heap->concurrent_mark()->mark_bitmap()),
_is_alive_mutator(heap->ref_processor_stw(), &_is_alive),
_humongous_compaction_regions(8),
@@ -142,11 +142,13 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap,
}
for (uint i = 0; i < _num_workers; i++) {
- _markers[i] = new G1FullGCMarker(this, i, _preserved_marks_set.get(i), _live_stats);
- _compaction_points[i] = new G1FullGCCompactionPoint(this);
+ _markers[i] = new G1FullGCMarker(this, i, _live_stats);
+ _compaction_points[i] = new G1FullGCCompactionPoint(this, _preserved_marks_set.get(i));
_oop_queue_set.register_queue(i, marker(i)->oop_stack());
_array_queue_set.register_queue(i, marker(i)->objarray_stack());
}
+ _serial_compaction_point.set_preserved_stack(_preserved_marks_set.get(0));
+ _humongous_compaction_point.set_preserved_stack(_preserved_marks_set.get(0));
_region_attr_table.initialize(heap->reserved(), HeapRegion::GrainBytes);
}
@@ -189,6 +191,7 @@ void G1FullCollector::prepare_collection() {
_heap->gc_prologue(true);
_heap->retire_tlabs();
+ _heap->flush_region_pin_cache();
_heap->prepare_heap_for_full_collection();
PrepareRegionsClosure cl(this);
@@ -309,6 +312,13 @@ void G1FullCollector::phase1_mark_live_objects() {
reference_processor()->set_active_mt_degree(old_active_mt_degree);
}
+ {
+ GCTraceTime(Debug, gc, phases) debug("Phase 1: Flush Mark Stats Cache", scope()->timer());
+ for (uint i = 0; i < workers(); i++) {
+ marker(i)->flush_mark_stats_cache();
+ }
+ }
+
// Weak oops cleanup.
{
GCTraceTime(Debug, gc, phases) debug("Phase 1: Weak Processing", scope()->timer());
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp b/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
index 9150d2f2ccdfe..7aa04449bcf7b 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
@@ -28,7 +28,7 @@
#include "gc/g1/g1FullCollector.hpp"
#include "gc/g1/g1FullGCHeapRegionAttr.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "oops/oopsHierarchy.hpp"
#include "runtime/atomic.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
index a45c3eb17dea5..3eca766b2fcf7 100644
--- a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
@@ -32,7 +32,7 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCMarker.hpp"
#include "gc/g1/g1FullGCOopClosures.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/weakProcessor.inline.hpp"
@@ -91,8 +91,8 @@ void G1FullGCAdjustTask::work(uint worker_id) {
ResourceMark rm;
// Adjust preserved marks first since they are not balanced.
- G1FullGCMarker* marker = collector()->marker(worker_id);
- marker->preserved_stack()->adjust_during_full_gc();
+ G1FullGCCompactionPoint* cp = collector()->compaction_point(worker_id);
+ cp->preserved_stack()->adjust_during_full_gc();
{
// Adjust the weak roots.
diff --git a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp
index c9b190acd0659..26ac183d10d75 100644
--- a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp
@@ -27,8 +27,8 @@
#include "gc/g1/g1FullGCOopClosures.hpp"
#include "gc/g1/g1FullGCTask.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1RootProcessor.hpp"
-#include "gc/g1/heapRegionManager.hpp"
#include "gc/shared/weakProcessor.hpp"
#include "utilities/ticks.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
index 2ed6ccd4735fb..5e3a9687f9808 100644
--- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
@@ -28,7 +28,7 @@
#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCCompactTask.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "logging/log.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp
index a7e2ea38c1775..7f7f144397baf 100644
--- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp
@@ -28,7 +28,7 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCScope.hpp"
#include "gc/g1/g1FullGCTask.hpp"
-#include "gc/g1/heapRegionManager.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/shared/referenceProcessor.hpp"
class G1CollectedHeap;
diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp
index 7669771eb6c3c..7b19aa431beb1 100644
--- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp
@@ -25,15 +25,16 @@
#include "precompiled.hpp"
#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCCompactionPoint.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "oops/oop.inline.hpp"
#include "utilities/debug.hpp"
-G1FullGCCompactionPoint::G1FullGCCompactionPoint(G1FullCollector* collector) :
+G1FullGCCompactionPoint::G1FullGCCompactionPoint(G1FullCollector* collector, PreservedMarks* preserved_stack) :
_collector(collector),
_current_region(nullptr),
- _compaction_top(nullptr) {
+ _compaction_top(nullptr),
+ _preserved_stack(preserved_stack) {
_compaction_regions = new (mtGC) GrowableArray(32, mtGC);
_compaction_region_iterator = _compaction_regions->begin();
}
@@ -102,6 +103,9 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
// Store a forwarding pointer if the object should be moved.
if (cast_from_oop(object) != _compaction_top) {
+ if (!object->is_forwarded()) {
+ preserved_stack()->push_if_necessary(object, object->mark());
+ }
object->forward_to(cast_to_oop(_compaction_top));
assert(object->is_forwarded(), "must be forwarded");
} else {
@@ -165,7 +169,7 @@ void G1FullGCCompactionPoint::forward_humongous(HeapRegion* hr) {
}
// Preserve the mark for the humongous object as the region was initially not compacting.
- _collector->marker(0)->preserved_stack()->push_if_necessary(obj, obj->mark());
+ preserved_stack()->push_if_necessary(obj, obj->mark());
HeapRegion* dest_hr = _compaction_regions->at(range_begin);
obj->forward_to(cast_to_oop(dest_hr->bottom()));
diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp
index ca76f7e6b9408..6d87e44b213ab 100644
--- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp
@@ -32,11 +32,13 @@
class G1FullCollector;
class HeapRegion;
+class PreservedMarks;
class G1FullGCCompactionPoint : public CHeapObj {
G1FullCollector* _collector;
HeapRegion* _current_region;
HeapWord* _compaction_top;
+ PreservedMarks* _preserved_stack;
GrowableArray* _compaction_regions;
GrowableArrayIterator _compaction_region_iterator;
@@ -47,7 +49,7 @@ class G1FullGCCompactionPoint : public CHeapObj {
uint find_contiguous_before(HeapRegion* hr, uint num_regions);
public:
- G1FullGCCompactionPoint(G1FullCollector* collector);
+ G1FullGCCompactionPoint(G1FullCollector* collector, PreservedMarks* preserved_stack);
~G1FullGCCompactionPoint();
bool has_regions();
@@ -63,6 +65,16 @@ class G1FullGCCompactionPoint : public CHeapObj {
HeapRegion* current_region();
GrowableArray* regions();
+
+ PreservedMarks* preserved_stack() const {
+ assert(_preserved_stack != nullptr, "must be initialized");
+ return _preserved_stack;
+ }
+
+ void set_preserved_stack(PreservedMarks* preserved_stack) {
+ assert(_preserved_stack == nullptr, "only initialize once");
+ _preserved_stack = preserved_stack;
+ }
};
#endif // SHARE_GC_G1_G1FULLGCCOMPACTIONPOINT_HPP
diff --git a/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp b/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp
index 209e89ae4abb1..e27439143d900 100644
--- a/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp
@@ -56,7 +56,6 @@ void G1FullGCMarkTask::work(uint worker_id) {
// Mark stack is populated, now process and drain it.
marker->complete_marking(collector()->oop_queue_set(), collector()->array_queue_set(), &_terminator);
- marker->flush_mark_stats_cache();
// This is the point where the entire marking should have completed.
assert(marker->oop_stack()->is_empty(), "Marking should have completed");
diff --git a/src/hotspot/share/gc/g1/g1FullGCMarkTask.hpp b/src/hotspot/share/gc/g1/g1FullGCMarkTask.hpp
index 5e45b65c65faf..cf87bcfa7c882 100644
--- a/src/hotspot/share/gc/g1/g1FullGCMarkTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCMarkTask.hpp
@@ -28,8 +28,8 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCScope.hpp"
#include "gc/g1/g1FullGCTask.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1RootProcessor.hpp"
-#include "gc/g1/heapRegionManager.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "utilities/ticks.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.cpp b/src/hotspot/share/gc/g1/g1FullGCMarker.cpp
index ac71b82d04a67..b98bb41240ced 100644
--- a/src/hotspot/share/gc/g1/g1FullGCMarker.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCMarker.cpp
@@ -33,14 +33,12 @@
G1FullGCMarker::G1FullGCMarker(G1FullCollector* collector,
uint worker_id,
- PreservedMarks* preserved_stack,
G1RegionMarkStats* mark_stats) :
_collector(collector),
_worker_id(worker_id),
_bitmap(collector->mark_bitmap()),
_oop_stack(),
_objarray_stack(),
- _preserved_stack(preserved_stack),
_mark_closure(worker_id, this, ClassLoaderData::_claim_stw_fullgc_mark, G1CollectedHeap::heap()->ref_processor_stw()),
_stack_closure(this),
_cld_closure(mark_closure(), ClassLoaderData::_claim_stw_fullgc_mark),
diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.hpp b/src/hotspot/share/gc/g1/g1FullGCMarker.hpp
index cfe9d125e20ff..3dcc746a26ed6 100644
--- a/src/hotspot/share/gc/g1/g1FullGCMarker.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCMarker.hpp
@@ -28,7 +28,6 @@
#include "gc/g1/g1FullGCOopClosures.hpp"
#include "gc/g1/g1OopClosures.hpp"
#include "gc/g1/g1RegionMarkStatsCache.hpp"
-#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/stringdedup/stringDedup.hpp"
#include "gc/shared/taskqueue.hpp"
#include "memory/iterator.hpp"
@@ -59,7 +58,6 @@ class G1FullGCMarker : public CHeapObj {
// Mark stack
OopQueue _oop_stack;
ObjArrayTaskQueue _objarray_stack;
- PreservedMarks* _preserved_stack;
// Marking closures
G1MarkAndPushClosure _mark_closure;
@@ -89,14 +87,12 @@ class G1FullGCMarker : public CHeapObj {
public:
G1FullGCMarker(G1FullCollector* collector,
uint worker_id,
- PreservedMarks* preserved_stack,
G1RegionMarkStats* mark_stats);
~G1FullGCMarker();
// Stack getters
OopQueue* oop_stack() { return &_oop_stack; }
ObjArrayTaskQueue* objarray_stack() { return &_objarray_stack; }
- PreservedMarks* preserved_stack() { return _preserved_stack; }
// Marking entry points
template inline void mark_and_push(T* p);
diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp
index 8080a1d113e75..646cf9e3df5a8 100644
--- a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp
@@ -51,13 +51,6 @@ inline bool G1FullGCMarker::mark_object(oop obj) {
return false;
}
- // Marked by us, preserve if needed.
- if (_collector->is_compacting(obj)) {
- // It is not necessary to preserve marks for objects in regions we do not
- // compact because we do not change their headers (i.e. forward them).
- preserved_stack()->push_if_necessary(obj, obj->mark());
- }
-
// Check if deduplicatable string.
if (StringDedup::is_enabled() &&
java_lang_String::is_instance(obj) &&
diff --git a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp
index aa194d16203b7..f10f884b24253 100644
--- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp
@@ -28,10 +28,10 @@
#include "gc/g1/g1FullGCOopClosures.hpp"
#include "gc/g1/g1Allocator.inline.hpp"
-#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
+#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCMarker.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "memory/universe.hpp"
#include "oops/access.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp
index 7cef60293974c..e116367133d91 100644
--- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp
@@ -30,7 +30,7 @@
#include "gc/g1/g1FullGCMarker.hpp"
#include "gc/g1/g1FullGCOopClosures.inline.hpp"
#include "gc/g1/g1FullGCPrepareTask.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "logging/log.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp
index 7f09f0553e9f7..9618986dcdd0c 100644
--- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp
@@ -26,7 +26,7 @@
#define SHARE_GC_G1_G1FULLGCPREPARETASK_HPP
#include "gc/g1/g1FullGCTask.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
class G1CollectedHeap;
diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp
index 2a2add76f48c6..5cff8e5f412a1 100644
--- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp
@@ -31,7 +31,7 @@
#include "gc/g1/g1FullCollector.hpp"
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCScope.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
void G1DetermineCompactionQueueClosure::free_empty_humongous_region(HeapRegion* hr) {
_g1h->free_humongous_region(hr, nullptr);
diff --git a/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.hpp b/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.hpp
index 25745ff0abcc4..f424906d56377 100644
--- a/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1FULLGCRESETMETADATATASK_HPP
#define SHARE_GC_G1_G1FULLGCRESETMETADATATASK_HPP
#include "gc/g1/g1FullGCTask.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
class G1FullGCResetMetadataTask : public G1FullGCTask {
G1FullCollector* _collector;
diff --git a/src/hotspot/share/gc/g1/g1HRPrinter.cpp b/src/hotspot/share/gc/g1/g1HRPrinter.cpp
index af83284f0618d..7ec4f2a0d96e3 100644
--- a/src/hotspot/share/gc/g1/g1HRPrinter.cpp
+++ b/src/hotspot/share/gc/g1/g1HRPrinter.cpp
@@ -24,8 +24,8 @@
#include "precompiled.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1HRPrinter.hpp"
-#include "gc/g1/heapRegionSet.hpp"
void G1HRPrinter::cleanup(FreeRegionList* cleanup_list) {
if (is_active()) {
diff --git a/src/hotspot/share/gc/g1/g1HRPrinter.hpp b/src/hotspot/share/gc/g1/g1HRPrinter.hpp
index 876be95f0746c..9d6d151084fbd 100644
--- a/src/hotspot/share/gc/g1/g1HRPrinter.hpp
+++ b/src/hotspot/share/gc/g1/g1HRPrinter.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1HRPRINTER_HPP
#define SHARE_GC_G1_G1HRPRINTER_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "logging/log.hpp"
#define SKIP_RETIRED_FULL_REGIONS 1
diff --git a/src/hotspot/share/gc/g1/heapRegion.cpp b/src/hotspot/share/gc/g1/g1HeapRegion.cpp
similarity index 98%
rename from src/hotspot/share/gc/g1/heapRegion.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegion.cpp
index 29fe5031a354a..cb230ee598890 100644
--- a/src/hotspot/share/gc/g1/heapRegion.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,14 +29,14 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionBounds.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionTracer.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionBounds.inline.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionTracer.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
#include "memory/iterator.inline.hpp"
@@ -424,7 +424,7 @@ void HeapRegion::print_on(outputStream* st) const {
st->print("|-");
}
}
- st->print("|%3u", Atomic::load(&_pinned_object_count));
+ st->print("|%3zu", Atomic::load(&_pinned_object_count));
st->print_cr("");
}
diff --git a/src/hotspot/share/gc/g1/heapRegion.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.hpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegion.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegion.hpp
index 2706b814000a5..b9187ddfddc2f 100644
--- a/src/hotspot/share/gc/g1/heapRegion.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,14 +22,14 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGION_HPP
-#define SHARE_GC_G1_HEAPREGION_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGION_HPP
+#define SHARE_GC_G1_G1HEAPREGION_HPP
#include "gc/g1/g1BlockOffsetTable.hpp"
+#include "gc/g1/g1HeapRegionTracer.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
+#include "gc/g1/g1HeapRegionType.hpp"
#include "gc/g1/g1SurvRateGroup.hpp"
-#include "gc/g1/heapRegionTracer.hpp"
-#include "gc/g1/heapRegionType.hpp"
#include "gc/shared/ageTable.hpp"
#include "gc/shared/spaceDecorator.hpp"
#include "gc/shared/verifyOption.hpp"
@@ -257,7 +257,7 @@ class HeapRegion : public CHeapObj {
uint _node_index;
// Number of objects in this region that are currently pinned.
- volatile uint _pinned_object_count;
+ volatile size_t _pinned_object_count;
void report_region_type_change(G1HeapRegionTraceType::Type to);
@@ -302,8 +302,9 @@ class HeapRegion : public CHeapObj {
static uint LogOfHRGrainBytes;
static uint LogCardsPerRegion;
- inline void increment_pinned_object_count();
- inline void decrement_pinned_object_count();
+ // Atomically adjust the pinned object count by the given value. Value must not
+ // be zero.
+ inline void add_pinned_object_count(size_t value);
static size_t GrainBytes;
static size_t GrainWords;
@@ -408,7 +409,7 @@ class HeapRegion : public CHeapObj {
bool is_old_or_humongous() const { return _type.is_old_or_humongous(); }
- uint pinned_count() const { return Atomic::load(&_pinned_object_count); }
+ size_t pinned_count() const { return Atomic::load(&_pinned_object_count); }
bool has_pinned_objects() const { return pinned_count() > 0; }
void set_free();
@@ -621,4 +622,4 @@ class HeapRegionIndexClosure : public StackObj {
bool is_complete() { return _is_complete; }
};
-#endif // SHARE_GC_G1_HEAPREGION_HPP
+#endif // SHARE_GC_G1_G1HEAPREGION_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegion.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegion.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
index ec68407b546bd..146f5aff86680 100644
--- a/src/hotspot/share/gc/g1/heapRegion.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,10 +22,10 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGION_INLINE_HPP
-#define SHARE_GC_G1_HEAPREGION_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGION_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGION_INLINE_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "classfile/vmClasses.hpp"
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
@@ -553,12 +553,10 @@ inline void HeapRegion::record_surv_words_in_group(size_t words_survived) {
_surv_rate_group->record_surviving_words(age, words_survived);
}
-inline void HeapRegion::increment_pinned_object_count() {
- Atomic::add(&_pinned_object_count, 1u, memory_order_relaxed);
+inline void HeapRegion::add_pinned_object_count(size_t value) {
+ assert(value != 0, "wasted effort");
+ assert(!is_free(), "trying to pin free region %u, adding %zu", hrm_index(), value);
+ Atomic::add(&_pinned_object_count, value, memory_order_relaxed);
}
-inline void HeapRegion::decrement_pinned_object_count() {
- Atomic::sub(&_pinned_object_count, 1u, memory_order_relaxed);
-}
-
-#endif // SHARE_GC_G1_HEAPREGION_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGION_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionAttr.hpp b/src/hotspot/share/gc/g1/g1HeapRegionAttr.hpp
index e5a10858ed126..c3c7b94e287b7 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionAttr.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionAttr.hpp
@@ -26,7 +26,7 @@
#define SHARE_GC_G1_G1HEAPREGIONATTR_HPP
#include "gc/g1/g1BiasedArray.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
// Per-region attributes often used during garbage collection to avoid costly
// lookups for that information all over the place.
diff --git a/src/hotspot/share/gc/g1/heapRegionBounds.hpp b/src/hotspot/share/gc/g1/g1HeapRegionBounds.hpp
similarity index 94%
rename from src/hotspot/share/gc/g1/heapRegionBounds.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionBounds.hpp
index 5a6db9fac3b14..3cde2f77d4a5b 100644
--- a/src/hotspot/share/gc/g1/heapRegionBounds.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionBounds.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONBOUNDS_HPP
-#define SHARE_GC_G1_HEAPREGIONBOUNDS_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONBOUNDS_HPP
+#define SHARE_GC_G1_G1HEAPREGIONBOUNDS_HPP
#include "memory/allStatic.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -55,4 +55,4 @@ class HeapRegionBounds : public AllStatic {
static inline size_t target_number();
};
-#endif // SHARE_GC_G1_HEAPREGIONBOUNDS_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONBOUNDS_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionBounds.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionBounds.inline.hpp
similarity index 87%
rename from src/hotspot/share/gc/g1/heapRegionBounds.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionBounds.inline.hpp
index 73a44bcf1794f..473057144d30c 100644
--- a/src/hotspot/share/gc/g1/heapRegionBounds.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionBounds.inline.hpp
@@ -22,10 +22,10 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP
-#define SHARE_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONBOUNDS_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONBOUNDS_INLINE_HPP
-#include "gc/g1/heapRegionBounds.hpp"
+#include "gc/g1/g1HeapRegionBounds.hpp"
size_t HeapRegionBounds::min_size() {
return MIN_REGION_SIZE;
@@ -43,4 +43,4 @@ size_t HeapRegionBounds::target_number() {
return TARGET_REGION_NUMBER;
}
-#endif // SHARE_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONBOUNDS_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionEventSender.cpp b/src/hotspot/share/gc/g1/g1HeapRegionEventSender.cpp
index 19433e39c3235..5dce9b2aa909e 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionEventSender.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionEventSender.cpp
@@ -24,8 +24,8 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegionEventSender.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/gc_globals.hpp"
#include "jfr/jfrEvents.hpp"
#include "runtime/vmThread.hpp"
diff --git a/src/hotspot/share/gc/g1/heapRegionManager.cpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
similarity index 99%
rename from src/hotspot/share/gc/g1/heapRegionManager.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
index 455aacc0931a1..0fb8e7499d68f 100644
--- a/src/hotspot/share/gc/g1/heapRegionManager.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
@@ -27,10 +27,10 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1CommittedRegionMap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1NUMAStats.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
#include "jfr/jfrEvents.hpp"
#include "logging/logStream.hpp"
#include "memory/allocation.hpp"
diff --git a/src/hotspot/share/gc/g1/heapRegionManager.hpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
similarity index 98%
rename from src/hotspot/share/gc/g1/heapRegionManager.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
index 0a84010e73c55..8ecb5d6d66f64 100644
--- a/src/hotspot/share/gc/g1/heapRegionManager.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
@@ -22,13 +22,13 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONMANAGER_HPP
-#define SHARE_GC_G1_HEAPREGIONMANAGER_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONMANAGER_HPP
+#define SHARE_GC_G1_G1HEAPREGIONMANAGER_HPP
#include "gc/g1/g1BiasedArray.hpp"
#include "gc/g1/g1CommittedRegionMap.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1RegionToSpaceMapper.hpp"
-#include "gc/g1/heapRegionSet.hpp"
#include "memory/allocation.hpp"
#include "services/memoryUsage.hpp"
@@ -326,4 +326,4 @@ class HeapRegionClaimer : public StackObj {
// Claim the given region, returns true if successfully claimed.
bool claim_region(uint region_index);
};
-#endif // SHARE_GC_G1_HEAPREGIONMANAGER_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONMANAGER_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionManager.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.inline.hpp
similarity index 91%
rename from src/hotspot/share/gc/g1/heapRegionManager.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionManager.inline.hpp
index 1bcba292ffa2b..9768ee358baae 100644
--- a/src/hotspot/share/gc/g1/heapRegionManager.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.inline.hpp
@@ -22,14 +22,14 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONMANAGER_INLINE_HPP
-#define SHARE_GC_G1_HEAPREGIONMANAGER_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONMANAGER_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONMANAGER_INLINE_HPP
-#include "gc/g1/heapRegionManager.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1CommittedRegionMap.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
inline bool HeapRegionManager::is_available(uint region) const {
return _committed_map.active(region);
@@ -83,4 +83,4 @@ inline HeapRegion* HeapRegionManager::allocate_free_regions_starting_at(uint fir
return start;
}
-#endif // SHARE_GC_G1_HEAPREGIONMANAGER_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONMANAGER_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionRemSet.cpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionRemSet.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
index 19b383a08a39c..88a3e09bd8e9e 100644
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
@@ -29,8 +29,8 @@
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "memory/allocation.hpp"
#include "memory/padded.inline.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/heapRegionRemSet.hpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionRemSet.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
index 2e5ae7b1c3076..7e76965835b6b 100644
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONREMSET_HPP
-#define SHARE_GC_G1_HEAPREGIONREMSET_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONREMSET_HPP
+#define SHARE_GC_G1_G1HEAPREGIONREMSET_HPP
#include "gc/g1/g1CardSet.hpp"
#include "gc/g1/g1CardSetMemory.hpp"
@@ -185,4 +185,4 @@ class HeapRegionRemSet : public CHeapObj {
#endif
};
-#endif // SHARE_GC_G1_HEAPREGIONREMSET_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONREMSET_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
similarity index 95%
rename from src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
index ca8f675e6dfd7..78153cc0c3023 100644
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
@@ -22,14 +22,14 @@
*
*/
-#ifndef SHARE_VM_GC_G1_HEAPREGIONREMSET_INLINE_HPP
-#define SHARE_VM_GC_G1_HEAPREGIONREMSET_INLINE_HPP
+#ifndef SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
+#define SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "runtime/atomic.hpp"
#include "utilities/bitMap.inline.hpp"
@@ -144,4 +144,4 @@ void HeapRegionRemSet::print_info(outputStream* st, OopOrNarrowOopStar from) {
_card_set.print_info(st, to_card(from));
}
-#endif // SHARE_VM_GC_G1_HEAPREGIONREMSET_INLINE_HPP
+#endif // SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionSet.cpp b/src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
similarity index 99%
rename from src/hotspot/share/gc/g1/heapRegionSet.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
index 7243677ede8e2..832770ddb4c42 100644
--- a/src/hotspot/share/gc/g1/heapRegionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
@@ -24,9 +24,9 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1NUMA.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
uint FreeRegionList::_unrealistically_long_length = 0;
diff --git a/src/hotspot/share/gc/g1/heapRegionSet.hpp b/src/hotspot/share/gc/g1/g1HeapRegionSet.hpp
similarity index 98%
rename from src/hotspot/share/gc/g1/heapRegionSet.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionSet.hpp
index 5db2fbb696aa7..bfc60992c3d40 100644
--- a/src/hotspot/share/gc/g1/heapRegionSet.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionSet.hpp
@@ -22,10 +22,10 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONSET_HPP
-#define SHARE_GC_G1_HEAPREGIONSET_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONSET_HPP
+#define SHARE_GC_G1_G1HEAPREGIONSET_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "utilities/macros.hpp"
#define assert_heap_region_set(p, message) \
@@ -271,4 +271,4 @@ class FreeRegionListIterator : public StackObj {
}
};
-#endif // SHARE_GC_G1_HEAPREGIONSET_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONSET_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionSet.inline.hpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionSet.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionSet.inline.hpp
index 0d7c7ed315d13..fbd19466d9f9a 100644
--- a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionSet.inline.hpp
@@ -22,10 +22,10 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
-#define SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONSET_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONSET_INLINE_HPP
-#include "gc/g1/heapRegionSet.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1NUMA.hpp"
@@ -271,4 +271,4 @@ inline uint FreeRegionList::length(uint node_index) const {
}
}
-#endif // SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionTracer.cpp b/src/hotspot/share/gc/g1/g1HeapRegionTracer.cpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionTracer.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionTracer.cpp
index e039ae219a11e..c66ae0cb06941 100644
--- a/src/hotspot/share/gc/g1/heapRegionTracer.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionTracer.cpp
@@ -23,7 +23,7 @@
*/
#include "precompiled.hpp"
-#include "gc/g1/heapRegionTracer.hpp"
+#include "gc/g1/g1HeapRegionTracer.hpp"
#include "jfr/jfrEvents.hpp"
void HeapRegionTracer::send_region_type_change(uint index,
diff --git a/src/hotspot/share/gc/g1/heapRegionTracer.hpp b/src/hotspot/share/gc/g1/g1HeapRegionTracer.hpp
similarity index 92%
rename from src/hotspot/share/gc/g1/heapRegionTracer.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionTracer.hpp
index 647744d0d5e23..137bc6189aed0 100644
--- a/src/hotspot/share/gc/g1/heapRegionTracer.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionTracer.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONTRACER_HPP
-#define SHARE_GC_G1_HEAPREGIONTRACER_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONTRACER_HPP
+#define SHARE_GC_G1_G1HEAPREGIONTRACER_HPP
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "memory/allStatic.hpp"
@@ -38,4 +38,4 @@ class HeapRegionTracer : AllStatic {
size_t used);
};
-#endif // SHARE_GC_G1_HEAPREGIONTRACER_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONTRACER_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionType.cpp b/src/hotspot/share/gc/g1/g1HeapRegionType.cpp
similarity index 98%
rename from src/hotspot/share/gc/g1/heapRegionType.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionType.cpp
index bb5c4e9392c79..e55759b3bf38d 100644
--- a/src/hotspot/share/gc/g1/heapRegionType.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionType.cpp
@@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
-#include "gc/g1/heapRegionType.hpp"
+#include "gc/g1/g1HeapRegionType.hpp"
const HeapRegionType HeapRegionType::Eden = HeapRegionType(EdenTag);
const HeapRegionType HeapRegionType::Survivor = HeapRegionType(SurvTag);
diff --git a/src/hotspot/share/gc/g1/heapRegionType.hpp b/src/hotspot/share/gc/g1/g1HeapRegionType.hpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionType.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionType.hpp
index 85a89f77a9405..b62699cc3fb9e 100644
--- a/src/hotspot/share/gc/g1/heapRegionType.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionType.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONTYPE_HPP
-#define SHARE_GC_G1_HEAPREGIONTYPE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONTYPE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONTYPE_HPP
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -167,4 +167,4 @@ friend class VMStructs;
static const HeapRegionType Humongous;
};
-#endif // SHARE_GC_G1_HEAPREGIONTYPE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONTYPE_HPP
diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
index 6cca6e8b05610..b7484727e1fb1 100644
--- a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
@@ -27,12 +27,12 @@
#include "gc/g1/g1Allocator.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkThread.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1RootProcessor.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.hpp b/src/hotspot/share/gc/g1/g1HeapVerifier.hpp
index f10787c528e36..221ae9576489e 100644
--- a/src/hotspot/share/gc/g1/g1HeapVerifier.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapVerifier.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1HEAPVERIFIER_HPP
#define SHARE_GC_G1_G1HEAPVERIFIER_HPP
-#include "gc/g1/heapRegionSet.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/shared/verifyOption.hpp"
#include "memory/allocation.hpp"
#include "utilities/macros.hpp"
diff --git a/src/hotspot/share/gc/g1/g1MemoryPool.cpp b/src/hotspot/share/gc/g1/g1MemoryPool.cpp
index 428d2db212e2e..f924e5a7e6517 100644
--- a/src/hotspot/share/gc/g1/g1MemoryPool.cpp
+++ b/src/hotspot/share/gc/g1/g1MemoryPool.cpp
@@ -24,8 +24,8 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1MemoryPool.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/gc_globals.hpp"
G1MemoryPoolSuper::G1MemoryPoolSuper(G1CollectedHeap* g1h,
diff --git a/src/hotspot/share/gc/g1/g1MonitoringSupport.hpp b/src/hotspot/share/gc/g1/g1MonitoringSupport.hpp
index 769a6fe831c7f..a1b9d6c985bfb 100644
--- a/src/hotspot/share/gc/g1/g1MonitoringSupport.hpp
+++ b/src/hotspot/share/gc/g1/g1MonitoringSupport.hpp
@@ -180,8 +180,6 @@ class G1MonitoringSupport : public CHeapObj {
// Recalculate all the sizes.
void recalculate_sizes();
- void recalculate_eden_size();
-
public:
G1MonitoringSupport(G1CollectedHeap* g1h);
~G1MonitoringSupport();
diff --git a/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.cpp b/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.cpp
index 4ba4469bd964f..337acfe24058d 100644
--- a/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.cpp
+++ b/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.cpp
@@ -26,10 +26,10 @@
#include "ci/ciUtilities.hpp"
#include "gc/g1/g1CardSetMemory.inline.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1MonotonicArenaFreeMemoryTask.hpp"
#include "gc/g1/g1MonotonicArenaFreePool.hpp"
#include "gc/g1/g1_globals.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
diff --git a/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.hpp b/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.hpp
index 88611c63b5c10..d55c844037c40 100644
--- a/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.hpp
+++ b/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.hpp
@@ -26,9 +26,9 @@
#define SHARE_GC_G1_G1MONOTONICARENAFREEMEMORYTASK_HPP
#include "gc/g1/g1CardSetMemory.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1MonotonicArenaFreePool.hpp"
#include "gc/g1/g1ServiceThread.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/ticks.hpp"
diff --git a/src/hotspot/share/gc/g1/g1NUMA.hpp b/src/hotspot/share/gc/g1/g1NUMA.hpp
index acf5a114aebd4..cbefab411dff6 100644
--- a/src/hotspot/share/gc/g1/g1NUMA.hpp
+++ b/src/hotspot/share/gc/g1/g1NUMA.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_VM_GC_G1_NUMA_HPP
#define SHARE_VM_GC_G1_NUMA_HPP
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1NUMAStats.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "memory/allocation.hpp"
#include "runtime/os.hpp"
diff --git a/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp b/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
index 13890c026b2d6..eb86b81d2abbd 100644
--- a/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
+++ b/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_VM_GC_G1_G1OLDGENALLOCATIONTRACKER_HPP
#define SHARE_VM_GC_G1_G1OLDGENALLOCATIONTRACKER_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
class G1AdaptiveIHOPControl;
diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
index ea7f690d325fc..9eebfbd6a8682 100644
--- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
@@ -29,10 +29,10 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1ParScanThreadState.inline.hpp"
#include "gc/g1/g1RemSet.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
#include "memory/iterator.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index 92aec50def6f2..3480e0ae9945b 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -113,15 +113,15 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
initialize_numa_stats();
}
-size_t G1ParScanThreadState::flush_stats(size_t* surviving_young_words, uint num_workers) {
- _rdc_local_qset.flush();
+size_t G1ParScanThreadState::flush_stats(size_t* surviving_young_words, uint num_workers, BufferNodeList* rdc_buffers) {
+ *rdc_buffers = _rdc_local_qset.flush();
flush_numa_stats();
// Update allocation statistics.
_plab_allocator->flush_and_retire_stats(num_workers);
_g1h->policy()->record_age_table(&_age_table);
if (_evacuation_failed_info.has_failed()) {
- _g1h->gc_tracer_stw()->report_evacuation_failed(_evacuation_failed_info);
+ _g1h->gc_tracer_stw()->report_evacuation_failed(_evacuation_failed_info);
}
size_t sum = 0;
@@ -593,7 +593,6 @@ const size_t* G1ParScanThreadStateSet::surviving_young_words() const {
void G1ParScanThreadStateSet::flush_stats() {
assert(!_flushed, "thread local state from the per thread states should be flushed once");
-
for (uint worker_id = 0; worker_id < _num_workers; ++worker_id) {
G1ParScanThreadState* pss = _states[worker_id];
assert(pss != nullptr, "must be initialized");
@@ -604,7 +603,7 @@ void G1ParScanThreadStateSet::flush_stats() {
// because it resets the PLAB allocator where we get this info from.
size_t lab_waste_bytes = pss->lab_waste_words() * HeapWordSize;
size_t lab_undo_waste_bytes = pss->lab_undo_waste_words() * HeapWordSize;
- size_t copied_bytes = pss->flush_stats(_surviving_young_words_total, _num_workers) * HeapWordSize;
+ size_t copied_bytes = pss->flush_stats(_surviving_young_words_total, _num_workers, &_rdc_buffers[worker_id]) * HeapWordSize;
size_t evac_fail_enqueued_cards = pss->evac_failure_enqueued_cards();
p->record_or_add_thread_work_item(G1GCPhaseTimes::MergePSS, worker_id, copied_bytes, G1GCPhaseTimes::MergePSSCopiedBytes);
@@ -615,6 +614,11 @@ void G1ParScanThreadStateSet::flush_stats() {
delete pss;
_states[worker_id] = nullptr;
}
+
+ G1DirtyCardQueueSet& dcq = G1BarrierSet::dirty_card_queue_set();
+ dcq.merge_bufferlists(rdcqs());
+ rdcqs()->verify_empty();
+
_flushed = true;
}
@@ -706,6 +710,7 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
_rdcqs(G1BarrierSet::dirty_card_queue_set().allocator()),
_preserved_marks_set(true /* in_c_heap */),
_states(NEW_C_HEAP_ARRAY(G1ParScanThreadState*, num_workers, mtGC)),
+ _rdc_buffers(NEW_C_HEAP_ARRAY(BufferNodeList, num_workers, mtGC)),
_surviving_young_words_total(NEW_C_HEAP_ARRAY(size_t, collection_set->young_region_length() + 1, mtGC)),
_num_workers(num_workers),
_flushed(false),
@@ -713,6 +718,7 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
_preserved_marks_set.init(num_workers);
for (uint i = 0; i < num_workers; ++i) {
_states[i] = nullptr;
+ _rdc_buffers[i] = BufferNodeList();
}
memset(_surviving_young_words_total, 0, (collection_set->young_region_length() + 1) * sizeof(size_t));
}
@@ -721,5 +727,6 @@ G1ParScanThreadStateSet::~G1ParScanThreadStateSet() {
assert(_flushed, "thread local state from the per thread states should have been flushed");
FREE_C_HEAP_ARRAY(G1ParScanThreadState*, _states);
FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_total);
+ FREE_C_HEAP_ARRAY(BufferNodeList, _rdc_buffers);
_preserved_marks_set.reclaim();
}
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
index 4f72e61f82be7..72f7a0b84e7d9 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
@@ -166,7 +166,7 @@ class G1ParScanThreadState : public CHeapObj {
// Pass locally gathered statistics to global state. Returns the total number of
// HeapWords copied.
- size_t flush_stats(size_t* surviving_young_words, uint num_workers);
+ size_t flush_stats(size_t* surviving_young_words, uint num_workers, BufferNodeList* buffer_log);
private:
void do_partial_array(PartialArrayScanTask task);
@@ -247,6 +247,7 @@ class G1ParScanThreadStateSet : public StackObj {
G1RedirtyCardsQueueSet _rdcqs;
PreservedMarksSet _preserved_marks_set;
G1ParScanThreadState** _states;
+ BufferNodeList* _rdc_buffers;
size_t* _surviving_young_words_total;
uint _num_workers;
bool _flushed;
@@ -260,12 +261,14 @@ class G1ParScanThreadStateSet : public StackObj {
~G1ParScanThreadStateSet();
G1RedirtyCardsQueueSet* rdcqs() { return &_rdcqs; }
+ BufferNodeList* rdc_buffers() { return _rdc_buffers; }
PreservedMarksSet* preserved_marks_set() { return &_preserved_marks_set; }
void flush_stats();
void record_unused_optional_region(HeapRegion* hr);
G1ParScanThreadState* state_for_worker(uint worker_id);
+ uint num_workers() const { return _num_workers; }
const size_t* surviving_young_words() const;
};
diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp
index 9e22acc73ca33..10227dfe2add8 100644
--- a/src/hotspot/share/gc/g1/g1Policy.cpp
+++ b/src/hotspot/share/gc/g1/g1Policy.cpp
@@ -34,13 +34,13 @@
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineStats.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1IHOPControl.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1SurvivorRegions.hpp"
#include "gc/g1/g1YoungGenSizer.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/concurrentGCBreakpoints.hpp"
#include "gc/shared/gcPolicyCounters.hpp"
#include "logging/log.hpp"
diff --git a/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.cpp b/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.cpp
index a264fb436c690..35e80fb262e90 100644
--- a/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.cpp
+++ b/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.cpp
@@ -65,10 +65,12 @@ void G1RedirtyCardsLocalQueueSet::enqueue(void* value) {
}
}
-void G1RedirtyCardsLocalQueueSet::flush() {
+BufferNodeList G1RedirtyCardsLocalQueueSet::flush() {
flush_queue(_queue);
+ BufferNodeList cur_buffers = _buffers;
_shared_qset->add_bufferlist(_buffers);
_buffers = BufferNodeList();
+ return cur_buffers;
}
// G1RedirtyCardsLocalQueueSet::Queue
diff --git a/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp b/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp
index 970cce1623ec6..ca0bc35fd0436 100644
--- a/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp
+++ b/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp
@@ -56,7 +56,9 @@ class G1RedirtyCardsLocalQueueSet : private PtrQueueSet {
void enqueue(void* value);
// Transfer all completed buffers to the shared qset.
- void flush();
+ // Returns the flushed BufferNodeList which is later used
+ // as a shortcut into the shared qset.
+ BufferNodeList flush();
};
// Card table entries to be redirtied and the cards reprocessed later.
diff --git a/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp b/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp
index d3236f58fae25..f74453b81ce36 100644
--- a/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp
+++ b/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp
@@ -48,8 +48,6 @@ struct G1RegionMarkStats {
// are updated by the atomic mark. We do not remark objects after overflow.
void clear_during_overflow() {
}
-
- bool is_clear() const { return _live_words == 0; }
};
// Per-marking thread cache for the region mark statistics.
diff --git a/src/hotspot/share/gc/g1/g1RegionPinCache.hpp b/src/hotspot/share/gc/g1/g1RegionPinCache.hpp
new file mode 100644
index 0000000000000..1c04936930a8f
--- /dev/null
+++ b/src/hotspot/share/gc/g1/g1RegionPinCache.hpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1REGIONPINCACHE_HPP
+#define SHARE_GC_G1_G1REGIONPINCACHE_HPP
+
+#include "gc/g1/g1HeapRegion.hpp"
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// Holds (caches) the pending pinned object count adjustment for the region
+// _region_idx on a per thread basis.
+// Keeping such a cache avoids the expensive atomic operations when updating the
+// pin count for the very common case that the application pins and unpins the
+// same object without any interleaving by a garbage collection or pinning/unpinning
+// to an object in another region.
+class G1RegionPinCache : public StackObj {
+ uint _region_idx;
+ size_t _count;
+
+ void flush_and_set(uint new_region_idx, size_t new_count);
+
+public:
+ G1RegionPinCache() : _region_idx(G1_NO_HRM_INDEX), _count(0) { }
+
+#ifdef ASSERT
+ size_t count() const { return _count; }
+#endif
+
+ void inc_count(uint region_idx);
+ void dec_count(uint region_idx);
+
+ void flush();
+};
+
+#endif /* SHARE_GC_G1_G1REGIONPINCACHE_HPP */
diff --git a/src/hotspot/share/gc/g1/g1RegionPinCache.inline.hpp b/src/hotspot/share/gc/g1/g1RegionPinCache.inline.hpp
new file mode 100644
index 0000000000000..7df81d208f13c
--- /dev/null
+++ b/src/hotspot/share/gc/g1/g1RegionPinCache.inline.hpp
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1REGIONPINCACHE_INLINE_HPP
+#define SHARE_GC_G1_G1REGIONPINCACHE_INLINE_HPP
+
+#include "gc/g1/g1RegionPinCache.hpp"
+
+#include "gc/g1/g1CollectedHeap.inline.hpp"
+
+inline void G1RegionPinCache::inc_count(uint region_idx) {
+ if (region_idx == _region_idx) {
+ ++_count;
+ } else {
+ flush_and_set(region_idx, (size_t)1);
+ }
+}
+
+inline void G1RegionPinCache::dec_count(uint region_idx) {
+ if (region_idx == _region_idx) {
+ --_count;
+ } else {
+ flush_and_set(region_idx, ~(size_t)0);
+ }
+}
+
+inline void G1RegionPinCache::flush_and_set(uint new_region_idx, size_t new_count) {
+ if (_count != 0) {
+ G1CollectedHeap::heap()->region_at(_region_idx)->add_pinned_object_count(_count);
+ }
+ _region_idx = new_region_idx;
+ _count = new_count;
+}
+
+inline void G1RegionPinCache::flush() {
+ flush_and_set(G1_NO_HRM_INDEX, 0);
+}
+
+#endif /* SHARE_GC_G1_G1REGIONPINCACHE_INLINE_HPP */
diff --git a/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp b/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
index 5ec3c759369ce..88a4b9611dc2a 100644
--- a/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
+++ b/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
@@ -23,9 +23,9 @@
*/
#include "precompiled.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1RegionsOnNodes.hpp"
-#include "gc/g1/heapRegion.hpp"
G1RegionsOnNodes::G1RegionsOnNodes() : _count_per_node(nullptr), _numa(G1NUMA::numa()) {
_count_per_node = NEW_C_HEAP_ARRAY(uint, _numa->num_active_nodes(), mtGC);
diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp
index ac445850085cd..2112795221459 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp
@@ -35,14 +35,14 @@
#include "gc/g1/g1FromCardCache.hpp"
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RootClosures.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1_globals.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/bufferNodeList.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
@@ -1174,24 +1174,27 @@ class G1MergeHeapRootsTask : public WorkerTask {
}
HeapRegion* r = g1h->region_at(region_index);
- if (r->rem_set()->is_empty()) {
- return false;
- }
+
+ assert(r->rem_set()->is_complete(), "humongous candidates must have complete remset");
guarantee(r->rem_set()->occupancy_less_or_equal_than(G1EagerReclaimRemSetThreshold),
"Found a not-small remembered set here. This is inconsistent with previous assumptions.");
- _cl.merge_card_set_for_region(r);
+ if (!r->rem_set()->is_empty()) {
+ _cl.merge_card_set_for_region(r);
- // We should only clear the card based remembered set here as we will not
- // implicitly rebuild anything else during eager reclaim. Note that at the moment
- // (and probably never) we do not enter this path if there are other kind of
- // remembered sets for this region.
- // We want to continue collecting remembered set entries for humongous regions
- // that were not reclaimed.
- r->rem_set()->clear(true /* only_cardset */, true /* keep_tracked */);
+ // We should only clear the card based remembered set here as we will not
+ // implicitly rebuild anything else during eager reclaim. Note that at the moment
+ // (and probably never) we do not enter this path if there are other kind of
+ // remembered sets for this region.
+ // We want to continue collecting remembered set entries for humongous regions
+ // that were not reclaimed.
+ r->rem_set()->clear(true /* only_cardset */, true /* keep_tracked */);
+ }
- assert(r->rem_set()->is_empty() && r->rem_set()->is_complete(), "must be for eager reclaim candidates");
+ // Postcondition
+ assert(r->rem_set()->is_empty(), "must be empty after flushing");
+ assert(r->rem_set()->is_complete(), "should still be after flushing");
return false;
}
diff --git a/src/hotspot/share/gc/g1/g1RemSet.hpp b/src/hotspot/share/gc/g1/g1RemSet.hpp
index 053fc39aa8365..f38e5e7f0f484 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.hpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.hpp
@@ -26,10 +26,10 @@
#define SHARE_GC_G1_G1REMSET_HPP
#include "gc/g1/g1CardTable.hpp"
-#include "gc/g1/g1OopClosures.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1OopClosures.hpp"
#include "gc/g1/g1RemSetSummary.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "memory/allocation.hpp"
#include "memory/iterator.hpp"
#include "utilities/ticks.hpp"
diff --git a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
index 4156e85fbfa00..0cd1823a8ee57 100644
--- a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
@@ -29,10 +29,10 @@
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineThread.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1RemSetSummary.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "memory/allocation.inline.hpp"
#include "memory/iterator.hpp"
#include "runtime/javaThread.hpp"
diff --git a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
index c227b625b8de6..8738e2bc886c2 100644
--- a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
@@ -24,9 +24,9 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1RemSetTrackingPolicy.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "runtime/safepoint.hpp"
bool G1RemSetTrackingPolicy::needs_scan_for_rebuild(HeapRegion* r) const {
diff --git a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp
index 2c2734d4abf4a..94ff2734fbf8e 100644
--- a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_GC_G1_G1REMSETTRACKINGPOLICY_HPP
#define SHARE_GC_G1_G1REMSETTRACKINGPOLICY_HPP
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionType.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionType.hpp"
#include "memory/allocation.hpp"
// The remembered set tracking policy determines for a given region the state of
diff --git a/src/hotspot/share/gc/g1/g1RootProcessor.cpp b/src/hotspot/share/gc/g1/g1RootProcessor.cpp
index 3a9e30ec4034d..51a77eccfdf38 100644
--- a/src/hotspot/share/gc/g1/g1RootProcessor.cpp
+++ b/src/hotspot/share/gc/g1/g1RootProcessor.cpp
@@ -32,11 +32,11 @@
#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1ParScanThreadState.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RootClosures.hpp"
#include "gc/g1/g1RootProcessor.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
#include "gc/shared/oopStorage.inline.hpp"
#include "gc/shared/oopStorageSet.hpp"
#include "gc/shared/oopStorageSetParState.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp b/src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp
index f1cf554ca462a..b7790d154d4cf 100644
--- a/src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp
+++ b/src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp
@@ -25,9 +25,9 @@
#include "precompiled.hpp"
#include "gc/g1/g1BarrierSet.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/satbMarkQueue.hpp"
#include "oops/oop.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp b/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
index 2b09dfc668b59..48773a6634e96 100644
--- a/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
+++ b/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
@@ -24,9 +24,9 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1Predictions.hpp"
#include "gc/g1/g1SurvRateGroup.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "logging/log.hpp"
#include "memory/allocation.hpp"
diff --git a/src/hotspot/share/gc/g1/g1SurvivorRegions.cpp b/src/hotspot/share/gc/g1/g1SurvivorRegions.cpp
index dc821b135148f..fa86f02075deb 100644
--- a/src/hotspot/share/gc/g1/g1SurvivorRegions.cpp
+++ b/src/hotspot/share/gc/g1/g1SurvivorRegions.cpp
@@ -23,8 +23,8 @@
*/
#include "precompiled.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SurvivorRegions.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp b/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp
index e17a8c7960a4d..d0dcb59d7f015 100644
--- a/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp
+++ b/src/hotspot/share/gc/g1/g1ThreadLocalData.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1RegionPinCache.hpp"
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/satbMarkQueue.hpp"
#include "runtime/javaThread.hpp"
@@ -37,9 +38,15 @@ class G1ThreadLocalData {
SATBMarkQueue _satb_mark_queue;
G1DirtyCardQueue _dirty_card_queue;
+ // Per-thread cache of pinned object count to reduce atomic operation traffic
+ // due to region pinning. Holds the last region where the mutator pinned an
+ // object and the number of pin operations since the last change of the region.
+ G1RegionPinCache _pin_cache;
+
G1ThreadLocalData() :
_satb_mark_queue(&G1BarrierSet::satb_mark_queue_set()),
- _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()) {}
+ _dirty_card_queue(&G1BarrierSet::dirty_card_queue_set()),
+ _pin_cache() {}
static G1ThreadLocalData* data(Thread* thread) {
assert(UseG1GC, "Sanity");
@@ -90,6 +97,10 @@ class G1ThreadLocalData {
static ByteSize dirty_card_queue_buffer_offset() {
return dirty_card_queue_offset() + G1DirtyCardQueue::byte_offset_of_buf();
}
+
+ static G1RegionPinCache& pin_count_cache(Thread* thread) {
+ return data(thread)->_pin_cache;
+ }
};
#endif // SHARE_GC_G1_G1THREADLOCALDATA_HPP
diff --git a/src/hotspot/share/gc/g1/g1Trace.cpp b/src/hotspot/share/gc/g1/g1Trace.cpp
index 453728f7e3dcd..598a238e93b92 100644
--- a/src/hotspot/share/gc/g1/g1Trace.cpp
+++ b/src/hotspot/share/gc/g1/g1Trace.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -70,7 +70,7 @@ static void register_jfr_type_constants() {
#endif
void G1NewTracer::initialize() {
- JFR_ONLY(register_jfr_type_constants());
+ JFR_ONLY(register_jfr_type_constants();)
}
void G1NewTracer::report_young_gc_pause(G1GCPauseType pause) {
diff --git a/src/hotspot/share/gc/g1/g1YoungCollector.cpp b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
index ec32f8b589b5c..4fa3b928763ee 100644
--- a/src/hotspot/share/gc/g1/g1YoungCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,6 +41,7 @@
#include "gc/g1/g1ParScanThreadState.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RedirtyCardsQueue.hpp"
+#include "gc/g1/g1RegionPinCache.inline.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1RootProcessor.hpp"
#include "gc/g1/g1Trace.hpp"
@@ -403,7 +404,7 @@ class G1PrepareEvacuationTask : public WorkerTask {
_g1h->register_region_with_region_attr(hr);
}
log_debug(gc, humongous)("Humongous region %u (object size %zu @ " PTR_FORMAT ") remset %zu code roots %zu "
- "marked %d pinned count %u reclaim candidate %d type array %d",
+ "marked %d pinned count %zu reclaim candidate %d type array %d",
index,
cast_to_oop(hr->bottom())->size() * HeapWordSize,
p2i(hr->bottom()),
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
index 377f5f550d17e..18aa381988824 100644
--- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
@@ -34,12 +34,12 @@
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
#include "gc/g1/g1EvacInfo.hpp"
#include "gc/g1/g1EvacStats.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1ParScanThreadState.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "jfr/jfrEvents.hpp"
@@ -590,23 +590,17 @@ class G1PostEvacuateCollectionSetCleanupTask2::ProcessEvacuationFailedRegionsTas
};
class G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask : public G1AbstractSubTask {
- G1RedirtyCardsQueueSet* _rdcqs;
- BufferNode* volatile _nodes;
+ BufferNodeList* _rdc_buffers;
+ uint _num_buffer_lists;
G1EvacFailureRegions* _evac_failure_regions;
public:
- RedirtyLoggedCardsTask(G1RedirtyCardsQueueSet* rdcqs, G1EvacFailureRegions* evac_failure_regions) :
+ RedirtyLoggedCardsTask(G1EvacFailureRegions* evac_failure_regions, BufferNodeList* rdc_buffers, uint num_buffer_lists) :
G1AbstractSubTask(G1GCPhaseTimes::RedirtyCards),
- _rdcqs(rdcqs),
- _nodes(rdcqs->all_completed_buffers()),
+ _rdc_buffers(rdc_buffers),
+ _num_buffer_lists(num_buffer_lists),
_evac_failure_regions(evac_failure_regions) { }
- virtual ~RedirtyLoggedCardsTask() {
- G1DirtyCardQueueSet& dcq = G1BarrierSet::dirty_card_queue_set();
- dcq.merge_bufferlists(_rdcqs);
- _rdcqs->verify_empty();
- }
-
double worker_cost() const override {
// Needs more investigation.
return G1CollectedHeap::heap()->workers()->active_workers();
@@ -614,13 +608,23 @@ class G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask : public G
void do_work(uint worker_id) override {
RedirtyLoggedCardTableEntryClosure cl(G1CollectedHeap::heap(), _evac_failure_regions);
- BufferNode* next = Atomic::load(&_nodes);
- while (next != nullptr) {
- BufferNode* node = next;
- next = Atomic::cmpxchg(&_nodes, node, node->next());
- if (next == node) {
- cl.apply_to_buffer(node, worker_id);
- next = node->next();
+
+ uint start = worker_id;
+ for (uint i = 0; i < _num_buffer_lists; i++) {
+ uint index = (start + i) % _num_buffer_lists;
+
+ BufferNode* next = Atomic::load(&_rdc_buffers[index]._head);
+ BufferNode* tail = Atomic::load(&_rdc_buffers[index]._tail);
+
+ while (next != nullptr) {
+ BufferNode* node = next;
+ next = Atomic::cmpxchg(&_rdc_buffers[index]._head, node, (node != tail ) ? node->next() : nullptr);
+ if (next == node) {
+ cl.apply_to_buffer(node, worker_id);
+ next = (node != tail ) ? node->next() : nullptr;
+ } else {
+ break; // If there is contention, move to the next BufferNodeList
+ }
}
}
record_work_item(worker_id, 0, cl.num_dirtied());
@@ -970,7 +974,10 @@ G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2
add_parallel_task(new RestorePreservedMarksTask(per_thread_states->preserved_marks_set()));
add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions));
}
- add_parallel_task(new RedirtyLoggedCardsTask(per_thread_states->rdcqs(), evac_failure_regions));
+ add_parallel_task(new RedirtyLoggedCardsTask(evac_failure_regions,
+ per_thread_states->rdc_buffers(),
+ per_thread_states->num_workers()));
+
if (UseTLAB && ResizeTLAB) {
add_parallel_task(new ResizeTLABsTask());
}
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp
index d3a6436e432f5..d746062366966 100644
--- a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,8 @@
#include "gc/g1/g1ConcurrentRefineStats.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
#include "gc/g1/g1YoungGCPreEvacuateTasks.hpp"
+#include "gc/g1/g1RegionPinCache.inline.hpp"
+#include "gc/g1/g1ThreadLocalData.hpp"
#include "gc/shared/barrierSet.inline.hpp"
#include "gc/shared/threadLocalAllocBuffer.inline.hpp"
#include "memory/allocation.inline.hpp"
@@ -57,12 +59,15 @@ class G1PreEvacuateCollectionSetBatchTask::JavaThreadRetireTLABAndFlushLogs : pu
assert(thread->is_Java_thread(), "must be");
// Flushes deferred card marks, so must precede concatenating logs.
BarrierSet::barrier_set()->make_parsable((JavaThread*)thread);
+ // Retire TLABs.
if (UseTLAB) {
thread->tlab().retire(&_tlab_stats);
}
-
+ // Concatenate logs.
G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
_refinement_stats += qset.concatenate_log_and_stats(thread);
+ // Flush region pin count cache.
+ G1ThreadLocalData::pin_count_cache(thread).flush();
}
};
@@ -132,6 +137,8 @@ class G1PreEvacuateCollectionSetBatchTask::NonJavaThreadFlushLogs : public G1Abs
void do_thread(Thread* thread) override {
G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
_refinement_stats += qset.concatenate_log_and_stats(thread);
+
+ assert(G1ThreadLocalData::pin_count_cache(thread).count() == 0, "NonJava thread has pinned Java objects");
}
} _tc;
diff --git a/src/hotspot/share/gc/g1/g1YoungGenSizer.cpp b/src/hotspot/share/gc/g1/g1YoungGenSizer.cpp
index 13bbb22252cb9..30b9c6cb7a1a4 100644
--- a/src/hotspot/share/gc/g1/g1YoungGenSizer.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGenSizer.cpp
@@ -24,8 +24,8 @@
#include "precompiled.hpp"
#include "gc/g1/g1Arguments.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1YoungGenSizer.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "logging/log.hpp"
#include "runtime/globals_extension.hpp"
diff --git a/src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp b/src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp
index b04c02a3fc978..46c850cb643a4 100644
--- a/src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp
+++ b/src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp
@@ -23,7 +23,7 @@
*/
#include "precompiled.hpp"
-#include "gc/g1/heapRegionBounds.inline.hpp"
+#include "gc/g1/g1HeapRegionBounds.inline.hpp"
#include "gc/g1/jvmFlagConstraintsG1.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/ptrQueue.hpp"
diff --git a/src/hotspot/share/gc/g1/vmStructs_g1.hpp b/src/hotspot/share/gc/g1/vmStructs_g1.hpp
index 94ade0f387c00..62d91fe310099 100644
--- a/src/hotspot/share/gc/g1/vmStructs_g1.hpp
+++ b/src/hotspot/share/gc/g1/vmStructs_g1.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,8 +26,8 @@
#define SHARE_GC_G1_VMSTRUCTS_G1_HPP
#include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionManager.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "utilities/macros.hpp"
#define VM_STRUCTS_G1GC(nonstatic_field, \
@@ -41,7 +41,7 @@
nonstatic_field(HeapRegion, _bottom, HeapWord* const) \
nonstatic_field(HeapRegion, _top, HeapWord* volatile) \
nonstatic_field(HeapRegion, _end, HeapWord* const) \
- nonstatic_field(HeapRegion, _pinned_object_count, volatile uint) \
+ volatile_nonstatic_field(HeapRegion, _pinned_object_count, size_t) \
\
nonstatic_field(HeapRegionType, _tag, HeapRegionType::Tag volatile) \
\
diff --git a/src/hotspot/share/gc/parallel/mutableSpace.cpp b/src/hotspot/share/gc/parallel/mutableSpace.cpp
index 73329aa6d33cb..c19ca96216235 100644
--- a/src/hotspot/share/gc/parallel/mutableSpace.cpp
+++ b/src/hotspot/share/gc/parallel/mutableSpace.cpp
@@ -120,11 +120,12 @@ void MutableSpace::initialize(MemRegion mr,
}
if (AlwaysPreTouch) {
+ size_t pretouch_page_size = UseLargePages ? page_size : os::vm_page_size();
PretouchTask::pretouch("ParallelGC PreTouch head", (char*)head.start(), (char*)head.end(),
- page_size, pretouch_workers);
+ pretouch_page_size, pretouch_workers);
PretouchTask::pretouch("ParallelGC PreTouch tail", (char*)tail.start(), (char*)tail.end(),
- page_size, pretouch_workers);
+ pretouch_page_size, pretouch_workers);
}
// Remember where we stopped so that we can continue later.
diff --git a/src/hotspot/share/gc/parallel/parMarkBitMap.cpp b/src/hotspot/share/gc/parallel/parMarkBitMap.cpp
index 54db633af5289..deffcc019c95c 100644
--- a/src/hotspot/share/gc/parallel/parMarkBitMap.cpp
+++ b/src/hotspot/share/gc/parallel/parMarkBitMap.cpp
@@ -173,27 +173,26 @@ ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure,
// The bitmap routines require the right boundary to be word-aligned.
const idx_t search_end = align_range_end(range_end);
- idx_t cur_beg = find_obj_beg(range_beg, search_end);
- while (cur_beg < range_end) {
- const idx_t cur_end = find_obj_end(cur_beg, search_end);
- if (cur_end >= range_end) {
- // The obj ends outside the range.
- live_closure->set_source(bit_to_addr(cur_beg));
- return incomplete;
+ idx_t cur_beg = range_beg;
+ while (true) {
+ cur_beg = find_obj_beg(cur_beg, search_end);
+ if (cur_beg >= range_end) {
+ break;
}
- const size_t size = obj_size(cur_beg, cur_end);
+ const size_t size = obj_size(cur_beg);
IterationStatus status = live_closure->do_addr(bit_to_addr(cur_beg), size);
if (status != incomplete) {
assert(status == would_overflow || status == full, "sanity");
return status;
}
- // Successfully processed the object; look for the next object.
- cur_beg = find_obj_beg(cur_end + 1, search_end);
+ cur_beg += words_to_bits(size);
+ if (cur_beg >= range_end) {
+ break;
+ }
}
- live_closure->set_source(bit_to_addr(range_end));
return complete;
}
@@ -210,45 +209,41 @@ ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure,
assert(range_end <= dead_range_end, "dead range invalid");
// The bitmap routines require the right boundary to be word-aligned.
- const idx_t live_search_end = align_range_end(range_end);
const idx_t dead_search_end = align_range_end(dead_range_end);
idx_t cur_beg = range_beg;
if (range_beg < range_end && is_unmarked(range_beg)) {
// The range starts with dead space. Look for the next object, then fill.
+ // This must be the beginning of old/eden/from/to-space, so it's must be
+ // large enough for a filler.
cur_beg = find_obj_beg(range_beg + 1, dead_search_end);
- const idx_t dead_space_end = MIN2(cur_beg - 1, dead_range_end - 1);
+ const idx_t dead_space_end = cur_beg - 1;
const size_t size = obj_size(range_beg, dead_space_end);
dead_closure->do_addr(bit_to_addr(range_beg), size);
}
while (cur_beg < range_end) {
- const idx_t cur_end = find_obj_end(cur_beg, live_search_end);
- if (cur_end >= range_end) {
- // The obj ends outside the range.
- live_closure->set_source(bit_to_addr(cur_beg));
- return incomplete;
- }
-
- const size_t size = obj_size(cur_beg, cur_end);
+ const size_t size = obj_size(cur_beg);
IterationStatus status = live_closure->do_addr(bit_to_addr(cur_beg), size);
if (status != incomplete) {
assert(status == would_overflow || status == full, "sanity");
return status;
}
+ const idx_t dead_space_beg = cur_beg + words_to_bits(size);
+ if (dead_space_beg >= dead_search_end) {
+ break;
+ }
// Look for the start of the next object.
- const idx_t dead_space_beg = cur_end + 1;
cur_beg = find_obj_beg(dead_space_beg, dead_search_end);
if (cur_beg > dead_space_beg) {
// Found dead space; compute the size and invoke the dead closure.
- const idx_t dead_space_end = MIN2(cur_beg - 1, dead_range_end - 1);
- const size_t size = obj_size(dead_space_beg, dead_space_end);
- dead_closure->do_addr(bit_to_addr(dead_space_beg), size);
+ const idx_t dead_space_end = cur_beg - 1;
+ dead_closure->do_addr(bit_to_addr(dead_space_beg),
+ obj_size(dead_space_beg, dead_space_end));
}
}
- live_closure->set_source(bit_to_addr(range_end));
return complete;
}
diff --git a/src/hotspot/share/gc/parallel/parMarkBitMap.hpp b/src/hotspot/share/gc/parallel/parMarkBitMap.hpp
index 7713af6e7a602..ad0e51fdcdf9b 100644
--- a/src/hotspot/share/gc/parallel/parMarkBitMap.hpp
+++ b/src/hotspot/share/gc/parallel/parMarkBitMap.hpp
@@ -81,10 +81,6 @@ class ParMarkBitMap: public CHeapObj
// range [live_range_beg, live_range_end). This is used to iterate over the
// compacted region of the heap. Return values:
//
- // incomplete The iteration is not complete. The last object that
- // begins in the range does not end in the range;
- // closure->source() is set to the start of that object.
- //
// complete The iteration is complete. All objects in the range
// were processed and the closure is not full;
// closure->source() is set one past the end of the range.
diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
index 2531ecb2d5cae..961451e5b5959 100644
--- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
+++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
@@ -102,7 +102,6 @@ jint ParallelScavengeHeap::initialize() {
assert(old_gen()->max_gen_size() == old_rs.size(), "Consistency check");
double max_gc_pause_sec = ((double) MaxGCPauseMillis)/1000.0;
- double max_gc_minor_pause_sec = ((double) MaxGCMinorPauseMillis)/1000.0;
const size_t eden_capacity = _young_gen->eden_space()->capacity_in_bytes();
const size_t old_capacity = _old_gen->capacity_in_bytes();
@@ -113,7 +112,6 @@ jint ParallelScavengeHeap::initialize() {
young_gen()->to_space()->capacity_in_bytes(),
GenAlignment,
max_gc_pause_sec,
- max_gc_minor_pause_sec,
GCTimeRatio
);
@@ -124,7 +122,7 @@ jint ParallelScavengeHeap::initialize() {
_gc_policy_counters =
new PSGCAdaptivePolicyCounters("ParScav:MSC", 2, 2, _size_policy);
- if (!PSParallelCompact::initialize()) {
+ if (!PSParallelCompact::initialize_aux_data()) {
return JNI_ENOMEM;
}
diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
index eec32a2c1c2f5..48dce98b55b71 100644
--- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
+++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
@@ -80,8 +80,6 @@ class ParallelScavengeHeap : public CollectedHeap {
static PSAdaptiveSizePolicy* _size_policy;
static PSGCAdaptivePolicyCounters* _gc_policy_counters;
- SoftRefPolicy _soft_ref_policy;
-
unsigned int _death_march_count;
GCMemoryManager* _young_manager;
@@ -121,12 +119,6 @@ class ParallelScavengeHeap : public CollectedHeap {
_old_pool(nullptr),
_workers("GC Thread", ParallelGCThreads) { }
- // For use by VM operations
- enum CollectionType {
- Scavenge,
- MarkSweep
- };
-
Name kind() const override {
return CollectedHeap::Parallel;
}
@@ -135,8 +127,6 @@ class ParallelScavengeHeap : public CollectedHeap {
return "Parallel";
}
- SoftRefPolicy* soft_ref_policy() override { return &_soft_ref_policy; }
-
GrowableArray memory_managers() override;
GrowableArray memory_pools() override;
diff --git a/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp b/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp
index be6d9a501b575..c8a15c2cb925b 100644
--- a/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp
+++ b/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp
@@ -41,7 +41,6 @@ PSAdaptiveSizePolicy::PSAdaptiveSizePolicy(size_t init_eden_size,
size_t init_survivor_size,
size_t space_alignment,
double gc_pause_goal_sec,
- double gc_minor_pause_goal_sec,
uint gc_cost_ratio) :
AdaptiveSizePolicy(init_eden_size,
init_promo_size,
@@ -55,7 +54,6 @@ PSAdaptiveSizePolicy::PSAdaptiveSizePolicy(size_t init_eden_size,
_major_pause_young_estimator(new LinearLeastSquareFit(AdaptiveSizePolicyWeight)),
_latest_major_mutator_interval_seconds(0),
_space_alignment(space_alignment),
- _gc_minor_pause_goal_sec(gc_minor_pause_goal_sec),
_live_at_last_full_gc(init_promo_size),
_change_old_gen_for_min_pauses(0),
_change_young_gen_for_maj_pauses(0),
@@ -282,7 +280,7 @@ void PSAdaptiveSizePolicy::compute_eden_space_size(
// at a time.
adjust_eden_for_pause_time(&desired_eden_size);
- } else if (_avg_minor_pause->padded_average() > gc_minor_pause_goal_sec()) {
+ } else if (_avg_minor_pause->padded_average() > gc_pause_goal_sec()) {
// Adjust only for the minor pause time goal
adjust_eden_for_minor_pause_time(&desired_eden_size);
@@ -583,12 +581,6 @@ void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(size_t* desired_eden
decrease_young_gen_for_min_pauses_true);
*desired_eden_size_ptr = *desired_eden_size_ptr -
eden_decrement_aligned_down(*desired_eden_size_ptr);
- } else {
- // EXPERIMENTAL ADJUSTMENT
- // Only record that the estimator indicated such an action.
- // *desired_eden_size_ptr = *desired_eden_size_ptr + eden_heap_delta;
- set_change_young_gen_for_min_pauses(
- increase_young_gen_for_min_pauses_true);
}
}
@@ -609,12 +601,6 @@ void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(size_t* desired_promo_siz
set_change_old_gen_for_maj_pauses(decrease_old_gen_for_maj_pauses_true);
promo_heap_delta = promo_decrement_aligned_down(*desired_promo_size_ptr);
*desired_promo_size_ptr = _promo_size - promo_heap_delta;
- } else {
- // EXPERIMENTAL ADJUSTMENT
- // Only record that the estimator indicated such an action.
- // *desired_promo_size_ptr = _promo_size +
- // promo_increment_aligned_up(*desired_promo_size_ptr);
- set_change_old_gen_for_maj_pauses(increase_old_gen_for_maj_pauses_true);
}
}
@@ -700,14 +686,6 @@ void PSAdaptiveSizePolicy::adjust_promo_for_throughput(bool is_full_gc,
set_change_old_gen_for_throughput(
increase_old_gen_for_throughput_true);
_old_gen_change_for_major_throughput++;
- } else {
- // EXPERIMENTAL ADJUSTMENT
- // Record that decreasing the old gen size would decrease
- // the major collection cost but don't do it.
- // *desired_promo_size_ptr = _promo_size -
- // promo_decrement_aligned_down(*desired_promo_size_ptr);
- set_change_old_gen_for_throughput(
- decrease_old_gen_for_throughput_true);
}
break;
@@ -787,14 +765,6 @@ void PSAdaptiveSizePolicy::adjust_eden_for_throughput(bool is_full_gc,
set_change_young_gen_for_throughput(
increase_young_gen_for_througput_true);
_young_gen_change_for_minor_throughput++;
- } else {
- // EXPERIMENTAL ADJUSTMENT
- // Record that decreasing the young gen size would decrease
- // the minor collection cost but don't do it.
- // *desired_eden_size_ptr = _eden_size -
- // eden_decrement_aligned_down(*desired_eden_size_ptr);
- set_change_young_gen_for_throughput(
- decrease_young_gen_for_througput_true);
}
break;
default:
diff --git a/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp b/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp
index f6d4be029fbab..997a478466701 100644
--- a/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp
+++ b/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp
@@ -91,8 +91,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
const size_t _space_alignment; // alignment for eden, survivors
- const double _gc_minor_pause_goal_sec; // goal for maximum minor gc pause
-
// The amount of live data in the heap at the last full GC, used
// as a baseline to help us determine when we need to perform the
// next full GC.
@@ -113,9 +111,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
private:
- // Accessors
- double gc_minor_pause_goal_sec() const { return _gc_minor_pause_goal_sec; }
-
void adjust_eden_for_minor_pause_time(size_t* desired_eden_size_ptr);
// Change the generation sizes to achieve a GC pause time goal
// Returned sizes are not necessarily aligned.
@@ -187,7 +182,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
size_t init_survivor_size,
size_t space_alignment,
double gc_pause_goal_sec,
- double gc_minor_pause_goal_sec,
uint gc_time_ratio);
// Methods indicating events of interest to the adaptive size policy,
diff --git a/src/hotspot/share/gc/parallel/psCardTable.cpp b/src/hotspot/share/gc/parallel/psCardTable.cpp
index 91fcf026069ce..345d81ed14766 100644
--- a/src/hotspot/share/gc/parallel/psCardTable.cpp
+++ b/src/hotspot/share/gc/parallel/psCardTable.cpp
@@ -38,13 +38,11 @@
// Checks an individual oop for missing precise marks. Mark
// may be either dirty or newgen.
-class CheckForUnmarkedOops : public BasicOopIterateClosure {
- private:
+class PSCheckForUnmarkedOops : public BasicOopIterateClosure {
PSYoungGen* _young_gen;
PSCardTable* _card_table;
HeapWord* _unmarked_addr;
- protected:
template void do_oop_work(T* p) {
oop obj = RawAccess<>::oop_load(p);
if (_young_gen->is_in_reserved(obj) &&
@@ -57,11 +55,11 @@ class CheckForUnmarkedOops : public BasicOopIterateClosure {
}
public:
- CheckForUnmarkedOops(PSYoungGen* young_gen, PSCardTable* card_table) :
+ PSCheckForUnmarkedOops(PSYoungGen* young_gen, PSCardTable* card_table) :
_young_gen(young_gen), _card_table(card_table), _unmarked_addr(nullptr) { }
- virtual void do_oop(oop* p) { CheckForUnmarkedOops::do_oop_work(p); }
- virtual void do_oop(narrowOop* p) { CheckForUnmarkedOops::do_oop_work(p); }
+ void do_oop(oop* p) override { do_oop_work(p); }
+ void do_oop(narrowOop* p) override { do_oop_work(p); }
bool has_unmarked_oop() {
return _unmarked_addr != nullptr;
@@ -70,13 +68,13 @@ class CheckForUnmarkedOops : public BasicOopIterateClosure {
// Checks all objects for the existence of some type of mark,
// precise or imprecise, dirty or newgen.
-class CheckForUnmarkedObjects : public ObjectClosure {
+class PSCheckForUnmarkedObjects : public ObjectClosure {
private:
PSYoungGen* _young_gen;
PSCardTable* _card_table;
public:
- CheckForUnmarkedObjects() {
+ PSCheckForUnmarkedObjects() {
ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
_young_gen = heap->young_gen();
_card_table = heap->card_table();
@@ -87,7 +85,7 @@ class CheckForUnmarkedObjects : public ObjectClosure {
// we test for missing precise marks first. If any are found, we don't
// fail unless the object head is also unmarked.
virtual void do_object(oop obj) {
- CheckForUnmarkedOops object_check(_young_gen, _card_table);
+ PSCheckForUnmarkedOops object_check(_young_gen, _card_table);
obj->oop_iterate(&object_check);
if (object_check.has_unmarked_oop()) {
guarantee(_card_table->is_dirty_for_addr(obj), "Found unmarked young_gen object");
@@ -379,7 +377,7 @@ void PSCardTable::scavenge_contents_parallel(ObjectStartArray* start_array,
// This should be called before a scavenge.
void PSCardTable::verify_all_young_refs_imprecise() {
- CheckForUnmarkedObjects check;
+ PSCheckForUnmarkedObjects check;
ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
PSOldGen* old_gen = heap->old_gen();
diff --git a/src/hotspot/share/gc/parallel/psCompactionManager.cpp b/src/hotspot/share/gc/parallel/psCompactionManager.cpp
index e3b35db4bff52..fe9bcbff703b9 100644
--- a/src/hotspot/share/gc/parallel/psCompactionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psCompactionManager.cpp
@@ -61,6 +61,7 @@ ParCompactionManager::ParCompactionManager() {
reset_bitmap_query_cache();
_deferred_obj_array = new (mtGC) GrowableArray(10, mtGC);
+ _marking_stats_cache = nullptr;
}
void ParCompactionManager::initialize(ParMarkBitMap* mbm) {
diff --git a/src/hotspot/share/gc/parallel/psCompactionManager.hpp b/src/hotspot/share/gc/parallel/psCompactionManager.hpp
index 458d33af74a87..b33ad06ee3ee1 100644
--- a/src/hotspot/share/gc/parallel/psCompactionManager.hpp
+++ b/src/hotspot/share/gc/parallel/psCompactionManager.hpp
@@ -111,7 +111,35 @@ class ParCompactionManager : public CHeapObj {
static RegionTaskQueueSet* region_task_queues() { return _region_task_queues; }
OopTaskQueue* oop_stack() { return &_oop_stack; }
- public:
+ // To collect per-region live-words in a worker local cache in order to
+ // reduce threads contention.
+ class MarkingStatsCache : public CHeapObj {
+ constexpr static size_t num_entries = 1024;
+ static_assert(is_power_of_2(num_entries), "inv");
+ static_assert(num_entries > 0, "inv");
+
+ constexpr static size_t entry_mask = num_entries - 1;
+
+ struct CacheEntry {
+ size_t region_id;
+ size_t live_words;
+ };
+
+ CacheEntry entries[num_entries] = {};
+
+ inline void push(size_t region_id, size_t live_words);
+
+ public:
+ inline void push(oop obj, size_t live_words);
+
+ inline void evict(size_t index);
+
+ inline void evict_all();
+ };
+
+ MarkingStatsCache* _marking_stats_cache;
+
+public:
static const size_t InvalidShadow = ~0;
static size_t pop_shadow_region_mt_safe(PSParallelCompact::RegionData* region_ptr);
static void push_shadow_region_mt_safe(size_t shadow_region);
@@ -198,6 +226,10 @@ class ParCompactionManager : public CHeapObj {
virtual void do_void();
};
+ inline void create_marking_stats_cache();
+
+ inline void flush_and_destroy_marking_stats_cache();
+
// Called after marking.
static void verify_all_marking_stack_empty() NOT_DEBUG_RETURN;
diff --git a/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp b/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
index c0eb546022892..ae2e449e0b6ab 100644
--- a/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
@@ -107,6 +107,8 @@ inline void ParCompactionManager::mark_and_push(T* p) {
assert(ParallelScavengeHeap::heap()->is_in(obj), "should be in heap");
if (mark_bitmap()->is_unmarked(obj) && PSParallelCompact::mark_obj(obj)) {
+ assert(_marking_stats_cache != nullptr, "inv");
+ _marking_stats_cache->push(obj, obj->size());
push(obj);
if (StringDedup::is_enabled() &&
@@ -174,4 +176,73 @@ inline void ParCompactionManager::follow_contents(oop obj) {
}
}
+inline void ParCompactionManager::MarkingStatsCache::push(size_t region_id, size_t live_words) {
+ size_t index = (region_id & entry_mask);
+ if (entries[index].region_id == region_id) {
+ // Hit
+ entries[index].live_words += live_words;
+ return;
+ }
+ // Miss
+ if (entries[index].live_words != 0) {
+ evict(index);
+ }
+ entries[index].region_id = region_id;
+ entries[index].live_words = live_words;
+}
+
+inline void ParCompactionManager::MarkingStatsCache::push(oop obj, size_t live_words) {
+ ParallelCompactData& data = PSParallelCompact::summary_data();
+ const size_t region_size = ParallelCompactData::RegionSize;
+
+ HeapWord* addr = cast_from_oop(obj);
+ const size_t start_region_id = data.addr_to_region_idx(addr);
+ const size_t end_region_id = data.addr_to_region_idx(addr + live_words - 1);
+ if (start_region_id == end_region_id) {
+ // Completely inside this region
+ push(start_region_id, live_words);
+ return;
+ }
+
+ // First region
+ push(start_region_id, region_size - data.region_offset(addr));
+
+ // Middle regions; bypass cache
+ for (size_t i = start_region_id + 1; i < end_region_id; ++i) {
+ data.region(i)->set_partial_obj_size(region_size);
+ data.region(i)->set_partial_obj_addr(addr);
+ }
+
+ // Last region; bypass cache
+ const size_t end_offset = data.region_offset(addr + live_words - 1);
+ data.region(end_region_id)->set_partial_obj_size(end_offset + 1);
+ data.region(end_region_id)->set_partial_obj_addr(addr);
+}
+
+inline void ParCompactionManager::MarkingStatsCache::evict(size_t index) {
+ ParallelCompactData& data = PSParallelCompact::summary_data();
+ // flush to global data
+ data.region(entries[index].region_id)->add_live_obj(entries[index].live_words);
+}
+
+inline void ParCompactionManager::MarkingStatsCache::evict_all() {
+ for (size_t i = 0; i < num_entries; ++i) {
+ if (entries[i].live_words != 0) {
+ evict(i);
+ entries[i].live_words = 0;
+ }
+ }
+}
+
+inline void ParCompactionManager::create_marking_stats_cache() {
+ assert(_marking_stats_cache == nullptr, "precondition");
+ _marking_stats_cache = new MarkingStatsCache();
+}
+
+inline void ParCompactionManager::flush_and_destroy_marking_stats_cache() {
+ _marking_stats_cache->evict_all();
+ delete _marking_stats_cache;
+ _marking_stats_cache = nullptr;
+}
+
#endif // SHARE_GC_PARALLEL_PSCOMPACTIONMANAGER_INLINE_HPP
diff --git a/src/hotspot/share/gc/parallel/psOldGen.cpp b/src/hotspot/share/gc/parallel/psOldGen.cpp
index 14be4745b153d..2e1de34258ae0 100644
--- a/src/hotspot/share/gc/parallel/psOldGen.cpp
+++ b/src/hotspot/share/gc/parallel/psOldGen.cpp
@@ -121,12 +121,6 @@ void PSOldGen::initialize_performance_counters(const char* perf_data_name, int l
_object_space, _gen_counters);
}
-// Assume that the generation has been allocated if its
-// reserved size is not 0.
-bool PSOldGen::is_allocated() {
- return virtual_space()->reserved_size() != 0;
-}
-
size_t PSOldGen::num_iterable_blocks() const {
return (object_space()->used_in_bytes() + IterateBlockSize - 1) / IterateBlockSize;
}
diff --git a/src/hotspot/share/gc/parallel/psOldGen.hpp b/src/hotspot/share/gc/parallel/psOldGen.hpp
index 4898baa3a4409..82aa920f4013b 100644
--- a/src/hotspot/share/gc/parallel/psOldGen.hpp
+++ b/src/hotspot/share/gc/parallel/psOldGen.hpp
@@ -105,9 +105,6 @@ class PSOldGen : public CHeapObj {
ObjectStartArray* start_array() { return &_start_array; }
PSVirtualSpace* virtual_space() const { return _virtual_space;}
- // Has the generation been successfully allocated?
- bool is_allocated();
-
// Size info
size_t capacity_in_bytes() const { return object_space()->capacity_in_bytes(); }
size_t used_in_bytes() const { return object_space()->used_in_bytes(); }
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp
index e45b60851124f..886a5d308c14b 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -246,7 +246,7 @@ void PSParallelCompact::print_region_ranges() {
}
}
-void
+static void
print_generic_summary_region(size_t i, const ParallelCompactData::RegionData* c)
{
#define REGION_IDX_FORMAT SIZE_FORMAT_W(7)
@@ -312,7 +312,7 @@ print_generic_summary_data(ParallelCompactData& summary_data,
}
}
-void
+static void
print_initial_summary_data(ParallelCompactData& summary_data,
const MutableSpace* space) {
if (space->top() == space->bottom()) {
@@ -393,7 +393,7 @@ print_initial_summary_data(ParallelCompactData& summary_data,
max_reclaimed_ratio_region, max_dead_to_right, max_live_to_right, max_reclaimed_ratio);
}
-void
+static void
print_initial_summary_data(ParallelCompactData& summary_data,
SpaceInfo* space_info) {
if (!log_develop_is_enabled(Trace, gc, compaction)) {
@@ -862,14 +862,10 @@ void PSParallelCompact::post_initialize() {
ParCompactionManager::initialize(mark_bitmap());
}
-bool PSParallelCompact::initialize() {
+bool PSParallelCompact::initialize_aux_data() {
ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
MemRegion mr = heap->reserved_region();
-
- // Was the old gen get allocated successfully?
- if (!heap->old_gen()->is_allocated()) {
- return false;
- }
+ assert(mr.byte_size() != 0, "heap should be reserved");
initialize_space_info();
initialize_dead_wood_limiter();
@@ -1048,142 +1044,6 @@ void PSParallelCompact::post_compact()
Universe::heap()->record_whole_heap_examined_timestamp();
}
-HeapWord*
-PSParallelCompact::compute_dense_prefix_via_density(const SpaceId id,
- bool maximum_compaction)
-{
- const size_t region_size = ParallelCompactData::RegionSize;
- const ParallelCompactData& sd = summary_data();
-
- const MutableSpace* const space = _space_info[id].space();
- HeapWord* const top_aligned_up = sd.region_align_up(space->top());
- const RegionData* const beg_cp = sd.addr_to_region_ptr(space->bottom());
- const RegionData* const end_cp = sd.addr_to_region_ptr(top_aligned_up);
-
- // Skip full regions at the beginning of the space--they are necessarily part
- // of the dense prefix.
- size_t full_count = 0;
- const RegionData* cp;
- for (cp = beg_cp; cp < end_cp && cp->data_size() == region_size; ++cp) {
- ++full_count;
- }
-
- const uint total_invocations = ParallelScavengeHeap::heap()->total_full_collections();
- assert(total_invocations >= _maximum_compaction_gc_num, "sanity");
- const size_t gcs_since_max = total_invocations - _maximum_compaction_gc_num;
- const bool interval_ended = gcs_since_max > HeapMaximumCompactionInterval;
- if (maximum_compaction || cp == end_cp || interval_ended) {
- _maximum_compaction_gc_num = total_invocations;
- return sd.region_to_addr(cp);
- }
-
- HeapWord* const new_top = _space_info[id].new_top();
- const size_t space_live = pointer_delta(new_top, space->bottom());
- const size_t space_used = space->used_in_words();
- const size_t space_capacity = space->capacity_in_words();
-
- const double cur_density = double(space_live) / space_capacity;
- const double deadwood_density =
- (1.0 - cur_density) * (1.0 - cur_density) * cur_density * cur_density;
- const size_t deadwood_goal = size_t(space_capacity * deadwood_density);
-
- log_develop_debug(gc, compaction)(
- "cur_dens=%5.3f dw_dens=%5.3f dw_goal=" SIZE_FORMAT,
- cur_density, deadwood_density, deadwood_goal);
- log_develop_debug(gc, compaction)(
- "space_live=" SIZE_FORMAT " space_used=" SIZE_FORMAT " "
- "space_cap=" SIZE_FORMAT,
- space_live, space_used,
- space_capacity);
-
- // XXX - Use binary search?
- HeapWord* dense_prefix = sd.region_to_addr(cp);
- const RegionData* full_cp = cp;
- const RegionData* const top_cp = sd.addr_to_region_ptr(space->top() - 1);
- while (cp < end_cp) {
- HeapWord* region_destination = cp->destination();
- const size_t cur_deadwood = pointer_delta(dense_prefix, region_destination);
-
- log_develop_trace(gc, compaction)(
- "c#=" SIZE_FORMAT_W(4) " dst=" PTR_FORMAT " "
- "dp=" PTR_FORMAT " cdw=" SIZE_FORMAT_W(8),
- sd.region(cp), p2i(region_destination),
- p2i(dense_prefix), cur_deadwood);
-
- if (cur_deadwood >= deadwood_goal) {
- // Found the region that has the correct amount of deadwood to the left.
- // This typically occurs after crossing a fairly sparse set of regions, so
- // iterate backwards over those sparse regions, looking for the region
- // that has the lowest density of live objects 'to the right.'
- size_t space_to_left = sd.region(cp) * region_size;
- size_t live_to_left = space_to_left - cur_deadwood;
- size_t space_to_right = space_capacity - space_to_left;
- size_t live_to_right = space_live - live_to_left;
- double density_to_right = double(live_to_right) / space_to_right;
- while (cp > full_cp) {
- --cp;
- const size_t prev_region_live_to_right = live_to_right -
- cp->data_size();
- const size_t prev_region_space_to_right = space_to_right + region_size;
- double prev_region_density_to_right =
- double(prev_region_live_to_right) / prev_region_space_to_right;
- if (density_to_right <= prev_region_density_to_right) {
- return dense_prefix;
- }
-
- log_develop_trace(gc, compaction)(
- "backing up from c=" SIZE_FORMAT_W(4) " d2r=%10.8f "
- "pc_d2r=%10.8f",
- sd.region(cp), density_to_right,
- prev_region_density_to_right);
-
- dense_prefix -= region_size;
- live_to_right = prev_region_live_to_right;
- space_to_right = prev_region_space_to_right;
- density_to_right = prev_region_density_to_right;
- }
- return dense_prefix;
- }
-
- dense_prefix += region_size;
- ++cp;
- }
-
- return dense_prefix;
-}
-
-#ifndef PRODUCT
-void PSParallelCompact::print_dense_prefix_stats(const char* const algorithm,
- const SpaceId id,
- const bool maximum_compaction,
- HeapWord* const addr)
-{
- const size_t region_idx = summary_data().addr_to_region_idx(addr);
- RegionData* const cp = summary_data().region(region_idx);
- const MutableSpace* const space = _space_info[id].space();
- HeapWord* const new_top = _space_info[id].new_top();
-
- const size_t space_live = pointer_delta(new_top, space->bottom());
- const size_t dead_to_left = pointer_delta(addr, cp->destination());
- const size_t space_cap = space->capacity_in_words();
- const double dead_to_left_pct = double(dead_to_left) / space_cap;
- const size_t live_to_right = new_top - cp->destination();
- const size_t dead_to_right = space->top() - addr - live_to_right;
-
- log_develop_debug(gc, compaction)(
- "%s=" PTR_FORMAT " dpc=" SIZE_FORMAT_W(5) " "
- "spl=" SIZE_FORMAT " "
- "d2l=" SIZE_FORMAT " d2l%%=%6.4f "
- "d2r=" SIZE_FORMAT " l2r=" SIZE_FORMAT " "
- "ratio=%10.8f",
- algorithm, p2i(addr), region_idx,
- space_live,
- dead_to_left, dead_to_left_pct,
- dead_to_right, live_to_right,
- double(dead_to_right) / live_to_right);
-}
-#endif // #ifndef PRODUCT
-
// Return a fraction indicating how much of the generation can be treated as
// "dead wood" (i.e., not reclaimed). The function uses a normal distribution
// based on the density of live objects in the generation to determine a limit,
@@ -1432,68 +1292,36 @@ void PSParallelCompact::summarize_spaces_quick()
}
}
-void PSParallelCompact::fill_dense_prefix_end(SpaceId id)
-{
+void PSParallelCompact::fill_dense_prefix_end(SpaceId id) {
+ // Since both markword and klass takes 1 heap word, the min-obj-size is 2
+ // heap words.
+ // If min-fill-size decreases to 1, this whole method becomes redundant.
+ assert(CollectedHeap::min_fill_size() == 2, "inv");
+#ifndef _LP64
+ // In 32-bit system, min-obj-alignment is >= 8 bytes, so the gap (if any)
+ // right before denses-prefix must be greater than min-fill-size; nothing to
+ // do.
+ return;
+#endif
HeapWord* const dense_prefix_end = dense_prefix(id);
- const RegionData* region = _summary_data.addr_to_region_ptr(dense_prefix_end);
- const idx_t dense_prefix_bit = _mark_bitmap.addr_to_bit(dense_prefix_end);
- if (dead_space_crosses_boundary(region, dense_prefix_bit)) {
- // Only enough dead space is filled so that any remaining dead space to the
- // left is larger than the minimum filler object. (The remainder is filled
- // during the copy/update phase.)
- //
- // The size of the dead space to the right of the boundary is not a
- // concern, since compaction will be able to use whatever space is
- // available.
- //
- // Here '||' is the boundary, 'x' represents a don't care bit and a box
- // surrounds the space to be filled with an object.
- //
- // In the 32-bit VM, each bit represents two 32-bit words:
- // +---+
- // a) beg_bits: ... x x x | 0 | || 0 x x ...
- // end_bits: ... x x x | 0 | || 0 x x ...
- // +---+
- //
- // In the 64-bit VM, each bit represents one 64-bit word:
- // +------------+
- // b) beg_bits: ... x x x | 0 || 0 | x x ...
- // end_bits: ... x x 1 | 0 || 0 | x x ...
- // +------------+
- // +-------+
- // c) beg_bits: ... x x | 0 0 | || 0 x x ...
- // end_bits: ... x 1 | 0 0 | || 0 x x ...
- // +-------+
- // +-----------+
- // d) beg_bits: ... x | 0 0 0 | || 0 x x ...
- // end_bits: ... 1 | 0 0 0 | || 0 x x ...
- // +-----------+
- // +-------+
- // e) beg_bits: ... 0 0 | 0 0 | || 0 x x ...
- // end_bits: ... 0 0 | 0 0 | || 0 x x ...
- // +-------+
-
- // Initially assume case a, c or e will apply.
- size_t obj_len = CollectedHeap::min_fill_size();
- HeapWord* obj_beg = dense_prefix_end - obj_len;
-
-#ifdef _LP64
- if (MinObjAlignment > 1) { // object alignment > heap word size
- // Cases a, c or e.
- } else if (_mark_bitmap.is_obj_end(dense_prefix_bit - 2)) {
- // Case b above.
- obj_beg = dense_prefix_end - 1;
- } else if (!_mark_bitmap.is_obj_end(dense_prefix_bit - 3) &&
- _mark_bitmap.is_obj_end(dense_prefix_bit - 4)) {
- // Case d above.
- obj_beg = dense_prefix_end - 3;
- obj_len = 3;
- }
-#endif // #ifdef _LP64
+ RegionData* const region_after_dense_prefix = _summary_data.addr_to_region_ptr(dense_prefix_end);
+ idx_t const dense_prefix_bit = _mark_bitmap.addr_to_bit(dense_prefix_end);
+ if (region_after_dense_prefix->partial_obj_size() != 0 ||
+ _mark_bitmap.is_obj_beg(dense_prefix_bit)) {
+ // The region after the dense prefix starts with live bytes.
+ return;
+ }
+
+ if (_mark_bitmap.is_obj_end(dense_prefix_bit - 2)) {
+ // There is exactly one heap word gap right before the dense prefix end, so we need a filler object.
+ // The filler object will extend into the region after the last dense prefix region.
+ const size_t obj_len = 2; // min-fill-size
+ HeapWord* const obj_beg = dense_prefix_end - 1;
CollectedHeap::fill_with_object(obj_beg, obj_len);
_mark_bitmap.mark_obj(obj_beg, obj_len);
- _summary_data.add_obj(obj_beg, obj_len);
+ _summary_data.addr_to_region_ptr(obj_beg)->add_live_obj(1);
+ region_after_dense_prefix->set_partial_obj_size(1);
assert(start_array(id) != nullptr, "sanity");
start_array(id)->update_for_block(obj_beg, obj_beg + obj_len);
}
@@ -1511,15 +1339,6 @@ PSParallelCompact::summarize_space(SpaceId id, bool maximum_compaction)
HeapWord* dense_prefix_end = compute_dense_prefix(id, maximum_compaction);
_space_info[id].set_dense_prefix(dense_prefix_end);
-#ifndef PRODUCT
- if (log_is_enabled(Debug, gc, compaction)) {
- print_dense_prefix_stats("ratio", id, maximum_compaction,
- dense_prefix_end);
- HeapWord* addr = compute_dense_prefix_via_density(id, maximum_compaction);
- print_dense_prefix_stats("density", id, maximum_compaction, addr);
- }
-#endif // #ifndef PRODUCT
-
// Recompute the summary data, taking into account the dense prefix. If
// every last byte will be reclaimed, then the existing summary data which
// compacts everything can be left in place.
@@ -1969,6 +1788,7 @@ class MarkFromRootsTask : public WorkerTask {
virtual void work(uint worker_id) {
ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(worker_id);
+ cm->create_marking_stats_cache();
PCMarkAndPushClosure mark_and_push_closure(cm);
{
@@ -2017,6 +1837,13 @@ class ParallelCompactRefProcProxyTask : public RefProcProxyTask {
}
};
+static void flush_marking_stats_cache(const uint num_workers) {
+ for (uint i = 0; i < num_workers; ++i) {
+ ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(i);
+ cm->flush_and_destroy_marking_stats_cache();
+ }
+}
+
void PSParallelCompact::marking_phase(ParallelOldTracer *gc_tracer) {
// Recursively traverse all live objects and mark them
GCTraceTime(Info, gc, phases) tm("Marking Phase", &_gc_timer);
@@ -2046,6 +1873,12 @@ void PSParallelCompact::marking_phase(ParallelOldTracer *gc_tracer) {
pt.print_all_references();
}
+ {
+ GCTraceTime(Debug, gc, phases) tm("Flush Marking Stats", &_gc_timer);
+
+ flush_marking_stats_cache(active_gc_threads);
+ }
+
// This is the point where the entire marking should have completed.
ParCompactionManager::verify_all_marking_stack_empty();
@@ -2537,7 +2370,6 @@ void PSParallelCompact::verify_complete(SpaceId space_id) {
#endif // #ifdef ASSERT
inline void UpdateOnlyClosure::do_addr(HeapWord* addr) {
- _start_array->update_for_block(addr, addr + cast_to_oop(addr)->size());
compaction_manager()->update_contents(cast_to_oop(addr));
}
@@ -2562,8 +2394,10 @@ PSParallelCompact::update_and_deadwood_in_dense_prefix(ParCompactionManager* cm,
assert(sd.region(claim_region)->claim_unsafe(), "claim() failed");
}
#endif // #ifdef ASSERT
+ HeapWord* const space_bottom = space(space_id)->bottom();
- if (beg_addr != space(space_id)->bottom()) {
+ // Check if it's the first region in this space.
+ if (beg_addr != space_bottom) {
// Find the first live object or block of dead space that *starts* in this
// range of regions. If a partial object crosses onto the region, skip it;
// it will be marked for 'deferred update' when the object head is
@@ -2575,8 +2409,11 @@ PSParallelCompact::update_and_deadwood_in_dense_prefix(ParCompactionManager* cm,
const RegionData* const cp = sd.region(beg_region);
if (cp->partial_obj_size() != 0) {
beg_addr = sd.partial_obj_end(beg_region);
- } else if (dead_space_crosses_boundary(cp, mbm->addr_to_bit(beg_addr))) {
- beg_addr = mbm->find_obj_beg(beg_addr, end_addr);
+ } else {
+ idx_t beg_bit = mbm->addr_to_bit(beg_addr);
+ if (!mbm->is_obj_beg(beg_bit) && !mbm->is_obj_end(beg_bit - 1)) {
+ beg_addr = mbm->find_obj_beg(beg_addr, end_addr);
+ }
}
}
@@ -2587,12 +2424,7 @@ PSParallelCompact::update_and_deadwood_in_dense_prefix(ParCompactionManager* cm,
// Create closures and iterate.
UpdateOnlyClosure update_closure(mbm, cm, space_id);
FillClosure fill_closure(cm, space_id);
- ParMarkBitMap::IterationStatus status;
- status = mbm->iterate(&update_closure, &fill_closure, beg_addr, end_addr,
- dense_prefix_end);
- if (status == ParMarkBitMap::incomplete) {
- update_closure.do_addr(update_closure.source());
- }
+ mbm->iterate(&update_closure, &fill_closure, beg_addr, end_addr, dense_prefix_end);
}
// Mark the regions as filled.
@@ -2903,25 +2735,6 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu
src_space_top);
IterationStatus status = bitmap->iterate(&closure, cur_addr, end_addr);
- if (status == ParMarkBitMap::incomplete) {
- // The last obj that starts in the source region does not end in the
- // region.
- assert(closure.source() < end_addr, "sanity");
- HeapWord* const obj_beg = closure.source();
- HeapWord* const range_end = MIN2(obj_beg + closure.words_remaining(),
- src_space_top);
- HeapWord* const obj_end = bitmap->find_obj_end(obj_beg, range_end);
- if (obj_end < range_end) {
- // The end was found; the entire object will fit.
- status = closure.do_addr(obj_beg, bitmap->obj_size(obj_beg, obj_end));
- assert(status != ParMarkBitMap::would_overflow, "sanity");
- } else {
- // The end was not found; the object will not fit.
- assert(range_end < src_space_top, "obj cannot cross space boundary");
- status = ParMarkBitMap::would_overflow;
- }
- }
-
if (status == ParMarkBitMap::would_overflow) {
// The last object did not fit. Note that interior oop updates were
// deferred, then copy enough of the object to fill the region.
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.hpp
index 94995bd4f195c..6a158e1385e91 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.hpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.hpp
@@ -744,7 +744,6 @@ class ParMarkBitMapClosure: public StackObj {
private:
ParMarkBitMap* const _bitmap;
ParCompactionManager* const _compaction_manager;
- DEBUG_ONLY(const size_t _initial_words_remaining;) // Useful in debugger.
size_t _words_remaining; // Words left to copy.
protected:
@@ -756,9 +755,6 @@ ParMarkBitMapClosure::ParMarkBitMapClosure(ParMarkBitMap* bitmap,
ParCompactionManager* cm,
size_t words):
_bitmap(bitmap), _compaction_manager(cm)
-#ifdef ASSERT
- , _initial_words_remaining(words)
-#endif
{
_words_remaining = words;
_source = nullptr;
@@ -975,11 +971,6 @@ class PSParallelCompact : AllStatic {
// Mark live objects
static void marking_phase(ParallelOldTracer *gc_tracer);
- // Compute the dense prefix for the designated space. This is an experimental
- // implementation currently not used in production.
- static HeapWord* compute_dense_prefix_via_density(const SpaceId id,
- bool maximum_compaction);
-
// Methods used to compute the dense prefix.
// Compute the value of the normal distribution at x = density. The mean and
@@ -1021,14 +1012,8 @@ class PSParallelCompact : AllStatic {
static HeapWord* compute_dense_prefix(const SpaceId id,
bool maximum_compaction);
- // Return true if dead space crosses onto the specified Region; bit must be
- // the bit index corresponding to the first word of the Region.
- static inline bool dead_space_crosses_boundary(const RegionData* region,
- idx_t bit);
-
- // Summary phase utility routine to fill dead space (if any) at the dense
- // prefix boundary. Should only be called if the dense prefix is
- // non-empty.
+ // Create a filler obj (if needed) right before the dense-prefix-boundary to
+ // make the heap parsable.
static void fill_dense_prefix_end(SpaceId id);
static void summarize_spaces_quick();
@@ -1069,7 +1054,7 @@ class PSParallelCompact : AllStatic {
// allocations. This should be called during the VM initialization
// at a pointer where it would be appropriate to return a JNI_ENOMEM
// in the event of a failure.
- static bool initialize();
+ static bool initialize_aux_data();
// Closure accessors
static BoolObjectClosure* is_alive_closure() { return &_is_alive_closure; }
@@ -1170,10 +1155,6 @@ class PSParallelCompact : AllStatic {
// Debugging support.
static const char* space_names[last_space_id];
static void print_region_ranges();
- static void print_dense_prefix_stats(const char* const algorithm,
- const SpaceId id,
- const bool maximum_compaction,
- HeapWord* const addr);
static void summary_phase_msg(SpaceId dst_space_id,
HeapWord* dst_beg, HeapWord* dst_end,
SpaceId src_space_id,
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
index ff6052d5070a7..12dbcab895980 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
@@ -46,20 +46,6 @@ inline double PSParallelCompact::normal_distribution(double density) {
return _dwl_first_term * exp(-0.5 * squared_term * squared_term);
}
-inline bool PSParallelCompact::dead_space_crosses_boundary(const RegionData* region,
- idx_t bit) {
- assert(bit > 0, "cannot call this for the first bit/region");
- assert(_summary_data.region_to_addr(region) == _mark_bitmap.bit_to_addr(bit),
- "sanity check");
-
- // Dead space crosses the boundary if (1) a partial object does not extend
- // onto the region, (2) an object does not start at the beginning of the
- // region, and (3) an object does not end at the end of the prior region.
- return region->partial_obj_size() == 0 &&
- !_mark_bitmap.is_obj_beg(bit) &&
- !_mark_bitmap.is_obj_end(bit - 1);
-}
-
inline bool PSParallelCompact::is_in(HeapWord* p, HeapWord* beg_addr, HeapWord* end_addr) {
return p >= beg_addr && p < end_addr;
}
@@ -100,7 +86,6 @@ inline void PSParallelCompact::check_new_location(HeapWord* old_addr, HeapWord*
inline bool PSParallelCompact::mark_obj(oop obj) {
const size_t obj_size = obj->size();
if (mark_bitmap()->mark_obj(obj, obj_size)) {
- _summary_data.add_obj(obj, obj_size);
ContinuationGCSupport::transform_stack_chunk(obj);
return true;
} else {
diff --git a/src/hotspot/share/gc/parallel/psScavenge.cpp b/src/hotspot/share/gc/parallel/psScavenge.cpp
index c864ded21c5eb..926006a8a0ecd 100644
--- a/src/hotspot/share/gc/parallel/psScavenge.cpp
+++ b/src/hotspot/share/gc/parallel/psScavenge.cpp
@@ -298,8 +298,6 @@ class ScavengeRootsTask : public WorkerTask {
_active_workers(active_workers),
_is_old_gen_empty(old_gen->object_space()->is_empty()),
_terminator(active_workers, PSPromotionManager::vm_thread_promotion_manager()->stack_array_depth()) {
- assert(_old_gen != nullptr, "Sanity");
-
if (!_is_old_gen_empty) {
PSCardTable* card_table = ParallelScavengeHeap::heap()->card_table();
card_table->pre_scavenge(active_workers);
diff --git a/src/hotspot/share/gc/serial/cardTableRS.cpp b/src/hotspot/share/gc/serial/cardTableRS.cpp
index c4af2c8edb15c..87c0eba819298 100644
--- a/src/hotspot/share/gc/serial/cardTableRS.cpp
+++ b/src/hotspot/share/gc/serial/cardTableRS.cpp
@@ -26,37 +26,29 @@
#include "classfile/classLoaderDataGraph.hpp"
#include "gc/serial/cardTableRS.hpp"
#include "gc/serial/generation.hpp"
-#include "gc/serial/serialHeap.hpp"
+#include "gc/serial/serialHeap.inline.hpp"
#include "gc/shared/space.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "utilities/align.hpp"
-void CardTableRS::younger_refs_in_space_iterate(TenuredSpace* sp,
- OopIterateClosure* cl) {
- verify_used_region_at_save_marks(sp);
-
+void CardTableRS::scan_old_to_young_refs(TenuredSpace* sp) {
+ const MemRegion ur = sp->used_region();
const MemRegion urasm = sp->used_region_at_save_marks();
- if (!urasm.is_empty()) {
- non_clean_card_iterate(sp, urasm, cl, this);
- }
-}
-
-#ifdef ASSERT
-void CardTableRS::verify_used_region_at_save_marks(Space* sp) const {
- MemRegion ur = sp->used_region();
- MemRegion urasm = sp->used_region_at_save_marks();
assert(ur.contains(urasm),
"Did you forget to call save_marks()? "
"[" PTR_FORMAT ", " PTR_FORMAT ") is not contained in "
"[" PTR_FORMAT ", " PTR_FORMAT ")",
p2i(urasm.start()), p2i(urasm.end()), p2i(ur.start()), p2i(ur.end()));
-}
-#endif
-void CardTableRS::maintain_old_to_young_invariant(Generation* old_gen, bool is_young_gen_empty) {
- assert(SerialHeap::heap()->is_old_gen(old_gen), "precondition");
+ if (!urasm.is_empty()) {
+ OldGenScanClosure cl(SerialHeap::heap()->young_gen());
+ non_clean_card_iterate(sp, urasm, &cl);
+ }
+}
+void CardTableRS::maintain_old_to_young_invariant(TenuredGeneration* old_gen,
+ bool is_young_gen_empty) {
if (is_young_gen_empty) {
clear_MemRegion(old_gen->prev_used_region());
} else {
@@ -72,274 +64,59 @@ void CardTableRS::maintain_old_to_young_invariant(Generation* old_gen, bool is_y
}
}
-class VerifyCleanCardClosure: public BasicOopIterateClosure {
-private:
- HeapWord* _boundary;
- HeapWord* _begin;
- HeapWord* _end;
-protected:
+class SerialCheckForUnmarkedOops : public BasicOopIterateClosure {
+ DefNewGeneration* _young_gen;
+ CardTableRS* _card_table;
+ HeapWord* _unmarked_addr;
+
template void do_oop_work(T* p) {
- HeapWord* jp = (HeapWord*)p;
- assert(jp >= _begin && jp < _end,
- "Error: jp " PTR_FORMAT " should be within "
- "[_begin, _end) = [" PTR_FORMAT "," PTR_FORMAT ")",
- p2i(jp), p2i(_begin), p2i(_end));
oop obj = RawAccess<>::oop_load(p);
- guarantee(obj == nullptr || cast_from_oop(obj) >= _boundary,
- "pointer " PTR_FORMAT " at " PTR_FORMAT " on "
- "clean card crosses boundary" PTR_FORMAT,
- p2i(obj), p2i(jp), p2i(_boundary));
+ if (_young_gen->is_in_reserved(obj) &&
+ !_card_table->is_dirty_for_addr(p)) {
+ // Don't overwrite the first missing card mark
+ if (_unmarked_addr == nullptr) {
+ _unmarked_addr = (HeapWord*)p;
+ }
+ }
}
-public:
- VerifyCleanCardClosure(HeapWord* b, HeapWord* begin, HeapWord* end) :
- _boundary(b), _begin(begin), _end(end) {
- assert(b <= begin,
- "Error: boundary " PTR_FORMAT " should be at or below begin " PTR_FORMAT,
- p2i(b), p2i(begin));
- assert(begin <= end,
- "Error: begin " PTR_FORMAT " should be strictly below end " PTR_FORMAT,
- p2i(begin), p2i(end));
- }
+ public:
+ SerialCheckForUnmarkedOops(DefNewGeneration* young_gen, CardTableRS* card_table) :
+ _young_gen(young_gen),
+ _card_table(card_table),
+ _unmarked_addr(nullptr) {}
- virtual void do_oop(oop* p) { VerifyCleanCardClosure::do_oop_work(p); }
- virtual void do_oop(narrowOop* p) { VerifyCleanCardClosure::do_oop_work(p); }
-};
+ void do_oop(oop* p) override { do_oop_work(p); }
+ void do_oop(narrowOop* p) override { do_oop_work(p); }
-class VerifyCTSpaceClosure: public SpaceClosure {
-private:
- CardTableRS* _ct;
- HeapWord* _boundary;
-public:
- VerifyCTSpaceClosure(CardTableRS* ct, HeapWord* boundary) :
- _ct(ct), _boundary(boundary) {}
- virtual void do_space(Space* s) { _ct->verify_space(s, _boundary); }
+ bool has_unmarked_oop() {
+ return _unmarked_addr != nullptr;
+ }
};
-class VerifyCTGenClosure: public SerialHeap::GenClosure {
- CardTableRS* _ct;
-public:
- VerifyCTGenClosure(CardTableRS* ct) : _ct(ct) {}
- void do_generation(Generation* gen) {
- // Skip the youngest generation.
- if (SerialHeap::heap()->is_young_gen(gen)) {
- return;
+void CardTableRS::verify() {
+ class CheckForUnmarkedObjects : public ObjectClosure {
+ DefNewGeneration* _young_gen;
+ CardTableRS* _card_table;
+
+ public:
+ CheckForUnmarkedObjects() {
+ SerialHeap* heap = SerialHeap::heap();
+ _young_gen = heap->young_gen();
+ _card_table = heap->rem_set();
}
- // Normally, we're interested in pointers to younger generations.
- VerifyCTSpaceClosure blk(_ct, gen->reserved().start());
- gen->space_iterate(&blk, true);
- }
-};
-void CardTableRS::verify_space(Space* s, HeapWord* gen_boundary) {
- // We don't need to do young-gen spaces.
- if (s->end() <= gen_boundary) return;
- MemRegion used = s->used_region();
-
- CardValue* cur_entry = byte_for(used.start());
- CardValue* limit = byte_after(used.last());
- while (cur_entry < limit) {
- if (*cur_entry == clean_card_val()) {
- CardValue* first_dirty = cur_entry+1;
- while (first_dirty < limit &&
- *first_dirty == clean_card_val()) {
- first_dirty++;
+ void do_object(oop obj) override {
+ SerialCheckForUnmarkedOops object_check(_young_gen, _card_table);
+ obj->oop_iterate(&object_check);
+ // If this obj is imprecisely-marked, the card for obj-start must be dirty.
+ if (object_check.has_unmarked_oop()) {
+ guarantee(_card_table->is_dirty_for_addr(obj), "Found unmarked old-to-young pointer");
}
- // If the first object is a regular object, and it has a
- // young-to-old field, that would mark the previous card.
- HeapWord* boundary = addr_for(cur_entry);
- HeapWord* end = (first_dirty >= limit) ? used.end() : addr_for(first_dirty);
- HeapWord* boundary_block = s->block_start(boundary);
- HeapWord* begin = boundary; // Until proven otherwise.
- HeapWord* start_block = boundary_block; // Until proven otherwise.
- if (boundary_block < boundary) {
- if (s->block_is_obj(boundary_block) && s->obj_is_alive(boundary_block)) {
- oop boundary_obj = cast_to_oop(boundary_block);
- if (!boundary_obj->is_objArray() &&
- !boundary_obj->is_typeArray()) {
- guarantee(cur_entry > byte_for(used.start()),
- "else boundary would be boundary_block");
- if (*byte_for(boundary_block) != clean_card_val()) {
- begin = boundary_block + s->block_size(boundary_block);
- start_block = begin;
- }
- }
- }
- }
- // Now traverse objects until end.
- if (begin < end) {
- MemRegion mr(begin, end);
- VerifyCleanCardClosure verify_blk(gen_boundary, begin, end);
- for (HeapWord* cur = start_block; cur < end; cur += s->block_size(cur)) {
- if (s->block_is_obj(cur) && s->obj_is_alive(cur)) {
- cast_to_oop(cur)->oop_iterate(&verify_blk, mr);
- }
- }
- }
- cur_entry = first_dirty;
- } else {
- // We'd normally expect that cur_youngergen_and_prev_nonclean_card
- // is a transient value, that cannot be in the card table
- // except during GC, and thus assert that:
- // guarantee(*cur_entry != cur_youngergen_and_prev_nonclean_card,
- // "Illegal CT value");
- // That however, need not hold, as will become clear in the
- // following...
-
- // We'd normally expect that if we are in the parallel case,
- // we can't have left a prev value (which would be different
- // from the current value) in the card table, and so we'd like to
- // assert that:
- // guarantee(cur_youngergen_card_val() == youngergen_card
- // || !is_prev_youngergen_card_val(*cur_entry),
- // "Illegal CT value");
- // That, however, may not hold occasionally, because of
- // CMS or MSC in the old gen. To wit, consider the
- // following two simple illustrative scenarios:
- // (a) CMS: Consider the case where a large object L
- // spanning several cards is allocated in the old
- // gen, and has a young gen reference stored in it, dirtying
- // some interior cards. A young collection scans the card,
- // finds a young ref and installs a youngergenP_n value.
- // L then goes dead. Now a CMS collection starts,
- // finds L dead and sweeps it up. Assume that L is
- // abutting _unallocated_blk, so _unallocated_blk is
- // adjusted down to (below) L. Assume further that
- // no young collection intervenes during this CMS cycle.
- // The next young gen cycle will not get to look at this
- // youngergenP_n card since it lies in the unoccupied
- // part of the space.
- // Some young collections later the blocks on this
- // card can be re-allocated either due to direct allocation
- // or due to absorbing promotions. At this time, the
- // before-gc verification will fail the above assert.
- // (b) MSC: In this case, an object L with a young reference
- // is on a card that (therefore) holds a youngergen_n value.
- // Suppose also that L lies towards the end of the used
- // the used space before GC. An MSC collection
- // occurs that compacts to such an extent that this
- // card is no longer in the occupied part of the space.
- // Since current code in MSC does not always clear cards
- // in the unused part of old gen, this stale youngergen_n
- // value is left behind and can later be covered by
- // an object when promotion or direct allocation
- // re-allocates that part of the heap.
- //
- // Fortunately, the presence of such stale card values is
- // "only" a minor annoyance in that subsequent young collections
- // might needlessly scan such cards, but would still never corrupt
- // the heap as a result. However, it's likely not to be a significant
- // performance inhibitor in practice. For instance,
- // some recent measurements with unoccupied cards eagerly cleared
- // out to maintain this invariant, showed next to no
- // change in young collection times; of course one can construct
- // degenerate examples where the cost can be significant.)
- // Note, in particular, that if the "stale" card is modified
- // after re-allocation, it would be dirty, not "stale". Thus,
- // we can never have a younger ref in such a card and it is
- // safe not to scan that card in any collection. [As we see
- // below, we do some unnecessary scanning
- // in some cases in the current parallel scanning algorithm.]
- //
- // The main point below is that the parallel card scanning code
- // deals correctly with these stale card values. There are two main
- // cases to consider where we have a stale "young gen" value and a
- // "derivative" case to consider, where we have a stale
- // "cur_younger_gen_and_prev_non_clean" value, as will become
- // apparent in the case analysis below.
- // o Case 1. If the stale value corresponds to a younger_gen_n
- // value other than the cur_younger_gen value then the code
- // treats this as being tantamount to a prev_younger_gen
- // card. This means that the card may be unnecessarily scanned.
- // There are two sub-cases to consider:
- // o Case 1a. Let us say that the card is in the occupied part
- // of the generation at the time the collection begins. In
- // that case the card will be either cleared when it is scanned
- // for young pointers, or will be set to cur_younger_gen as a
- // result of promotion. (We have elided the normal case where
- // the scanning thread and the promoting thread interleave
- // possibly resulting in a transient
- // cur_younger_gen_and_prev_non_clean value before settling
- // to cur_younger_gen. [End Case 1a.]
- // o Case 1b. Consider now the case when the card is in the unoccupied
- // part of the space which becomes occupied because of promotions
- // into it during the current young GC. In this case the card
- // will never be scanned for young references. The current
- // code will set the card value to either
- // cur_younger_gen_and_prev_non_clean or leave
- // it with its stale value -- because the promotions didn't
- // result in any younger refs on that card. Of these two
- // cases, the latter will be covered in Case 1a during
- // a subsequent scan. To deal with the former case, we need
- // to further consider how we deal with a stale value of
- // cur_younger_gen_and_prev_non_clean in our case analysis
- // below. This we do in Case 3 below. [End Case 1b]
- // [End Case 1]
- // o Case 2. If the stale value corresponds to cur_younger_gen being
- // a value not necessarily written by a current promotion, the
- // card will not be scanned by the younger refs scanning code.
- // (This is OK since as we argued above such cards cannot contain
- // any younger refs.) The result is that this value will be
- // treated as a prev_younger_gen value in a subsequent collection,
- // which is addressed in Case 1 above. [End Case 2]
- // o Case 3. We here consider the "derivative" case from Case 1b. above
- // because of which we may find a stale
- // cur_younger_gen_and_prev_non_clean card value in the table.
- // Once again, as in Case 1, we consider two subcases, depending
- // on whether the card lies in the occupied or unoccupied part
- // of the space at the start of the young collection.
- // o Case 3a. Let us say the card is in the occupied part of
- // the old gen at the start of the young collection. In that
- // case, the card will be scanned by the younger refs scanning
- // code which will set it to cur_younger_gen. In a subsequent
- // scan, the card will be considered again and get its final
- // correct value. [End Case 3a]
- // o Case 3b. Now consider the case where the card is in the
- // unoccupied part of the old gen, and is occupied as a result
- // of promotions during thus young gc. In that case,
- // the card will not be scanned for younger refs. The presence
- // of newly promoted objects on the card will then result in
- // its keeping the value cur_younger_gen_and_prev_non_clean
- // value, which we have dealt with in Case 3 here. [End Case 3b]
- // [End Case 3]
- //
- // (Please refer to the code in the helper class
- // ClearNonCleanCardWrapper and in CardTable for details.)
- //
- // The informal arguments above can be tightened into a formal
- // correctness proof and it behooves us to write up such a proof,
- // or to use model checking to prove that there are no lingering
- // concerns.
- //
- // Clearly because of Case 3b one cannot bound the time for
- // which a card will retain what we have called a "stale" value.
- // However, one can obtain a Loose upper bound on the redundant
- // work as a result of such stale values. Note first that any
- // time a stale card lies in the occupied part of the space at
- // the start of the collection, it is scanned by younger refs
- // code and we can define a rank function on card values that
- // declines when this is so. Note also that when a card does not
- // lie in the occupied part of the space at the beginning of a
- // young collection, its rank can either decline or stay unchanged.
- // In this case, no extra work is done in terms of redundant
- // younger refs scanning of that card.
- // Then, the case analysis above reveals that, in the worst case,
- // any such stale card will be scanned unnecessarily at most twice.
- //
- // It is nonetheless advisable to try and get rid of some of this
- // redundant work in a subsequent (low priority) re-design of
- // the card-scanning code, if only to simplify the underlying
- // state machine analysis/proof. ysr 1/28/2002. XXX
- cur_entry++;
}
- }
-}
+ } check;
-void CardTableRS::verify() {
- // At present, we only know how to verify the card table RS for
- // generational heaps.
- VerifyCTGenClosure blk(this);
- SerialHeap::heap()->generation_iterate(&blk, false);
+ SerialHeap::heap()->old_gen()->object_iterate(&check);
}
CardTableRS::CardTableRS(MemRegion whole_heap) :
@@ -391,7 +168,6 @@ CardTable::CardValue* CardTableRS::find_first_dirty_card(CardValue* const start_
template
CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_card,
CardValue* const end_card,
- CardTableRS* ct,
Func& object_start) {
for (CardValue* current_card = start_card; current_card < end_card; /* empty */) {
if (is_dirty(current_card)) {
@@ -400,7 +176,7 @@ CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_
}
// A potential candidate.
- HeapWord* addr = ct->addr_for(current_card);
+ HeapWord* addr = addr_for(current_card);
HeapWord* obj_start_addr = object_start(addr);
if (obj_start_addr == addr) {
@@ -416,7 +192,7 @@ CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_
}
// Final card occupied by obj.
- CardValue* obj_final_card = ct->byte_for(obj_start_addr + obj->size() - 1);
+ CardValue* obj_final_card = byte_for(obj_start_addr + obj->size() - 1);
if (is_clean(obj_final_card)) {
return obj_final_card;
}
@@ -440,7 +216,7 @@ static void prefetch_write(void *p) {
}
static void scan_obj_with_limit(oop obj,
- OopIterateClosure* cl,
+ OldGenScanClosure* cl,
HeapWord* start,
HeapWord* end) {
if (!obj->is_typeArray()) {
@@ -451,8 +227,7 @@ static void scan_obj_with_limit(oop obj,
void CardTableRS::non_clean_card_iterate(TenuredSpace* sp,
MemRegion mr,
- OopIterateClosure* cl,
- CardTableRS* ct) {
+ OldGenScanClosure* cl) {
struct {
HeapWord* start_addr;
HeapWord* end_addr;
@@ -471,8 +246,8 @@ void CardTableRS::non_clean_card_iterate(TenuredSpace* sp,
return result;
};
- CardValue* const start_card = ct->byte_for(mr.start());
- CardValue* const end_card = ct->byte_for(mr.last()) + 1;
+ CardValue* const start_card = byte_for(mr.start());
+ CardValue* const end_card = byte_for(mr.last()) + 1;
// if mr.end() is not card-aligned, that final card should not be cleared
// because it can be annotated dirty due to old-to-young pointers in
@@ -482,22 +257,20 @@ void CardTableRS::non_clean_card_iterate(TenuredSpace* sp,
for (CardValue* current_card = start_card; current_card < end_card; /* empty */) {
CardValue* const dirty_l = find_first_dirty_card(current_card, end_card);
-
if (dirty_l == end_card) {
// No dirty cards to iterate.
return;
}
- HeapWord* const addr_l = ct->addr_for(dirty_l);
+ HeapWord* const addr_l = addr_for(dirty_l);
HeapWord* obj_addr = object_start(addr_l);
CardValue* const dirty_r = find_first_clean_card(dirty_l + 1,
end_card,
- ct,
object_start);
assert(dirty_l < dirty_r, "inv");
HeapWord* const addr_r = dirty_r == end_card ? mr.end()
- : ct->addr_for(dirty_r);
+ : addr_for(dirty_r);
clear_cards(MIN2(dirty_l, clear_limit_card),
MIN2(dirty_r, clear_limit_card));
diff --git a/src/hotspot/share/gc/serial/cardTableRS.hpp b/src/hotspot/share/gc/serial/cardTableRS.hpp
index a4f333803a0ce..7ad6f8aef0b13 100644
--- a/src/hotspot/share/gc/serial/cardTableRS.hpp
+++ b/src/hotspot/share/gc/serial/cardTableRS.hpp
@@ -29,8 +29,8 @@
#include "memory/memRegion.hpp"
#include "oops/oop.hpp"
-class Generation;
-class Space;
+class OldGenScanClosure;
+class TenuredGeneration;
class TenuredSpace;
// This RemSet uses a card table both as shared data structure
@@ -38,10 +38,6 @@ class TenuredSpace;
class CardTableRS : public CardTable {
friend class VMStructs;
- // Below are private classes used in impl.
- friend class VerifyCTSpaceClosure;
-
- void verify_space(Space* s, HeapWord* gen_start);
static bool is_dirty(const CardValue* const v) {
return !is_clean(v);
@@ -59,36 +55,37 @@ class CardTableRS : public CardTable {
template
CardValue* find_first_clean_card(CardValue* start_card,
CardValue* end_card,
- CardTableRS* ct,
Func& object_start);
public:
CardTableRS(MemRegion whole_heap);
- void younger_refs_in_space_iterate(TenuredSpace* sp, OopIterateClosure* cl);
-
- virtual void verify_used_region_at_save_marks(Space* sp) const NOT_DEBUG_RETURN;
+ void scan_old_to_young_refs(TenuredSpace* sp);
void inline_write_ref_field_gc(void* field) {
CardValue* byte = byte_for(field);
*byte = dirty_card_val();
}
+ bool is_dirty_for_addr(const void* p) const {
+ CardValue* card = byte_for(p);
+ return is_dirty(card);
+ }
+
void verify();
// Update old gen cards to maintain old-to-young-pointer invariant: Clear
// the old generation card table completely if the young generation had been
// completely evacuated, otherwise dirties the whole old generation to
// conservatively not loose any old-to-young pointer.
- void maintain_old_to_young_invariant(Generation* old_gen, bool is_young_gen_empty);
+ void maintain_old_to_young_invariant(TenuredGeneration* old_gen, bool is_young_gen_empty);
// Iterate over the portion of the card-table which covers the given
// region mr in the given space and apply cl to any dirty sub-regions
// of mr. Clears the dirty cards as they are processed.
void non_clean_card_iterate(TenuredSpace* sp,
MemRegion mr,
- OopIterateClosure* cl,
- CardTableRS* ct);
+ OldGenScanClosure* cl);
bool is_in_young(const void* p) const override;
};
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp
index b569fbac48f0f..3f1cc7c6e4f93 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.cpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp
@@ -61,91 +61,6 @@
#include "utilities/globalDefinitions.hpp"
#include "utilities/stack.inline.hpp"
-class ScavengeHelper {
- DefNewGeneration* _young_gen;
- HeapWord* _young_gen_end;
-public:
- ScavengeHelper(DefNewGeneration* young_gen) :
- _young_gen(young_gen),
- _young_gen_end(young_gen->reserved().end()) {}
-
- bool is_in_young_gen(void* p) const {
- return p < _young_gen_end;
- }
-
- template
- void try_scavenge(T* p, Func&& f) {
- T heap_oop = RawAccess<>::oop_load(p);
- // Should we copy the obj?
- if (!CompressedOops::is_null(heap_oop)) {
- oop obj = CompressedOops::decode_not_null(heap_oop);
- if (is_in_young_gen(obj)) {
- assert(!_young_gen->to()->is_in_reserved(obj), "Scanning field twice?");
- oop new_obj = obj->is_forwarded() ? obj->forwardee()
- : _young_gen->copy_to_survivor_space(obj);
- RawAccess::oop_store(p, new_obj);
-
- // callback
- f(new_obj);
- }
- }
- }
-};
-
-class InHeapScanClosure : public BasicOopIterateClosure {
- ScavengeHelper _helper;
-protected:
- bool is_in_young_gen(void* p) const {
- return _helper.is_in_young_gen(p);
- }
-
- template
- void try_scavenge(T* p, Func&& f) {
- _helper.try_scavenge(p, f);
- }
-
- InHeapScanClosure(DefNewGeneration* young_gen) :
- BasicOopIterateClosure(young_gen->ref_processor()),
- _helper(young_gen) {}
-};
-
-class OffHeapScanClosure : public OopClosure {
- ScavengeHelper _helper;
-protected:
- bool is_in_young_gen(void* p) const {
- return _helper.is_in_young_gen(p);
- }
-
- template
- void try_scavenge(T* p, Func&& f) {
- _helper.try_scavenge(p, f);
- }
-
- OffHeapScanClosure(DefNewGeneration* young_gen) : _helper(young_gen) {}
-};
-
-class OldGenScanClosure : public InHeapScanClosure {
- CardTableRS* _rs;
-
- template
- void do_oop_work(T* p) {
- assert(!is_in_young_gen(p), "precondition");
-
- try_scavenge(p, [&] (oop new_obj) {
- // If p points to a younger generation, mark the card.
- if (is_in_young_gen(new_obj)) {
- _rs->inline_write_ref_field_gc(p);
- }
- });
- }
-public:
- OldGenScanClosure(DefNewGeneration* g) : InHeapScanClosure(g),
- _rs(SerialHeap::heap()->rem_set()) {}
-
- void do_oop(oop* p) { do_oop_work(p); }
- void do_oop(narrowOop* p) { do_oop_work(p); }
-};
-
class PromoteFailureClosure : public InHeapScanClosure {
template
void do_oop_work(T* p) {
@@ -420,9 +335,9 @@ void DefNewGeneration::compute_space_boundaries(uintx minimum_eden_size,
char *to_end = to_start + survivor_size;
assert(to_end == _virtual_space.high(), "just checking");
- assert(Space::is_aligned(eden_start), "checking alignment");
- assert(Space::is_aligned(from_start), "checking alignment");
- assert(Space::is_aligned(to_start), "checking alignment");
+ assert(is_aligned(eden_start, SpaceAlignment), "checking alignment");
+ assert(is_aligned(from_start, SpaceAlignment), "checking alignment");
+ assert(is_aligned(to_start, SpaceAlignment), "checking alignment");
MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)from_start);
MemRegion fromMR((HeapWord*)from_start, (HeapWord*)to_start);
@@ -649,6 +564,12 @@ size_t DefNewGeneration::max_capacity() const {
return reserved_bytes - compute_survivor_size(reserved_bytes, SpaceAlignment);
}
+bool DefNewGeneration::is_in(const void* p) const {
+ return eden()->is_in(p)
+ || from()->is_in(p)
+ || to() ->is_in(p);
+}
+
size_t DefNewGeneration::unsafe_max_alloc_nogc() const {
return eden()->free();
}
@@ -667,12 +588,15 @@ void DefNewGeneration::object_iterate(ObjectClosure* blk) {
from()->object_iterate(blk);
}
-
-void DefNewGeneration::space_iterate(SpaceClosure* blk,
- bool usedOnly) {
- blk->do_space(eden());
- blk->do_space(from());
- blk->do_space(to());
+HeapWord* DefNewGeneration::block_start(const void* p) const {
+ if (eden()->is_in_reserved(p)) {
+ return eden()->block_start_const(p);
+ }
+ if (from()->is_in_reserved(p)) {
+ return from()->block_start_const(p);
+ }
+ assert(to()->is_in_reserved(p), "inv");
+ return to()->block_start_const(p);
}
// The last collection bailed out, we are running out of heap space,
@@ -776,11 +700,19 @@ void DefNewGeneration::collect(bool full,
{
StrongRootsScope srs(0);
RootScanClosure root_cl{this};
- CLDScanClosure cld_scan_closure{this};
+ CLDScanClosure cld_cl{this};
+
+ MarkingCodeBlobClosure code_cl(&root_cl,
+ CodeBlobToOopClosure::FixRelocations,
+ false /* keepalive_nmethods */);
- heap->young_process_roots(&root_cl,
- &old_gen_cl,
- &cld_scan_closure);
+ heap->process_roots(SerialHeap::SO_ScavengeCodeCache,
+ &root_cl,
+ &cld_cl,
+ &cld_cl,
+ &code_cl);
+
+ _old_gen->scan_old_to_young_refs();
}
// "evacuate followers".
@@ -930,6 +862,9 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) {
handle_promotion_failure(old);
return old;
}
+
+ ContinuationGCSupport::transform_stack_chunk(obj);
+
new_obj_is_tenured = true;
} else {
// Prefetch beyond obj
@@ -1104,11 +1039,6 @@ const char* DefNewGeneration::name() const {
return "def new generation";
}
-// Moved from inline file as they are not called inline
-ContiguousSpace* DefNewGeneration::first_compaction_space() const {
- return eden();
-}
-
HeapWord* DefNewGeneration::allocate(size_t word_size, bool is_tlab) {
// This is the slow-path allocation for the DefNewGeneration.
// Most allocations are fast-path in compiled code.
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.hpp b/src/hotspot/share/gc/serial/defNewGeneration.hpp
index e9586993e547b..5fcfa5813adec 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.hpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
#include "gc/serial/cSpaceCounters.hpp"
#include "gc/serial/generation.hpp"
+#include "gc/serial/tenuredGeneration.hpp"
#include "gc/shared/ageTable.hpp"
#include "gc/shared/copyFailedInfo.hpp"
#include "gc/shared/gc_globals.hpp"
@@ -52,7 +53,8 @@ class STWGCTimer;
class DefNewGeneration: public Generation {
friend class VMStructs;
- Generation* _old_gen;
+ TenuredGeneration* _old_gen;
+
uint _tenuring_threshold; // Tenuring threshold for next collection.
AgeTable _age_table;
// Size of object to pretenure in words; command line provides bytes
@@ -157,8 +159,6 @@ class DefNewGeneration: public Generation {
size_t max_byte_size,
const char* policy="Serial young collection pauses");
- virtual Generation::Name kind() { return Generation::DefNew; }
-
// allocate and initialize ("weak") refs processing support
void ref_processor_init();
ReferenceProcessor* ref_processor() { return _ref_processor; }
@@ -168,15 +168,23 @@ class DefNewGeneration: public Generation {
ContiguousSpace* from() const { return _from_space; }
ContiguousSpace* to() const { return _to_space; }
- virtual ContiguousSpace* first_compaction_space() const;
-
// Space enquiries
size_t capacity() const;
size_t used() const;
size_t free() const;
size_t max_capacity() const;
size_t capacity_before_gc() const;
+
+ // Returns "TRUE" iff "p" points into the used areas in each space of young-gen.
+ bool is_in(const void* p) const;
+
+ // Return an estimate of the maximum allocation that could be performed
+ // in the generation without triggering any collection or expansion
+ // activity. It is "unsafe" because no locks are taken; the result
+ // should be treated as an approximation, not a guarantee, for use in
+ // heuristic resizing decisions.
size_t unsafe_max_alloc_nogc() const;
+
size_t contiguous_available() const;
size_t max_eden_size() const { return _max_eden_size; }
@@ -193,14 +201,11 @@ class DefNewGeneration: public Generation {
// Return true if the expansion was successful.
bool expand(size_t bytes);
- // DefNewGeneration cannot currently expand except at
- // a GC.
- virtual bool is_maximal_no_gc() const { return true; }
// Iteration
void object_iterate(ObjectClosure* blk);
- void space_iterate(SpaceClosure* blk, bool usedOnly = false);
+ HeapWord* block_start(const void* p) const;
// Allocation support
virtual bool should_allocate(size_t word_size, bool is_tlab) {
@@ -226,10 +231,10 @@ class DefNewGeneration: public Generation {
HeapWord* par_allocate(size_t word_size, bool is_tlab);
- virtual void gc_epilogue(bool full);
+ void gc_epilogue(bool full);
// Save the tops for eden, from, and to
- virtual void record_spaces_top();
+ void record_spaces_top();
// Accessing marks
void save_marks();
@@ -250,14 +255,14 @@ class DefNewGeneration: public Generation {
void reset_scratch();
// GC support
- virtual void compute_new_size();
+ void compute_new_size();
// Returns true if the collection is likely to be safely
// completed. Even if this method returns true, a collection
// may not be guaranteed to succeed, and the system should be
// able to safely unwind and recover from that failure, albeit
- // at some additional cost. Override superclass's implementation.
- virtual bool collection_attempt_is_safe();
+ // at some additional cost.
+ bool collection_attempt_is_safe();
virtual void collect(bool full,
bool clear_all_soft_refs,
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.inline.hpp b/src/hotspot/share/gc/serial/defNewGeneration.inline.hpp
index 1f0a5df78b1cd..808e04a300261 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.inline.hpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.inline.hpp
@@ -41,7 +41,6 @@ void DefNewGeneration::oop_since_save_marks_iterate(OopClosureType* cl) {
assert(from()->saved_mark_at_top(), "inv");
to()->oop_since_save_marks_iterate(cl);
- to()->set_saved_mark();
}
#endif // SHARE_GC_SERIAL_DEFNEWGENERATION_INLINE_HPP
diff --git a/src/hotspot/share/gc/serial/genMarkSweep.cpp b/src/hotspot/share/gc/serial/genMarkSweep.cpp
index 28ba4615f73a2..d0594a3a50037 100644
--- a/src/hotspot/share/gc/serial/genMarkSweep.cpp
+++ b/src/hotspot/share/gc/serial/genMarkSweep.cpp
@@ -30,7 +30,6 @@
#include "classfile/systemDictionary.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "compiler/oopMap.hpp"
#include "gc/serial/cardTableRS.hpp"
#include "gc/serial/defNewGeneration.hpp"
@@ -441,10 +440,9 @@ void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
// Increment the invocation count
_total_invocations++;
- // Capture used regions for each generation that will be
- // subject to collection, so that card table adjustments can
- // be made intelligently (see clear / invalidate further below).
- gch->save_used_regions();
+ // Capture used regions for old-gen to reestablish old-to-young invariant
+ // after full-gc.
+ gch->old_gen()->save_used_region();
allocate_stacks();
diff --git a/src/hotspot/share/gc/serial/generation.cpp b/src/hotspot/share/gc/serial/generation.cpp
index 302f8f231e7be..c1c195aaf38bd 100644
--- a/src/hotspot/share/gc/serial/generation.cpp
+++ b/src/hotspot/share/gc/serial/generation.cpp
@@ -27,7 +27,6 @@
#include "gc/serial/generation.hpp"
#include "gc/serial/serialHeap.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
-#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shared/gcLocker.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTrace.hpp"
@@ -83,26 +82,6 @@ void Generation::print_summary_info_on(outputStream* st) {
sr->invocations > 0 ? time / sr->invocations : 0.0);
}
-// Utility iterator classes
-
-class GenerationIsInClosure : public SpaceClosure {
- public:
- const void* _p;
- Space* sp;
- virtual void do_space(Space* s) {
- if (sp == nullptr) {
- if (s->is_in(_p)) sp = s;
- }
- }
- GenerationIsInClosure(const void* p) : _p(p), sp(nullptr) {}
-};
-
-bool Generation::is_in(const void* p) const {
- GenerationIsInClosure blk(p);
- ((Generation*)this)->space_iterate(&blk);
- return blk.sp != nullptr;
-}
-
size_t Generation::max_contiguous_available() const {
// The largest number of contiguous free words in this or any higher generation.
size_t avail = contiguous_available();
@@ -112,97 +91,3 @@ size_t Generation::max_contiguous_available() const {
}
return MAX2(avail, old_avail);
}
-
-bool Generation::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
- size_t available = max_contiguous_available();
- bool res = (available >= max_promotion_in_bytes);
- log_trace(gc)("Generation: promo attempt is%s safe: available(" SIZE_FORMAT ") %s max_promo(" SIZE_FORMAT ")",
- res? "":" not", available, res? ">=":"<", max_promotion_in_bytes);
- return res;
-}
-
-// Ignores "ref" and calls allocate().
-oop Generation::promote(oop obj, size_t obj_size) {
- assert(obj_size == obj->size(), "bad obj_size passed in");
-
-#ifndef PRODUCT
- if (SerialHeap::heap()->promotion_should_fail()) {
- return nullptr;
- }
-#endif // #ifndef PRODUCT
-
- // Allocate new object.
- HeapWord* result = allocate(obj_size, false);
- if (result == nullptr) {
- // Promotion of obj into gen failed. Try to expand and allocate.
- result = expand_and_allocate(obj_size, false);
- if (result == nullptr) {
- return nullptr;
- }
- }
-
- // Copy to new location.
- Copy::aligned_disjoint_words(cast_from_oop(obj), result, obj_size);
- oop new_obj = cast_to_oop(result);
-
- // Transform object if it is a stack chunk.
- ContinuationGCSupport::transform_stack_chunk(new_obj);
-
- return new_obj;
-}
-
-// Some of these are mediocre general implementations. Should be
-// overridden to get better performance.
-
-class GenerationBlockStartClosure : public SpaceClosure {
- public:
- const void* _p;
- HeapWord* _start;
- virtual void do_space(Space* s) {
- if (_start == nullptr && s->is_in_reserved(_p)) {
- _start = s->block_start(_p);
- }
- }
- GenerationBlockStartClosure(const void* p) { _p = p; _start = nullptr; }
-};
-
-HeapWord* Generation::block_start(const void* p) const {
- GenerationBlockStartClosure blk(p);
- // Cast away const
- ((Generation*)this)->space_iterate(&blk);
- return blk._start;
-}
-
-class GenerationBlockIsObjClosure : public SpaceClosure {
- public:
- const HeapWord* _p;
- bool is_obj;
- virtual void do_space(Space* s) {
- if (!is_obj && s->is_in_reserved(_p)) {
- is_obj |= s->block_is_obj(_p);
- }
- }
- GenerationBlockIsObjClosure(const HeapWord* p) { _p = p; is_obj = false; }
-};
-
-bool Generation::block_is_obj(const HeapWord* p) const {
- GenerationBlockIsObjClosure blk(p);
- // Cast away const
- ((Generation*)this)->space_iterate(&blk);
- return blk.is_obj;
-}
-
-class GenerationObjIterateClosure : public SpaceClosure {
- private:
- ObjectClosure* _cl;
- public:
- virtual void do_space(Space* s) {
- s->object_iterate(_cl);
- }
- GenerationObjIterateClosure(ObjectClosure* cl) : _cl(cl) {}
-};
-
-void Generation::object_iterate(ObjectClosure* cl) {
- GenerationObjIterateClosure blk(cl);
- space_iterate(&blk);
-}
diff --git a/src/hotspot/share/gc/serial/generation.hpp b/src/hotspot/share/gc/serial/generation.hpp
index d93c2b66df717..114ab3d63120b 100644
--- a/src/hotspot/share/gc/serial/generation.hpp
+++ b/src/hotspot/share/gc/serial/generation.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_SHARED_GENERATION_HPP
-#define SHARE_GC_SHARED_GENERATION_HPP
+#ifndef SHARE_GC_SERIAL_GENERATION_HPP
+#define SHARE_GC_SERIAL_GENERATION_HPP
#include "gc/shared/collectorCounters.hpp"
#include "gc/shared/referenceProcessor.hpp"
@@ -55,22 +55,9 @@ class ContiguousSpace;
class OopClosure;
class GCStats;
-// A "ScratchBlock" represents a block of memory in one generation usable by
-// another. It represents "num_words" free words, starting at and including
-// the address of "this".
-struct ScratchBlock {
- ScratchBlock* next;
- size_t num_words;
- HeapWord scratch_space[1]; // Actually, of size "num_words-2" (assuming
- // first two fields are word-sized.)
-};
-
class Generation: public CHeapObj {
friend class VMStructs;
private:
- MemRegion _prev_used_region; // for collectors that want to "remember" a value for
- // used region at some specific point during collection.
-
GCMemoryManager* _gc_manager;
protected:
@@ -93,13 +80,6 @@ class Generation: public CHeapObj {
Generation(ReservedSpace rs, size_t initial_byte_size);
public:
- // The set of possible generation kinds.
- enum Name {
- DefNew,
- MarkSweepCompact,
- Other
- };
-
enum SomePublicConstants {
// Generations are GenGrain-aligned and have size that are multiples of
// GenGrain.
@@ -109,9 +89,6 @@ class Generation: public CHeapObj {
GenGrain = 1 << LogOfGenGrain
};
-
- virtual Generation::Name kind() { return Generation::Other; }
-
virtual size_t capacity() const = 0; // The maximum number of object bytes the
// generation can currently hold.
virtual size_t used() const = 0; // The number of used bytes in the gen.
@@ -122,63 +99,19 @@ class Generation: public CHeapObj {
// for the allocation of objects.
virtual size_t max_capacity() const;
- // If this is a young generation, the maximum number of bytes that can be
- // allocated in this generation before a GC is triggered.
- virtual size_t capacity_before_gc() const { return 0; }
-
// The largest number of contiguous free bytes in the generation,
// including expansion (Assumes called at a safepoint.)
virtual size_t contiguous_available() const = 0;
// The largest number of contiguous free bytes in this or any higher generation.
virtual size_t max_contiguous_available() const;
- // Returns true if promotions of the specified amount are
- // likely to succeed without a promotion failure.
- // Promotion of the full amount is not guaranteed but
- // might be attempted in the worst case.
- virtual bool promotion_attempt_is_safe(size_t max_promotion_in_bytes) const;
-
- // Return an estimate of the maximum allocation that could be performed
- // in the generation without triggering any collection or expansion
- // activity. It is "unsafe" because no locks are taken; the result
- // should be treated as an approximation, not a guarantee, for use in
- // heuristic resizing decisions.
- virtual size_t unsafe_max_alloc_nogc() const = 0;
-
- // Returns true if this generation cannot be expanded further
- // without a GC. Override as appropriate.
- virtual bool is_maximal_no_gc() const {
- return _virtual_space.uncommitted_size() == 0;
- }
-
MemRegion reserved() const { return _reserved; }
- // Returns a region guaranteed to contain all the objects in the
- // generation.
- virtual MemRegion used_region() const { return _reserved; }
-
- MemRegion prev_used_region() const { return _prev_used_region; }
- virtual void save_used_region() { _prev_used_region = used_region(); }
-
- // Returns "TRUE" iff "p" points into the committed areas in the generation.
- // For some kinds of generations, this may be an expensive operation.
- // To avoid performance problems stemming from its inadvertent use in
- // product jvm's, we restrict its use to assertion checking or
- // verification only.
- virtual bool is_in(const void* p) const;
-
/* Returns "TRUE" iff "p" points into the reserved area of the generation. */
bool is_in_reserved(const void* p) const {
return _reserved.contains(p);
}
- // Iteration - do not use for time critical operations
- virtual void space_iterate(SpaceClosure* blk, bool usedOnly = false) = 0;
-
- // Returns the first space, if any, in the generation that can participate
- // in compaction, or else "null".
- virtual ContiguousSpace* first_compaction_space() const = 0;
-
// Returns "true" iff this generation should be used to allocate an
// object of the given size. Young generations might
// wish to exclude very large objects, for example, since, if allocated
@@ -202,33 +135,12 @@ class Generation: public CHeapObj {
// Thread-local allocation buffers
virtual bool supports_tlab_allocation() const { return false; }
- virtual size_t tlab_capacity() const {
- guarantee(false, "Generation doesn't support thread local allocation buffers");
- return 0;
- }
- virtual size_t tlab_used() const {
- guarantee(false, "Generation doesn't support thread local allocation buffers");
- return 0;
- }
- virtual size_t unsafe_max_tlab_alloc() const {
- guarantee(false, "Generation doesn't support thread local allocation buffers");
- return 0;
- }
-
- // "obj" is the address of an object in a younger generation. Allocate space
- // for "obj" in the current (or some higher) generation, and copy "obj" into
- // the newly allocated space, if possible, returning the result (or null if
- // the allocation failed).
- //
- // The "obj_size" argument is just obj->size(), passed along so the caller can
- // avoid repeating the virtual call to retrieve it.
- virtual oop promote(oop obj, size_t obj_size);
// Returns "true" iff collect() should subsequently be called on this
// this generation. See comment below.
// This is a generic implementation which can be overridden.
//
- // Note: in the current (1.4) implementation, when genCollectedHeap's
+ // Note: in the current (1.4) implementation, when serialHeap's
// incremental_collection_will_fail flag is set, all allocations are
// slow path (the only fast-path place to allocate is DefNew, which
// will be full if the flag is set).
@@ -240,16 +152,6 @@ class Generation: public CHeapObj {
return (full || should_allocate(word_size, is_tlab));
}
- // Returns true if the collection is likely to be safely
- // completed. Even if this method returns true, a collection
- // may not be guaranteed to succeed, and the system should be
- // able to safely unwind and recover from that failure, albeit
- // at some additional cost.
- virtual bool collection_attempt_is_safe() {
- guarantee(false, "Are you sure you want to call this method?");
- return true;
- }
-
// Perform a garbage collection.
// If full is true attempt a full garbage collection of this generation.
// Otherwise, attempting to (at least) free enough space to support an
@@ -266,17 +168,6 @@ class Generation: public CHeapObj {
// still unsuccessful, return "null".
virtual HeapWord* expand_and_allocate(size_t word_size, bool is_tlab) = 0;
- // Some generations may require some cleanup or preparation actions before
- // allowing a collection. The default is to do nothing.
- virtual void gc_prologue(bool full) {}
-
- // Some generations may require some cleanup actions after a collection.
- // The default is to do nothing.
- virtual void gc_epilogue(bool full) {}
-
- // Save the high water marks for the used space in a generation.
- virtual void record_spaces_top() {}
-
// Generations may keep statistics about collection. This method
// updates those statistics. current_generation is the generation
// that was most recently collected. This allows the generation to
@@ -286,45 +177,10 @@ class Generation: public CHeapObj {
GCStats* gc_stats() const { return _gc_stats; }
virtual void update_gc_stats(Generation* current_generation, bool full) {}
- // Accessing "marks".
-
- // This function gives a generation a chance to note a point between
- // collections. For example, a contiguous generation might note the
- // beginning allocation point post-collection, which might allow some later
- // operations to be optimized.
- virtual void save_marks() {}
-
- // This function is "true" iff any no allocations have occurred in the
- // generation since the last call to "save_marks".
- virtual bool no_allocs_since_save_marks() = 0;
-
- // When an older generation has been collected, and perhaps resized,
- // this method will be invoked on all younger generations (from older to
- // younger), allowing them to resize themselves as appropriate.
- virtual void compute_new_size() = 0;
-
// Printing
virtual const char* name() const = 0;
virtual const char* short_name() const = 0;
- // Iteration.
-
- // Iterate over all objects in the generation, calling "cl.do_object" on
- // each.
- virtual void object_iterate(ObjectClosure* cl);
-
- // Block abstraction.
-
- // Returns the address of the start of the "block" that contains the
- // address "addr". We say "blocks" instead of "object" since some heaps
- // may not pack objects densely; a chunk may either be an object or a
- // non-object.
- virtual HeapWord* block_start(const void* addr) const;
-
- // Requires "addr" to be the start of a block, and returns "TRUE" iff
- // the block is an object.
- virtual bool block_is_obj(const HeapWord* addr) const;
-
virtual void print() const;
virtual void print_on(outputStream* st) const;
diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp
index 9361a4e5eb961..2fd5dc1889aea 100644
--- a/src/hotspot/share/gc/serial/serialHeap.cpp
+++ b/src/hotspot/share/gc/serial/serialHeap.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,26 +23,80 @@
*/
#include "precompiled.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
+#include "classfile/stringTable.hpp"
+#include "classfile/symbolTable.hpp"
+#include "classfile/vmSymbols.hpp"
+#include "code/codeCache.hpp"
+#include "compiler/oopMap.hpp"
+#include "gc/serial/cardTableRS.hpp"
#include "gc/serial/defNewGeneration.inline.hpp"
+#include "gc/serial/genMarkSweep.hpp"
+#include "gc/serial/markSweep.hpp"
#include "gc/serial/serialHeap.hpp"
+#include "gc/serial/serialMemoryPools.hpp"
+#include "gc/serial/serialVMOperations.hpp"
#include "gc/serial/tenuredGeneration.inline.hpp"
+#include "gc/shared/cardTableBarrierSet.hpp"
+#include "gc/shared/classUnloadingContext.hpp"
+#include "gc/shared/collectedHeap.inline.hpp"
+#include "gc/shared/collectorCounters.hpp"
+#include "gc/shared/continuationGCSupport.inline.hpp"
+#include "gc/shared/gcId.hpp"
+#include "gc/shared/gcInitLogger.hpp"
#include "gc/shared/gcLocker.inline.hpp"
-#include "gc/shared/genMemoryPools.hpp"
+#include "gc/shared/gcPolicyCounters.hpp"
+#include "gc/shared/gcTrace.hpp"
+#include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/gcVMOperations.hpp"
+#include "gc/shared/genArguments.hpp"
+#include "gc/shared/locationPrinter.inline.hpp"
+#include "gc/shared/oopStorage.inline.hpp"
+#include "gc/shared/oopStorageParState.inline.hpp"
+#include "gc/shared/oopStorageSet.inline.hpp"
#include "gc/shared/scavengableNMethods.hpp"
+#include "gc/shared/space.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/shared/weakProcessor.hpp"
+#include "gc/shared/workerThread.hpp"
+#include "memory/iterator.hpp"
+#include "memory/metaspaceCounters.hpp"
+#include "memory/metaspaceUtils.hpp"
+#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/handles.hpp"
+#include "runtime/handles.inline.hpp"
+#include "runtime/java.hpp"
#include "runtime/mutexLocker.hpp"
+#include "runtime/threads.hpp"
+#include "runtime/vmThread.hpp"
#include "services/memoryManager.hpp"
+#include "services/memoryService.hpp"
+#include "utilities/autoRestore.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/formatBuffer.hpp"
+#include "utilities/macros.hpp"
+#include "utilities/stack.inline.hpp"
+#include "utilities/vmError.hpp"
+#if INCLUDE_JVMCI
+#include "jvmci/jvmci.hpp"
+#endif
SerialHeap* SerialHeap::heap() {
return named_heap(CollectedHeap::Serial);
}
SerialHeap::SerialHeap() :
- GenCollectedHeap(Generation::DefNew,
- Generation::MarkSweepCompact,
- "Copy:MSC"),
+ CollectedHeap(),
+ _young_gen(nullptr),
+ _old_gen(nullptr),
+ _rem_set(nullptr),
+ _gc_policy_counters(new GCPolicyCounters("Copy:MSC", 2, 2)),
+ _incremental_collection_failed(false),
+ _young_manager(nullptr),
+ _old_manager(nullptr),
_eden_pool(nullptr),
_survivor_pool(nullptr),
_old_pool(nullptr) {
@@ -51,7 +105,6 @@ SerialHeap::SerialHeap() :
}
void SerialHeap::initialize_serviceability() {
-
DefNewGeneration* young = young_gen();
// Add a memory pool for each space and young gen doesn't
@@ -65,7 +118,7 @@ void SerialHeap::initialize_serviceability() {
young->max_survivor_size(),
false /* support_usage_threshold */);
TenuredGeneration* old = old_gen();
- _old_pool = new GenerationPool(old, "Tenured Gen", true);
+ _old_pool = new TenuredGenerationPool(old, "Tenured Gen", true);
_young_manager->add_pool(_eden_pool);
_young_manager->add_pool(_survivor_pool);
@@ -75,7 +128,6 @@ void SerialHeap::initialize_serviceability() {
_old_manager->add_pool(_survivor_pool);
_old_manager->add_pool(_old_pool);
old->set_gc_manager(_old_manager);
-
}
GrowableArray SerialHeap::memory_managers() {
@@ -93,17 +145,6 @@ GrowableArray SerialHeap::memory_pools() {
return memory_pools;
}
-void SerialHeap::young_process_roots(OopClosure* root_closure,
- OopIterateClosure* old_gen_closure,
- CLDClosure* cld_closure) {
- MarkingCodeBlobClosure mark_code_closure(root_closure, CodeBlobToOopClosure::FixRelocations, false /* keepalive nmethods */);
-
- process_roots(SO_ScavengeCodeCache, root_closure,
- cld_closure, cld_closure, &mark_code_closure);
-
- old_gen()->younger_refs_iterate(old_gen_closure);
-}
-
void SerialHeap::safepoint_synchronize_begin() {
if (UseStringDeduplication) {
SuspendibleThreadSet::synchronize();
@@ -133,3 +174,878 @@ void SerialHeap::pin_object(JavaThread* thread, oop obj) {
void SerialHeap::unpin_object(JavaThread* thread, oop obj) {
GCLocker::unlock_critical(thread);
}
+
+jint SerialHeap::initialize() {
+ // Allocate space for the heap.
+
+ ReservedHeapSpace heap_rs = allocate(HeapAlignment);
+
+ if (!heap_rs.is_reserved()) {
+ vm_shutdown_during_initialization(
+ "Could not reserve enough space for object heap");
+ return JNI_ENOMEM;
+ }
+
+ initialize_reserved_region(heap_rs);
+
+ ReservedSpace young_rs = heap_rs.first_part(MaxNewSize);
+ ReservedSpace old_rs = heap_rs.last_part(MaxNewSize);
+
+ _rem_set = new CardTableRS(heap_rs.region());
+ _rem_set->initialize(young_rs.base(), old_rs.base());
+
+ CardTableBarrierSet *bs = new CardTableBarrierSet(_rem_set);
+ bs->initialize();
+ BarrierSet::set_barrier_set(bs);
+
+ _young_gen = new DefNewGeneration(young_rs, NewSize, MinNewSize, MaxNewSize);
+ _old_gen = new TenuredGeneration(old_rs, OldSize, MinOldSize, MaxOldSize, rem_set());
+
+ GCInitLogger::print();
+
+ return JNI_OK;
+}
+
+ReservedHeapSpace SerialHeap::allocate(size_t alignment) {
+ // Now figure out the total size.
+ const size_t pageSize = UseLargePages ? os::large_page_size() : os::vm_page_size();
+ assert(alignment % pageSize == 0, "Must be");
+
+ // Check for overflow.
+ size_t total_reserved = MaxNewSize + MaxOldSize;
+ if (total_reserved < MaxNewSize) {
+ vm_exit_during_initialization("The size of the object heap + VM data exceeds "
+ "the maximum representable size");
+ }
+ assert(total_reserved % alignment == 0,
+ "Gen size; total_reserved=" SIZE_FORMAT ", alignment="
+ SIZE_FORMAT, total_reserved, alignment);
+
+ ReservedHeapSpace heap_rs = Universe::reserve_heap(total_reserved, alignment);
+ size_t used_page_size = heap_rs.page_size();
+
+ os::trace_page_sizes("Heap",
+ MinHeapSize,
+ total_reserved,
+ heap_rs.base(),
+ heap_rs.size(),
+ used_page_size);
+
+ return heap_rs;
+}
+
+class GenIsScavengable : public BoolObjectClosure {
+public:
+ bool do_object_b(oop obj) {
+ return SerialHeap::heap()->is_in_young(obj);
+ }
+};
+
+static GenIsScavengable _is_scavengable;
+
+void SerialHeap::post_initialize() {
+ CollectedHeap::post_initialize();
+
+ DefNewGeneration* def_new_gen = (DefNewGeneration*)_young_gen;
+
+ def_new_gen->ref_processor_init();
+
+ MarkSweep::initialize();
+
+ ScavengableNMethods::initialize(&_is_scavengable);
+}
+
+PreGenGCValues SerialHeap::get_pre_gc_values() const {
+ const DefNewGeneration* const def_new_gen = (DefNewGeneration*) young_gen();
+
+ return PreGenGCValues(def_new_gen->used(),
+ def_new_gen->capacity(),
+ def_new_gen->eden()->used(),
+ def_new_gen->eden()->capacity(),
+ def_new_gen->from()->used(),
+ def_new_gen->from()->capacity(),
+ old_gen()->used(),
+ old_gen()->capacity());
+}
+
+size_t SerialHeap::capacity() const {
+ return _young_gen->capacity() + _old_gen->capacity();
+}
+
+size_t SerialHeap::used() const {
+ return _young_gen->used() + _old_gen->used();
+}
+
+size_t SerialHeap::max_capacity() const {
+ return _young_gen->max_capacity() + _old_gen->max_capacity();
+}
+
+// Return true if any of the following is true:
+// . the allocation won't fit into the current young gen heap
+// . gc locker is occupied (jni critical section)
+// . heap memory is tight -- the most recent previous collection
+// was a full collection because a partial collection (would
+// have) failed and is likely to fail again
+bool SerialHeap::should_try_older_generation_allocation(size_t word_size) const {
+ size_t young_capacity = _young_gen->capacity_before_gc();
+ return (word_size > heap_word_size(young_capacity))
+ || GCLocker::is_active_and_needs_gc()
+ || incremental_collection_failed();
+}
+
+HeapWord* SerialHeap::expand_heap_and_allocate(size_t size, bool is_tlab) {
+ HeapWord* result = nullptr;
+ if (_old_gen->should_allocate(size, is_tlab)) {
+ result = _old_gen->expand_and_allocate(size, is_tlab);
+ }
+ if (result == nullptr) {
+ if (_young_gen->should_allocate(size, is_tlab)) {
+ result = _young_gen->expand_and_allocate(size, is_tlab);
+ }
+ }
+ assert(result == nullptr || is_in_reserved(result), "result not in heap");
+ return result;
+}
+
+HeapWord* SerialHeap::mem_allocate_work(size_t size,
+ bool is_tlab) {
+
+ HeapWord* result = nullptr;
+
+ // Loop until the allocation is satisfied, or unsatisfied after GC.
+ for (uint try_count = 1, gclocker_stalled_count = 0; /* return or throw */; try_count += 1) {
+
+ // First allocation attempt is lock-free.
+ Generation *young = _young_gen;
+ if (young->should_allocate(size, is_tlab)) {
+ result = young->par_allocate(size, is_tlab);
+ if (result != nullptr) {
+ assert(is_in_reserved(result), "result not in heap");
+ return result;
+ }
+ }
+ uint gc_count_before; // Read inside the Heap_lock locked region.
+ {
+ MutexLocker ml(Heap_lock);
+ log_trace(gc, alloc)("SerialHeap::mem_allocate_work: attempting locked slow path allocation");
+ // Note that only large objects get a shot at being
+ // allocated in later generations.
+ bool first_only = !should_try_older_generation_allocation(size);
+
+ result = attempt_allocation(size, is_tlab, first_only);
+ if (result != nullptr) {
+ assert(is_in_reserved(result), "result not in heap");
+ return result;
+ }
+
+ if (GCLocker::is_active_and_needs_gc()) {
+ if (is_tlab) {
+ return nullptr; // Caller will retry allocating individual object.
+ }
+ if (!is_maximal_no_gc()) {
+ // Try and expand heap to satisfy request.
+ result = expand_heap_and_allocate(size, is_tlab);
+ // Result could be null if we are out of space.
+ if (result != nullptr) {
+ return result;
+ }
+ }
+
+ if (gclocker_stalled_count > GCLockerRetryAllocationCount) {
+ return nullptr; // We didn't get to do a GC and we didn't get any memory.
+ }
+
+ // If this thread is not in a jni critical section, we stall
+ // the requestor until the critical section has cleared and
+ // GC allowed. When the critical section clears, a GC is
+ // initiated by the last thread exiting the critical section; so
+ // we retry the allocation sequence from the beginning of the loop,
+ // rather than causing more, now probably unnecessary, GC attempts.
+ JavaThread* jthr = JavaThread::current();
+ if (!jthr->in_critical()) {
+ MutexUnlocker mul(Heap_lock);
+ // Wait for JNI critical section to be exited
+ GCLocker::stall_until_clear();
+ gclocker_stalled_count += 1;
+ continue;
+ } else {
+ if (CheckJNICalls) {
+ fatal("Possible deadlock due to allocating while"
+ " in jni critical section");
+ }
+ return nullptr;
+ }
+ }
+
+ // Read the gc count while the heap lock is held.
+ gc_count_before = total_collections();
+ }
+
+ VM_GenCollectForAllocation op(size, is_tlab, gc_count_before);
+ VMThread::execute(&op);
+ if (op.prologue_succeeded()) {
+ result = op.result();
+ if (op.gc_locked()) {
+ assert(result == nullptr, "must be null if gc_locked() is true");
+ continue; // Retry and/or stall as necessary.
+ }
+
+ assert(result == nullptr || is_in_reserved(result),
+ "result not in heap");
+ return result;
+ }
+
+ // Give a warning if we seem to be looping forever.
+ if ((QueuedAllocationWarningCount > 0) &&
+ (try_count % QueuedAllocationWarningCount == 0)) {
+ log_warning(gc, ergo)("SerialHeap::mem_allocate_work retries %d times,"
+ " size=" SIZE_FORMAT " %s", try_count, size, is_tlab ? "(TLAB)" : "");
+ }
+ }
+}
+
+HeapWord* SerialHeap::attempt_allocation(size_t size,
+ bool is_tlab,
+ bool first_only) {
+ HeapWord* res = nullptr;
+
+ if (_young_gen->should_allocate(size, is_tlab)) {
+ res = _young_gen->allocate(size, is_tlab);
+ if (res != nullptr || first_only) {
+ return res;
+ }
+ }
+
+ if (_old_gen->should_allocate(size, is_tlab)) {
+ res = _old_gen->allocate(size, is_tlab);
+ }
+
+ return res;
+}
+
+HeapWord* SerialHeap::mem_allocate(size_t size,
+ bool* gc_overhead_limit_was_exceeded) {
+ return mem_allocate_work(size,
+ false /* is_tlab */);
+}
+
+bool SerialHeap::must_clear_all_soft_refs() {
+ return _gc_cause == GCCause::_metadata_GC_clear_soft_refs ||
+ _gc_cause == GCCause::_wb_full_gc;
+}
+
+void SerialHeap::collect_generation(Generation* gen, bool full, size_t size,
+ bool is_tlab, bool run_verification, bool clear_soft_refs) {
+ FormatBuffer<> title("Collect gen: %s", gen->short_name());
+ GCTraceTime(Trace, gc, phases) t1(title);
+ TraceCollectorStats tcs(gen->counters());
+ TraceMemoryManagerStats tmms(gen->gc_manager(), gc_cause(), heap()->is_young_gen(gen) ? "end of minor GC" : "end of major GC");
+
+ gen->stat_record()->invocations++;
+ gen->stat_record()->accumulated_time.start();
+
+ // Must be done anew before each collection because
+ // a previous collection will do mangling and will
+ // change top of some spaces.
+ record_gen_tops_before_GC();
+
+ log_trace(gc)("%s invoke=%d size=" SIZE_FORMAT, heap()->is_young_gen(gen) ? "Young" : "Old", gen->stat_record()->invocations, size * HeapWordSize);
+
+ if (run_verification && VerifyBeforeGC) {
+ Universe::verify("Before GC");
+ }
+ COMPILER2_OR_JVMCI_PRESENT(DerivedPointerTable::clear());
+
+ // Do collection work
+ {
+ save_marks(); // save marks for all gens
+
+ gen->collect(full, clear_soft_refs, size, is_tlab);
+ }
+
+ COMPILER2_OR_JVMCI_PRESENT(DerivedPointerTable::update_pointers());
+
+ gen->stat_record()->accumulated_time.stop();
+
+ update_gc_stats(gen, full);
+
+ if (run_verification && VerifyAfterGC) {
+ Universe::verify("After GC");
+ }
+}
+
+void SerialHeap::do_collection(bool full,
+ bool clear_all_soft_refs,
+ size_t size,
+ bool is_tlab,
+ GenerationType max_generation) {
+ ResourceMark rm;
+ DEBUG_ONLY(Thread* my_thread = Thread::current();)
+
+ assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
+ assert(my_thread->is_VM_thread(), "only VM thread");
+ assert(Heap_lock->is_locked(),
+ "the requesting thread should have the Heap_lock");
+ guarantee(!is_gc_active(), "collection is not reentrant");
+
+ if (GCLocker::check_active_before_gc()) {
+ return; // GC is disabled (e.g. JNI GetXXXCritical operation)
+ }
+
+ const bool do_clear_all_soft_refs = clear_all_soft_refs ||
+ soft_ref_policy()->should_clear_all_soft_refs();
+
+ ClearedAllSoftRefs casr(do_clear_all_soft_refs, soft_ref_policy());
+
+ AutoModifyRestore temporarily(_is_gc_active, true);
+
+ bool complete = full && (max_generation == OldGen);
+ bool old_collects_young = complete && !ScavengeBeforeFullGC;
+ bool do_young_collection = !old_collects_young && _young_gen->should_collect(full, size, is_tlab);
+
+ const PreGenGCValues pre_gc_values = get_pre_gc_values();
+
+ bool run_verification = total_collections() >= VerifyGCStartAt;
+ bool prepared_for_verification = false;
+ bool do_full_collection = false;
+
+ if (do_young_collection) {
+ GCIdMark gc_id_mark;
+ GCTraceCPUTime tcpu(((DefNewGeneration*)_young_gen)->gc_tracer());
+ GCTraceTime(Info, gc) t("Pause Young", nullptr, gc_cause(), true);
+
+ print_heap_before_gc();
+
+ if (run_verification && VerifyBeforeGC) {
+ prepare_for_verify();
+ prepared_for_verification = true;
+ }
+
+ gc_prologue(complete);
+ increment_total_collections(complete);
+
+ collect_generation(_young_gen,
+ full,
+ size,
+ is_tlab,
+ run_verification,
+ do_clear_all_soft_refs);
+
+ if (size > 0 && (!is_tlab || _young_gen->supports_tlab_allocation()) &&
+ size * HeapWordSize <= _young_gen->unsafe_max_alloc_nogc()) {
+ // Allocation request was met by young GC.
+ size = 0;
+ }
+
+ // Ask if young collection is enough. If so, do the final steps for young collection,
+ // and fallthrough to the end.
+ do_full_collection = should_do_full_collection(size, full, is_tlab, max_generation);
+ if (!do_full_collection) {
+ // Adjust generation sizes.
+ _young_gen->compute_new_size();
+
+ print_heap_change(pre_gc_values);
+
+ // Track memory usage and detect low memory after GC finishes
+ MemoryService::track_memory_usage();
+
+ gc_epilogue(complete);
+ }
+
+ print_heap_after_gc();
+
+ } else {
+ // No young collection, ask if we need to perform Full collection.
+ do_full_collection = should_do_full_collection(size, full, is_tlab, max_generation);
+ }
+
+ if (do_full_collection) {
+ GCIdMark gc_id_mark;
+ GCTraceCPUTime tcpu(GenMarkSweep::gc_tracer());
+ GCTraceTime(Info, gc) t("Pause Full", nullptr, gc_cause(), true);
+
+ print_heap_before_gc();
+
+ if (!prepared_for_verification && run_verification && VerifyBeforeGC) {
+ prepare_for_verify();
+ }
+
+ if (!do_young_collection) {
+ gc_prologue(complete);
+ increment_total_collections(complete);
+ }
+
+ // Accounting quirk: total full collections would be incremented when "complete"
+ // is set, by calling increment_total_collections above. However, we also need to
+ // account Full collections that had "complete" unset.
+ if (!complete) {
+ increment_total_full_collections();
+ }
+
+ CodeCache::on_gc_marking_cycle_start();
+
+ ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
+ false /* unregister_nmethods_during_purge */,
+ false /* lock_codeblob_free_separately */);
+
+ collect_generation(_old_gen,
+ full,
+ size,
+ is_tlab,
+ run_verification,
+ do_clear_all_soft_refs);
+
+ CodeCache::on_gc_marking_cycle_finish();
+ CodeCache::arm_all_nmethods();
+
+ // Adjust generation sizes.
+ _old_gen->compute_new_size();
+ _young_gen->compute_new_size();
+
+ // Delete metaspaces for unloaded class loaders and clean up loader_data graph
+ ClassLoaderDataGraph::purge(/*at_safepoint*/true);
+ DEBUG_ONLY(MetaspaceUtils::verify();)
+
+ // Need to clear claim bits for the next mark.
+ ClassLoaderDataGraph::clear_claimed_marks();
+
+ // Resize the metaspace capacity after full collections
+ MetaspaceGC::compute_new_size();
+
+ print_heap_change(pre_gc_values);
+
+ // Track memory usage and detect low memory after GC finishes
+ MemoryService::track_memory_usage();
+
+ // Need to tell the epilogue code we are done with Full GC, regardless what was
+ // the initial value for "complete" flag.
+ gc_epilogue(true);
+
+ print_heap_after_gc();
+ }
+}
+
+bool SerialHeap::should_do_full_collection(size_t size, bool full, bool is_tlab,
+ SerialHeap::GenerationType max_gen) const {
+ return max_gen == OldGen && _old_gen->should_collect(full, size, is_tlab);
+}
+
+void SerialHeap::register_nmethod(nmethod* nm) {
+ ScavengableNMethods::register_nmethod(nm);
+}
+
+void SerialHeap::unregister_nmethod(nmethod* nm) {
+ ScavengableNMethods::unregister_nmethod(nm);
+}
+
+void SerialHeap::verify_nmethod(nmethod* nm) {
+ ScavengableNMethods::verify_nmethod(nm);
+}
+
+void SerialHeap::prune_scavengable_nmethods() {
+ ScavengableNMethods::prune_nmethods_not_into_young();
+}
+
+void SerialHeap::prune_unlinked_nmethods() {
+ ScavengableNMethods::prune_unlinked_nmethods();
+}
+
+HeapWord* SerialHeap::satisfy_failed_allocation(size_t size, bool is_tlab) {
+ GCCauseSetter x(this, GCCause::_allocation_failure);
+ HeapWord* result = nullptr;
+
+ assert(size != 0, "Precondition violated");
+ if (GCLocker::is_active_and_needs_gc()) {
+ // GC locker is active; instead of a collection we will attempt
+ // to expand the heap, if there's room for expansion.
+ if (!is_maximal_no_gc()) {
+ result = expand_heap_and_allocate(size, is_tlab);
+ }
+ return result; // Could be null if we are out of space.
+ } else if (!incremental_collection_will_fail(false /* don't consult_young */)) {
+ // Do an incremental collection.
+ do_collection(false, // full
+ false, // clear_all_soft_refs
+ size, // size
+ is_tlab, // is_tlab
+ SerialHeap::OldGen); // max_generation
+ } else {
+ log_trace(gc)(" :: Trying full because partial may fail :: ");
+ // Try a full collection; see delta for bug id 6266275
+ // for the original code and why this has been simplified
+ // with from-space allocation criteria modified and
+ // such allocation moved out of the safepoint path.
+ do_collection(true, // full
+ false, // clear_all_soft_refs
+ size, // size
+ is_tlab, // is_tlab
+ SerialHeap::OldGen); // max_generation
+ }
+
+ result = attempt_allocation(size, is_tlab, false /*first_only*/);
+
+ if (result != nullptr) {
+ assert(is_in_reserved(result), "result not in heap");
+ return result;
+ }
+
+ // OK, collection failed, try expansion.
+ result = expand_heap_and_allocate(size, is_tlab);
+ if (result != nullptr) {
+ return result;
+ }
+
+ // If we reach this point, we're really out of memory. Try every trick
+ // we can to reclaim memory. Force collection of soft references. Force
+ // a complete compaction of the heap. Any additional methods for finding
+ // free memory should be here, especially if they are expensive. If this
+ // attempt fails, an OOM exception will be thrown.
+ {
+ UIntFlagSetting flag_change(MarkSweepAlwaysCompactCount, 1); // Make sure the heap is fully compacted
+
+ do_collection(true, // full
+ true, // clear_all_soft_refs
+ size, // size
+ is_tlab, // is_tlab
+ SerialHeap::OldGen); // max_generation
+ }
+
+ result = attempt_allocation(size, is_tlab, false /* first_only */);
+ if (result != nullptr) {
+ assert(is_in_reserved(result), "result not in heap");
+ return result;
+ }
+
+ assert(!soft_ref_policy()->should_clear_all_soft_refs(),
+ "Flag should have been handled and cleared prior to this point");
+
+ // What else? We might try synchronous finalization later. If the total
+ // space available is large enough for the allocation, then a more
+ // complete compaction phase than we've tried so far might be
+ // appropriate.
+ return nullptr;
+}
+
+#ifdef ASSERT
+class AssertNonScavengableClosure: public OopClosure {
+public:
+ virtual void do_oop(oop* p) {
+ assert(!SerialHeap::heap()->is_in_partial_collection(*p),
+ "Referent should not be scavengable."); }
+ virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); }
+};
+static AssertNonScavengableClosure assert_is_non_scavengable_closure;
+#endif
+
+void SerialHeap::process_roots(ScanningOption so,
+ OopClosure* strong_roots,
+ CLDClosure* strong_cld_closure,
+ CLDClosure* weak_cld_closure,
+ CodeBlobToOopClosure* code_roots) {
+ // General roots.
+ assert(code_roots != nullptr, "code root closure should always be set");
+
+ ClassLoaderDataGraph::roots_cld_do(strong_cld_closure, weak_cld_closure);
+
+ // Only process code roots from thread stacks if we aren't visiting the entire CodeCache anyway
+ CodeBlobToOopClosure* roots_from_code_p = (so & SO_AllCodeCache) ? nullptr : code_roots;
+
+ Threads::oops_do(strong_roots, roots_from_code_p);
+
+ OopStorageSet::strong_oops_do(strong_roots);
+
+ if (so & SO_ScavengeCodeCache) {
+ assert(code_roots != nullptr, "must supply closure for code cache");
+
+ // We only visit parts of the CodeCache when scavenging.
+ ScavengableNMethods::nmethods_do(code_roots);
+ }
+ if (so & SO_AllCodeCache) {
+ assert(code_roots != nullptr, "must supply closure for code cache");
+
+ // CMSCollector uses this to do intermediate-strength collections.
+ // We scan the entire code cache, since CodeCache::do_unloading is not called.
+ CodeCache::blobs_do(code_roots);
+ }
+ // Verify that the code cache contents are not subject to
+ // movement by a scavenging collection.
+ DEBUG_ONLY(CodeBlobToOopClosure assert_code_is_non_scavengable(&assert_is_non_scavengable_closure, !CodeBlobToOopClosure::FixRelocations));
+ DEBUG_ONLY(ScavengableNMethods::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable));
+}
+
+bool SerialHeap::no_allocs_since_save_marks() {
+ return _young_gen->no_allocs_since_save_marks() &&
+ _old_gen->no_allocs_since_save_marks();
+}
+
+// public collection interfaces
+void SerialHeap::collect(GCCause::Cause cause) {
+ // The caller doesn't have the Heap_lock
+ assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");
+
+ unsigned int gc_count_before;
+ unsigned int full_gc_count_before;
+
+ {
+ MutexLocker ml(Heap_lock);
+ // Read the GC count while holding the Heap_lock
+ gc_count_before = total_collections();
+ full_gc_count_before = total_full_collections();
+ }
+
+ if (GCLocker::should_discard(cause, gc_count_before)) {
+ return;
+ }
+
+ bool should_run_young_gc = (cause == GCCause::_wb_young_gc)
+ || (cause == GCCause::_gc_locker)
+ DEBUG_ONLY(|| (cause == GCCause::_scavenge_alot));
+
+ const GenerationType max_generation = should_run_young_gc
+ ? YoungGen
+ : OldGen;
+
+ while (true) {
+ VM_GenCollectFull op(gc_count_before, full_gc_count_before,
+ cause, max_generation);
+ VMThread::execute(&op);
+
+ if (!GCCause::is_explicit_full_gc(cause)) {
+ return;
+ }
+
+ {
+ MutexLocker ml(Heap_lock);
+ // Read the GC count while holding the Heap_lock
+ if (full_gc_count_before != total_full_collections()) {
+ return;
+ }
+ }
+
+ if (GCLocker::is_active_and_needs_gc()) {
+ // If GCLocker is active, wait until clear before retrying.
+ GCLocker::stall_until_clear();
+ }
+ }
+}
+
+void SerialHeap::do_full_collection(bool clear_all_soft_refs) {
+ do_full_collection(clear_all_soft_refs, OldGen);
+}
+
+void SerialHeap::do_full_collection(bool clear_all_soft_refs,
+ GenerationType last_generation) {
+ do_collection(true, // full
+ clear_all_soft_refs, // clear_all_soft_refs
+ 0, // size
+ false, // is_tlab
+ last_generation); // last_generation
+ // Hack XXX FIX ME !!!
+ // A scavenge may not have been attempted, or may have
+ // been attempted and failed, because the old gen was too full
+ if (gc_cause() == GCCause::_gc_locker && incremental_collection_failed()) {
+ log_debug(gc, jni)("GC locker: Trying a full collection because scavenge failed");
+ // This time allow the old gen to be collected as well
+ do_collection(true, // full
+ clear_all_soft_refs, // clear_all_soft_refs
+ 0, // size
+ false, // is_tlab
+ OldGen); // last_generation
+ }
+}
+
+bool SerialHeap::is_in_young(const void* p) const {
+ bool result = p < _old_gen->reserved().start();
+ assert(result == _young_gen->is_in_reserved(p),
+ "incorrect test - result=%d, p=" PTR_FORMAT, result, p2i(p));
+ return result;
+}
+
+bool SerialHeap::requires_barriers(stackChunkOop obj) const {
+ return !is_in_young(obj);
+}
+
+// Returns "TRUE" iff "p" points into the committed areas of the heap.
+bool SerialHeap::is_in(const void* p) const {
+ return _young_gen->is_in(p) || _old_gen->is_in(p);
+}
+
+#ifdef ASSERT
+// Don't implement this by using is_in_young(). This method is used
+// in some cases to check that is_in_young() is correct.
+bool SerialHeap::is_in_partial_collection(const void* p) {
+ assert(is_in_reserved(p) || p == nullptr,
+ "Does not work if address is non-null and outside of the heap");
+ return p < _young_gen->reserved().end() && p != nullptr;
+}
+#endif
+
+void SerialHeap::object_iterate(ObjectClosure* cl) {
+ _young_gen->object_iterate(cl);
+ _old_gen->object_iterate(cl);
+}
+
+HeapWord* SerialHeap::block_start(const void* addr) const {
+ assert(is_in_reserved(addr), "block_start of address outside of heap");
+ if (_young_gen->is_in_reserved(addr)) {
+ assert(_young_gen->is_in(addr), "addr should be in allocated part of generation");
+ return _young_gen->block_start(addr);
+ }
+
+ assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address");
+ assert(_old_gen->is_in(addr), "addr should be in allocated part of generation");
+ return _old_gen->block_start(addr);
+}
+
+bool SerialHeap::block_is_obj(const HeapWord* addr) const {
+ assert(is_in_reserved(addr), "block_is_obj of address outside of heap");
+ assert(block_start(addr) == addr, "addr must be a block start");
+
+ if (_young_gen->is_in_reserved(addr)) {
+ return _young_gen->eden()->is_in(addr)
+ || _young_gen->from()->is_in(addr)
+ || _young_gen->to() ->is_in(addr);
+ }
+
+ assert(_old_gen->is_in_reserved(addr), "must be in old-gen");
+ return addr < _old_gen->space()->top();
+}
+
+size_t SerialHeap::tlab_capacity(Thread* thr) const {
+ assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!");
+ assert(_young_gen->supports_tlab_allocation(), "Young gen doesn't support TLAB allocation?!");
+ return _young_gen->tlab_capacity();
+}
+
+size_t SerialHeap::tlab_used(Thread* thr) const {
+ assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!");
+ assert(_young_gen->supports_tlab_allocation(), "Young gen doesn't support TLAB allocation?!");
+ return _young_gen->tlab_used();
+}
+
+size_t SerialHeap::unsafe_max_tlab_alloc(Thread* thr) const {
+ assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!");
+ assert(_young_gen->supports_tlab_allocation(), "Young gen doesn't support TLAB allocation?!");
+ return _young_gen->unsafe_max_tlab_alloc();
+}
+
+HeapWord* SerialHeap::allocate_new_tlab(size_t min_size,
+ size_t requested_size,
+ size_t* actual_size) {
+ HeapWord* result = mem_allocate_work(requested_size /* size */,
+ true /* is_tlab */);
+ if (result != nullptr) {
+ *actual_size = requested_size;
+ }
+
+ return result;
+}
+
+void SerialHeap::prepare_for_verify() {
+ ensure_parsability(false); // no need to retire TLABs
+}
+
+bool SerialHeap::is_maximal_no_gc() const {
+ // We don't expand young-gen except at a GC.
+ return _old_gen->is_maximal_no_gc();
+}
+
+void SerialHeap::save_marks() {
+ _young_gen->save_marks();
+ _old_gen->save_marks();
+}
+
+void SerialHeap::verify(VerifyOption option /* ignored */) {
+ log_debug(gc, verify)("%s", _old_gen->name());
+ _old_gen->verify();
+
+ log_debug(gc, verify)("%s", _young_gen->name());
+ _young_gen->verify();
+
+ log_debug(gc, verify)("RemSet");
+ rem_set()->verify();
+}
+
+void SerialHeap::print_on(outputStream* st) const {
+ if (_young_gen != nullptr) {
+ _young_gen->print_on(st);
+ }
+ if (_old_gen != nullptr) {
+ _old_gen->print_on(st);
+ }
+ MetaspaceUtils::print_on(st);
+}
+
+void SerialHeap::gc_threads_do(ThreadClosure* tc) const {
+}
+
+bool SerialHeap::print_location(outputStream* st, void* addr) const {
+ return BlockLocationPrinter::print_location(st, addr);
+}
+
+void SerialHeap::print_tracing_info() const {
+ if (log_is_enabled(Debug, gc, heap, exit)) {
+ LogStreamHandle(Debug, gc, heap, exit) lsh;
+ _young_gen->print_summary_info_on(&lsh);
+ _old_gen->print_summary_info_on(&lsh);
+ }
+}
+
+void SerialHeap::print_heap_change(const PreGenGCValues& pre_gc_values) const {
+ const DefNewGeneration* const def_new_gen = (DefNewGeneration*) young_gen();
+
+ log_info(gc, heap)(HEAP_CHANGE_FORMAT" "
+ HEAP_CHANGE_FORMAT" "
+ HEAP_CHANGE_FORMAT,
+ HEAP_CHANGE_FORMAT_ARGS(def_new_gen->short_name(),
+ pre_gc_values.young_gen_used(),
+ pre_gc_values.young_gen_capacity(),
+ def_new_gen->used(),
+ def_new_gen->capacity()),
+ HEAP_CHANGE_FORMAT_ARGS("Eden",
+ pre_gc_values.eden_used(),
+ pre_gc_values.eden_capacity(),
+ def_new_gen->eden()->used(),
+ def_new_gen->eden()->capacity()),
+ HEAP_CHANGE_FORMAT_ARGS("From",
+ pre_gc_values.from_used(),
+ pre_gc_values.from_capacity(),
+ def_new_gen->from()->used(),
+ def_new_gen->from()->capacity()));
+ log_info(gc, heap)(HEAP_CHANGE_FORMAT,
+ HEAP_CHANGE_FORMAT_ARGS(old_gen()->short_name(),
+ pre_gc_values.old_gen_used(),
+ pre_gc_values.old_gen_capacity(),
+ old_gen()->used(),
+ old_gen()->capacity()));
+ MetaspaceUtils::print_metaspace_change(pre_gc_values.metaspace_sizes());
+}
+
+void SerialHeap::gc_prologue(bool full) {
+ // Fill TLAB's and such
+ ensure_parsability(true); // retire TLABs
+
+ _old_gen->gc_prologue();
+};
+
+void SerialHeap::gc_epilogue(bool full) {
+#if COMPILER2_OR_JVMCI
+ assert(DerivedPointerTable::is_empty(), "derived pointer present");
+#endif // COMPILER2_OR_JVMCI
+
+ resize_all_tlabs();
+
+ _young_gen->gc_epilogue(full);
+ _old_gen->gc_epilogue();
+
+ MetaspaceCounters::update_performance_counters();
+};
+
+#ifndef PRODUCT
+void SerialHeap::record_gen_tops_before_GC() {
+ if (ZapUnusedHeapArea) {
+ _young_gen->record_spaces_top();
+ _old_gen->record_spaces_top();
+ }
+}
+#endif // not PRODUCT
diff --git a/src/hotspot/share/gc/serial/serialHeap.hpp b/src/hotspot/share/gc/serial/serialHeap.hpp
index da86f2909d78e..f1f2e5a1cc1a7 100644
--- a/src/hotspot/share/gc/serial/serialHeap.hpp
+++ b/src/hotspot/share/gc/serial/serialHeap.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,10 +26,17 @@
#define SHARE_GC_SERIAL_SERIALHEAP_HPP
#include "gc/serial/defNewGeneration.hpp"
+#include "gc/serial/generation.hpp"
#include "gc/serial/tenuredGeneration.hpp"
-#include "gc/shared/genCollectedHeap.hpp"
+#include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/oopStorageParState.hpp"
+#include "gc/shared/preGCValues.hpp"
+#include "gc/shared/softRefPolicy.hpp"
#include "utilities/growableArray.hpp"
+class CardTableRS;
+class GCPolicyCounters;
+
class GCMemoryManager;
class MemoryPool;
class OopIterateClosure;
@@ -55,7 +62,272 @@ class TenuredGeneration;
// +-----------------+--------+--------+--------+---------------+-------------------+
// |<- committed ->| |<- committed ->|
//
-class SerialHeap : public GenCollectedHeap {
+class SerialHeap : public CollectedHeap {
+ friend class Generation;
+ friend class DefNewGeneration;
+ friend class TenuredGeneration;
+ friend class GenMarkSweep;
+ friend class VM_GenCollectForAllocation;
+ friend class VM_GenCollectFull;
+ friend class VM_GC_HeapInspection;
+ friend class VM_HeapDumper;
+ friend class HeapInspection;
+ friend class GCCauseSetter;
+ friend class VMStructs;
+public:
+ friend class VM_PopulateDumpSharedSpace;
+
+ enum GenerationType {
+ YoungGen,
+ OldGen
+ };
+
+private:
+ DefNewGeneration* _young_gen;
+ TenuredGeneration* _old_gen;
+
+private:
+ // The singleton CardTable Remembered Set.
+ CardTableRS* _rem_set;
+
+ GCPolicyCounters* _gc_policy_counters;
+
+ // Indicates that the most recent previous incremental collection failed.
+ // The flag is cleared when an action is taken that might clear the
+ // condition that caused that incremental collection to fail.
+ bool _incremental_collection_failed;
+
+ // Collects the given generation.
+ void collect_generation(Generation* gen, bool full, size_t size, bool is_tlab,
+ bool run_verification, bool clear_soft_refs);
+
+ // Reserve aligned space for the heap as needed by the contained generations.
+ ReservedHeapSpace allocate(size_t alignment);
+
+ PreGenGCValues get_pre_gc_values() const;
+
+private:
+ GCMemoryManager* _young_manager;
+ GCMemoryManager* _old_manager;
+
+ // Helper functions for allocation
+ HeapWord* attempt_allocation(size_t size,
+ bool is_tlab,
+ bool first_only);
+
+ // Helper function for two callbacks below.
+ // Considers collection of the first max_level+1 generations.
+ void do_collection(bool full,
+ bool clear_all_soft_refs,
+ size_t size,
+ bool is_tlab,
+ GenerationType max_generation);
+
+ // Callback from VM_GenCollectForAllocation operation.
+ // This function does everything necessary/possible to satisfy an
+ // allocation request that failed in the youngest generation that should
+ // have handled it (including collection, expansion, etc.)
+ HeapWord* satisfy_failed_allocation(size_t size, bool is_tlab);
+
+ // Callback from VM_GenCollectFull operation.
+ // Perform a full collection of the first max_level+1 generations.
+ void do_full_collection(bool clear_all_soft_refs) override;
+ void do_full_collection(bool clear_all_soft_refs, GenerationType max_generation);
+
+ // Does the "cause" of GC indicate that
+ // we absolutely __must__ clear soft refs?
+ bool must_clear_all_soft_refs();
+
+public:
+ // Returns JNI_OK on success
+ jint initialize() override;
+
+ // Does operations required after initialization has been done.
+ void post_initialize() override;
+
+ bool is_young_gen(const Generation* gen) const { return gen == _young_gen; }
+ bool is_old_gen(const Generation* gen) const { return gen == _old_gen; }
+
+ MemRegion reserved_region() const { return _reserved; }
+ bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); }
+
+ // Performance Counter support
+ GCPolicyCounters* counters() { return _gc_policy_counters; }
+
+ size_t capacity() const override;
+ size_t used() const override;
+
+ size_t max_capacity() const override;
+
+ HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) override;
+
+ // Perform a full collection of the heap; intended for use in implementing
+ // "System.gc". This implies as full a collection as the CollectedHeap
+ // supports. Caller does not hold the Heap_lock on entry.
+ void collect(GCCause::Cause cause) override;
+
+ // Returns "TRUE" iff "p" points into the committed areas of the heap.
+ // The methods is_in() and is_in_youngest() may be expensive to compute
+ // in general, so, to prevent their inadvertent use in product jvm's, we
+ // restrict their use to assertion checking or verification only.
+ bool is_in(const void* p) const override;
+
+ // Returns true if p points into the reserved space for the young generation.
+ // Assumes the young gen address range is less than that of the old gen.
+ bool is_in_young(const void* p) const;
+
+ bool requires_barriers(stackChunkOop obj) const override;
+
+#ifdef ASSERT
+ bool is_in_partial_collection(const void* p);
+#endif
+
+ // Optimized nmethod scanning support routines
+ void register_nmethod(nmethod* nm) override;
+ void unregister_nmethod(nmethod* nm) override;
+ void verify_nmethod(nmethod* nm) override;
+
+ void prune_scavengable_nmethods();
+ void prune_unlinked_nmethods();
+
+ // Iteration functions.
+ void object_iterate(ObjectClosure* cl) override;
+
+ // A CollectedHeap is divided into a dense sequence of "blocks"; that is,
+ // each address in the (reserved) heap is a member of exactly
+ // one block. The defining characteristic of a block is that it is
+ // possible to find its size, and thus to progress forward to the next
+ // block. (Blocks may be of different sizes.) Thus, blocks may
+ // represent Java objects, or they might be free blocks in a
+ // free-list-based heap (or subheap), as long as the two kinds are
+ // distinguishable and the size of each is determinable.
+
+ // Returns the address of the start of the "block" that contains the
+ // address "addr". We say "blocks" instead of "object" since some heaps
+ // may not pack objects densely; a chunk may either be an object or a
+ // non-object.
+ HeapWord* block_start(const void* addr) const;
+
+ // Requires "addr" to be the start of a block, and returns "TRUE" iff
+ // the block is an object. Assumes (and verifies in non-product
+ // builds) that addr is in the allocated part of the heap and is
+ // the start of a chunk.
+ bool block_is_obj(const HeapWord* addr) const;
+
+ // Section on TLAB's.
+ size_t tlab_capacity(Thread* thr) const override;
+ size_t tlab_used(Thread* thr) const override;
+ size_t unsafe_max_tlab_alloc(Thread* thr) const override;
+ HeapWord* allocate_new_tlab(size_t min_size,
+ size_t requested_size,
+ size_t* actual_size) override;
+
+ // Update the gc statistics for each generation.
+ void update_gc_stats(Generation* current_generation, bool full) {
+ _old_gen->update_gc_stats(current_generation, full);
+ }
+
+ void prepare_for_verify() override;
+ void verify(VerifyOption option) override;
+
+ void print_on(outputStream* st) const override;
+ void gc_threads_do(ThreadClosure* tc) const override;
+ void print_tracing_info() const override;
+
+ // Used to print information about locations in the hs_err file.
+ bool print_location(outputStream* st, void* addr) const override;
+
+ void print_heap_change(const PreGenGCValues& pre_gc_values) const;
+
+ // The functions below are helper functions that a subclass of
+ // "CollectedHeap" can use in the implementation of its virtual
+ // functions.
+
+ class GenClosure : public StackObj {
+ public:
+ virtual void do_generation(Generation* gen) = 0;
+ };
+
+ // Return "true" if all generations have reached the
+ // maximal committed limit that they can reach, without a garbage
+ // collection.
+ virtual bool is_maximal_no_gc() const override;
+
+ // This function returns the CardTableRS object that allows us to scan
+ // generations in a fully generational heap.
+ CardTableRS* rem_set() { return _rem_set; }
+
+ // The ScanningOption determines which of the roots
+ // the closure is applied to:
+ // "SO_None" does none;
+ enum ScanningOption {
+ SO_None = 0x0,
+ SO_AllCodeCache = 0x8,
+ SO_ScavengeCodeCache = 0x10
+ };
+
+ protected:
+ virtual void gc_prologue(bool full);
+ virtual void gc_epilogue(bool full);
+
+ public:
+ // Apply closures on various roots in Young GC or marking/adjust phases of Full GC.
+ void process_roots(ScanningOption so,
+ OopClosure* strong_roots,
+ CLDClosure* strong_cld_closure,
+ CLDClosure* weak_cld_closure,
+ CodeBlobToOopClosure* code_roots);
+
+ // Set the saved marks of generations, if that makes sense.
+ // In particular, if any generation might iterate over the oops
+ // in other generations, it should call this method.
+ void save_marks();
+
+ // Returns "true" iff no allocations have occurred since the last
+ // call to "save_marks".
+ bool no_allocs_since_save_marks();
+
+ // Returns true if an incremental collection is likely to fail.
+ // We optionally consult the young gen, if asked to do so;
+ // otherwise we base our answer on whether the previous incremental
+ // collection attempt failed with no corrective action as of yet.
+ bool incremental_collection_will_fail(bool consult_young) {
+ // The first disjunct remembers if an incremental collection failed, even
+ // when we thought (second disjunct) that it would not.
+ return incremental_collection_failed() ||
+ (consult_young && !_young_gen->collection_attempt_is_safe());
+ }
+
+ // If a generation bails out of an incremental collection,
+ // it sets this flag.
+ bool incremental_collection_failed() const {
+ return _incremental_collection_failed;
+ }
+ void set_incremental_collection_failed() {
+ _incremental_collection_failed = true;
+ }
+ void clear_incremental_collection_failed() {
+ _incremental_collection_failed = false;
+ }
+
+private:
+ // Return true if an allocation should be attempted in the older generation
+ // if it fails in the younger generation. Return false, otherwise.
+ bool should_try_older_generation_allocation(size_t word_size) const;
+
+ // Try to allocate space by expanding the heap.
+ HeapWord* expand_heap_and_allocate(size_t size, bool is_tlab);
+
+ HeapWord* mem_allocate_work(size_t size,
+ bool is_tlab);
+
+ // Save the tops of the spaces in all generations
+ void record_gen_tops_before_GC() PRODUCT_RETURN;
+
+ // Return true if we need to perform full collection.
+ bool should_do_full_collection(size_t size, bool full,
+ bool is_tlab, GenerationType max_gen) const;
+
private:
MemoryPool* _eden_pool;
MemoryPool* _survivor_pool;
@@ -80,13 +352,11 @@ class SerialHeap : public GenCollectedHeap {
GrowableArray memory_pools() override;
DefNewGeneration* young_gen() const {
- assert(_young_gen->kind() == Generation::DefNew, "Wrong generation type");
- return static_cast(_young_gen);
+ return _young_gen;
}
TenuredGeneration* old_gen() const {
- assert(_old_gen->kind() == Generation::MarkSweepCompact, "Wrong generation type");
- return static_cast(_old_gen);
+ return _old_gen;
}
// Apply "cur->do_oop" or "older->do_oop" to all the oops in objects
@@ -97,10 +367,6 @@ class SerialHeap : public GenCollectedHeap {
void oop_since_save_marks_iterate(OopClosureType1* cur,
OopClosureType2* older);
- void young_process_roots(OopClosure* root_closure,
- OopIterateClosure* old_gen_closure,
- CLDClosure* cld_closure);
-
void safepoint_synchronize_begin() override;
void safepoint_synchronize_end() override;
diff --git a/src/hotspot/share/gc/serial/serialHeap.inline.hpp b/src/hotspot/share/gc/serial/serialHeap.inline.hpp
index 6e06c7b6d574e..27819f0d1c933 100644
--- a/src/hotspot/share/gc/serial/serialHeap.inline.hpp
+++ b/src/hotspot/share/gc/serial/serialHeap.inline.hpp
@@ -37,4 +37,89 @@ void SerialHeap::oop_since_save_marks_iterate(OopClosureType1* cur,
old_gen()->oop_since_save_marks_iterate(older);
}
+class ScavengeHelper {
+ DefNewGeneration* _young_gen;
+ HeapWord* _young_gen_end;
+public:
+ ScavengeHelper(DefNewGeneration* young_gen) :
+ _young_gen(young_gen),
+ _young_gen_end(young_gen->reserved().end()) {}
+
+ bool is_in_young_gen(void* p) const {
+ return p < _young_gen_end;
+ }
+
+ template
+ void try_scavenge(T* p, Func&& f) {
+ T heap_oop = RawAccess<>::oop_load(p);
+ // Should we copy the obj?
+ if (!CompressedOops::is_null(heap_oop)) {
+ oop obj = CompressedOops::decode_not_null(heap_oop);
+ if (is_in_young_gen(obj)) {
+ assert(!_young_gen->to()->is_in_reserved(obj), "Scanning field twice?");
+ oop new_obj = obj->is_forwarded() ? obj->forwardee()
+ : _young_gen->copy_to_survivor_space(obj);
+ RawAccess::oop_store(p, new_obj);
+
+ // callback
+ f(new_obj);
+ }
+ }
+ }
+};
+
+class InHeapScanClosure : public BasicOopIterateClosure {
+ ScavengeHelper _helper;
+protected:
+ bool is_in_young_gen(void* p) const {
+ return _helper.is_in_young_gen(p);
+ }
+
+ template
+ void try_scavenge(T* p, Func&& f) {
+ _helper.try_scavenge(p, f);
+ }
+
+ InHeapScanClosure(DefNewGeneration* young_gen) :
+ BasicOopIterateClosure(young_gen->ref_processor()),
+ _helper(young_gen) {}
+};
+
+class OffHeapScanClosure : public OopClosure {
+ ScavengeHelper _helper;
+protected:
+ bool is_in_young_gen(void* p) const {
+ return _helper.is_in_young_gen(p);
+ }
+
+ template
+ void try_scavenge(T* p, Func&& f) {
+ _helper.try_scavenge(p, f);
+ }
+
+ OffHeapScanClosure(DefNewGeneration* young_gen) : _helper(young_gen) {}
+};
+
+class OldGenScanClosure : public InHeapScanClosure {
+ CardTableRS* _rs;
+
+ template
+ void do_oop_work(T* p) {
+ assert(!is_in_young_gen(p), "precondition");
+
+ try_scavenge(p, [&] (oop new_obj) {
+ // If p points to a younger generation, mark the card.
+ if (is_in_young_gen(new_obj)) {
+ _rs->inline_write_ref_field_gc(p);
+ }
+ });
+ }
+public:
+ OldGenScanClosure(DefNewGeneration* g) : InHeapScanClosure(g),
+ _rs(SerialHeap::heap()->rem_set()) {}
+
+ void do_oop(oop* p) { do_oop_work(p); }
+ void do_oop(narrowOop* p) { do_oop_work(p); }
+};
+
#endif // SHARE_GC_SERIAL_SERIALHEAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/shared/genMemoryPools.cpp b/src/hotspot/share/gc/serial/serialMemoryPools.cpp
similarity index 86%
rename from src/hotspot/share/gc/shared/genMemoryPools.cpp
rename to src/hotspot/share/gc/serial/serialMemoryPools.cpp
index 9158e42a1b9d8..b82252b5a094f 100644
--- a/src/hotspot/share/gc/shared/genMemoryPools.cpp
+++ b/src/hotspot/share/gc/serial/serialMemoryPools.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,12 +23,10 @@
*/
#include "precompiled.hpp"
-#include "gc/serial/generation.hpp"
-#include "gc/shared/genMemoryPools.hpp"
-#include "gc/shared/space.hpp"
-#if INCLUDE_SERIALGC
#include "gc/serial/defNewGeneration.hpp"
-#endif
+#include "gc/serial/serialMemoryPools.hpp"
+#include "gc/serial/tenuredGeneration.hpp"
+#include "gc/shared/space.hpp"
ContiguousSpacePool::ContiguousSpacePool(ContiguousSpace* space,
const char* name,
@@ -50,8 +48,6 @@ MemoryUsage ContiguousSpacePool::get_memory_usage() {
return MemoryUsage(initial_size(), used, committed, maxSize);
}
-#if INCLUDE_SERIALGC
-
SurvivorContiguousSpacePool::SurvivorContiguousSpacePool(DefNewGeneration* young_gen,
const char* name,
size_t max_size,
@@ -76,20 +72,18 @@ MemoryUsage SurvivorContiguousSpacePool::get_memory_usage() {
return MemoryUsage(initial_size(), used, committed, maxSize);
}
-#endif // INCLUDE_SERIALGC
-
-GenerationPool::GenerationPool(Generation* gen,
- const char* name,
- bool support_usage_threshold) :
+TenuredGenerationPool::TenuredGenerationPool(TenuredGeneration* gen,
+ const char* name,
+ bool support_usage_threshold) :
CollectedMemoryPool(name, gen->capacity(), gen->max_capacity(),
support_usage_threshold), _gen(gen) {
}
-size_t GenerationPool::used_in_bytes() {
+size_t TenuredGenerationPool::used_in_bytes() {
return _gen->used();
}
-MemoryUsage GenerationPool::get_memory_usage() {
+MemoryUsage TenuredGenerationPool::get_memory_usage() {
size_t used = used_in_bytes();
size_t committed = _gen->capacity();
size_t maxSize = (available_for_allocation() ? max_size() : 0);
diff --git a/src/hotspot/share/gc/shared/genMemoryPools.hpp b/src/hotspot/share/gc/serial/serialMemoryPools.hpp
similarity index 83%
rename from src/hotspot/share/gc/shared/genMemoryPools.hpp
rename to src/hotspot/share/gc/serial/serialMemoryPools.hpp
index 27ee5c978228d..36cd978905578 100644
--- a/src/hotspot/share/gc/shared/genMemoryPools.hpp
+++ b/src/hotspot/share/gc/serial/serialMemoryPools.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_SHARED_GENMEMORYPOOLS_HPP
-#define SHARE_GC_SHARED_GENMEMORYPOOLS_HPP
+#ifndef SHARE_GC_SERIAL_SERIALMEMORYPOOLS_HPP
+#define SHARE_GC_SERIAL_SERIALMEMORYPOOLS_HPP
#include "services/memoryPool.hpp"
@@ -62,14 +62,14 @@ class SurvivorContiguousSpacePool : public CollectedMemoryPool {
size_t committed_in_bytes();
};
-class GenerationPool : public CollectedMemoryPool {
+class TenuredGenerationPool : public CollectedMemoryPool {
private:
- Generation* _gen;
+ TenuredGeneration* _gen;
public:
- GenerationPool(Generation* gen, const char* name, bool support_usage_threshold);
+ TenuredGenerationPool(TenuredGeneration* gen, const char* name, bool support_usage_threshold);
MemoryUsage get_memory_usage();
size_t used_in_bytes();
};
-#endif // SHARE_GC_SHARED_GENMEMORYPOOLS_HPP
+#endif // SHARE_GC_SERIAL_SERIALMEMORYPOOLS_HPP
diff --git a/src/hotspot/share/gc/serial/serialVMOperations.cpp b/src/hotspot/share/gc/serial/serialVMOperations.cpp
new file mode 100644
index 0000000000000..9e88fa5971196
--- /dev/null
+++ b/src/hotspot/share/gc/serial/serialVMOperations.cpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/serial/serialVMOperations.hpp"
+#include "gc/shared/gcLocker.hpp"
+
+void VM_GenCollectForAllocation::doit() {
+ SvcGCMarker sgcm(SvcGCMarker::MINOR);
+
+ SerialHeap* gch = SerialHeap::heap();
+ GCCauseSetter gccs(gch, _gc_cause);
+ _result = gch->satisfy_failed_allocation(_word_size, _tlab);
+ assert(_result == nullptr || gch->is_in_reserved(_result), "result not in heap");
+
+ if (_result == nullptr && GCLocker::is_active_and_needs_gc()) {
+ set_gc_locked();
+ }
+}
+
+void VM_GenCollectFull::doit() {
+ SvcGCMarker sgcm(SvcGCMarker::FULL);
+
+ SerialHeap* gch = SerialHeap::heap();
+ GCCauseSetter gccs(gch, _gc_cause);
+ gch->do_full_collection(gch->must_clear_all_soft_refs(), _max_generation);
+}
diff --git a/src/hotspot/share/gc/serial/serialVMOperations.hpp b/src/hotspot/share/gc/serial/serialVMOperations.hpp
new file mode 100644
index 0000000000000..a056740dd8549
--- /dev/null
+++ b/src/hotspot/share/gc/serial/serialVMOperations.hpp
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SERIAL_SERIALVMOPERATIONS_HPP
+#define SHARE_GC_SERIAL_SERIALVMOPERATIONS_HPP
+
+#include "gc/serial/serialHeap.hpp"
+#include "gc/shared/gcVMOperations.hpp"
+
+class VM_GenCollectForAllocation : public VM_CollectForAllocation {
+ private:
+ bool _tlab; // alloc is of a tlab.
+ public:
+ VM_GenCollectForAllocation(size_t word_size,
+ bool tlab,
+ uint gc_count_before)
+ : VM_CollectForAllocation(word_size, gc_count_before, GCCause::_allocation_failure),
+ _tlab(tlab) {
+ assert(word_size != 0, "An allocation should always be requested with this operation.");
+ }
+ ~VM_GenCollectForAllocation() {}
+ virtual VMOp_Type type() const { return VMOp_GenCollectForAllocation; }
+ virtual void doit();
+};
+
+// VM operation to invoke a collection of the heap as a
+// SerialHeap heap.
+class VM_GenCollectFull: public VM_GC_Operation {
+ private:
+ SerialHeap::GenerationType _max_generation;
+ public:
+ VM_GenCollectFull(uint gc_count_before,
+ uint full_gc_count_before,
+ GCCause::Cause gc_cause,
+ SerialHeap::GenerationType max_generation)
+ : VM_GC_Operation(gc_count_before, gc_cause, full_gc_count_before,
+ max_generation != SerialHeap::YoungGen /* full */),
+ _max_generation(max_generation) { }
+ ~VM_GenCollectFull() {}
+ virtual VMOp_Type type() const { return VMOp_GenCollectFull; }
+ virtual void doit();
+};
+
+
+#endif // SHARE_GC_SERIAL_SERIALVMOPERATIONS_HPP
diff --git a/src/hotspot/share/gc/serial/tenuredGeneration.cpp b/src/hotspot/share/gc/serial/tenuredGeneration.cpp
index ad5fda3aeabb8..fe37d9975498a 100644
--- a/src/hotspot/share/gc/serial/tenuredGeneration.cpp
+++ b/src/hotspot/share/gc/serial/tenuredGeneration.cpp
@@ -263,19 +263,12 @@ void TenuredGeneration::compute_new_size_inner() {
}
}
-void TenuredGeneration::space_iterate(SpaceClosure* blk,
- bool usedOnly) {
- blk->do_space(space());
+HeapWord* TenuredGeneration::block_start(const void* p) const {
+ return space()->block_start_const(p);
}
-void TenuredGeneration::younger_refs_iterate(OopIterateClosure* blk) {
- // Apply "cl->do_oop" to (the address of) (exactly) all the ref fields in
- // "sp" that point into the young generation.
- // The iteration is only over objects allocated at the start of the
- // iterations; objects allocated as a result of applying the closure are
- // not included.
-
- _rs->younger_refs_in_space_iterate(space(), blk);
+void TenuredGeneration::scan_old_to_young_refs() {
+ _rs->scan_old_to_young_refs(space());
}
TenuredGeneration::TenuredGeneration(ReservedSpace rs,
@@ -331,7 +324,7 @@ TenuredGeneration::TenuredGeneration(ReservedSpace rs,
_the_space, _gen_counters);
}
-void TenuredGeneration::gc_prologue(bool full) {
+void TenuredGeneration::gc_prologue() {
_capacity_at_prologue = capacity();
_used_at_prologue = used();
}
@@ -420,6 +413,31 @@ bool TenuredGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes)
return res;
}
+oop TenuredGeneration::promote(oop obj, size_t obj_size) {
+ assert(obj_size == obj->size(), "bad obj_size passed in");
+
+#ifndef PRODUCT
+ if (SerialHeap::heap()->promotion_should_fail()) {
+ return nullptr;
+ }
+#endif // #ifndef PRODUCT
+
+ // Allocate new object.
+ HeapWord* result = allocate(obj_size, false);
+ if (result == nullptr) {
+ // Promotion of obj into gen failed. Try to expand and allocate.
+ result = expand_and_allocate(obj_size, false);
+ if (result == nullptr) {
+ return nullptr;
+ }
+ }
+
+ // Copy to new location.
+ Copy::aligned_disjoint_words(cast_from_oop(obj), result, obj_size);
+ oop new_obj = cast_to_oop(result);
+ return new_obj;
+}
+
void TenuredGeneration::collect(bool full,
bool clear_all_soft_refs,
size_t size,
@@ -450,10 +468,6 @@ TenuredGeneration::expand_and_allocate(size_t word_size, bool is_tlab) {
return _the_space->allocate(word_size);
}
-size_t TenuredGeneration::unsafe_max_alloc_nogc() const {
- return _the_space->free();
-}
-
size_t TenuredGeneration::contiguous_available() const {
return _the_space->free() + _virtual_space.uncommitted_size();
}
@@ -485,7 +499,7 @@ bool TenuredGeneration::no_allocs_since_save_marks() {
return _the_space->saved_mark_at_top();
}
-void TenuredGeneration::gc_epilogue(bool full) {
+void TenuredGeneration::gc_epilogue() {
// update the generation and space performance counters
update_counters();
if (ZapUnusedHeapArea) {
diff --git a/src/hotspot/share/gc/serial/tenuredGeneration.hpp b/src/hotspot/share/gc/serial/tenuredGeneration.hpp
index 6b2a5891c64cd..b87926e049744 100644
--- a/src/hotspot/share/gc/serial/tenuredGeneration.hpp
+++ b/src/hotspot/share/gc/serial/tenuredGeneration.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,6 +29,7 @@
#include "gc/serial/generation.hpp"
#include "gc/shared/gcStats.hpp"
#include "gc/shared/generationCounters.hpp"
+#include "gc/shared/space.hpp"
#include "utilities/macros.hpp"
class SerialBlockOffsetSharedArray;
@@ -45,7 +46,7 @@ class TenuredGeneration: public Generation {
// Abstractly, this is a subtype that gets access to protected fields.
friend class VM_PopulateDumpSharedSpace;
- protected:
+ MemRegion _prev_used_region;
// This is shared with other generations.
CardTableRS* _rs;
@@ -70,7 +71,6 @@ class TenuredGeneration: public Generation {
GenerationCounters* _gen_counters;
CSpaceCounters* _space_counters;
-
// Attempt to expand the generation by "bytes". Expand by at a
// minimum "expand_bytes". Return true if some amount (not
// necessarily the full "bytes") was done.
@@ -81,7 +81,7 @@ class TenuredGeneration: public Generation {
void compute_new_size_inner();
public:
- virtual void compute_new_size();
+ void compute_new_size();
TenuredSpace* space() const { return _the_space; }
@@ -93,15 +93,22 @@ class TenuredGeneration: public Generation {
size_t capacity() const;
size_t used() const;
size_t free() const;
- MemRegion used_region() const;
- void space_iterate(SpaceClosure* blk, bool usedOnly = false);
+ MemRegion used_region() const { return space()->used_region(); }
+ MemRegion prev_used_region() const { return _prev_used_region; }
+ void save_used_region() { _prev_used_region = used_region(); }
- void younger_refs_iterate(OopIterateClosure* blk);
+ // Returns true if this generation cannot be expanded further
+ // without a GC.
+ bool is_maximal_no_gc() const {
+ return _virtual_space.uncommitted_size() == 0;
+ }
- bool is_in(const void* p) const;
+ HeapWord* block_start(const void* p) const;
+
+ void scan_old_to_young_refs();
- ContiguousSpace* first_compaction_space() const;
+ bool is_in(const void* p) const;
TenuredGeneration(ReservedSpace rs,
size_t initial_byte_size,
@@ -109,13 +116,10 @@ class TenuredGeneration: public Generation {
size_t max_byte_size,
CardTableRS* remset);
- Generation::Name kind() { return Generation::MarkSweepCompact; }
-
// Printing
const char* name() const { return "tenured generation"; }
const char* short_name() const { return "Tenured"; }
- size_t unsafe_max_alloc_nogc() const;
size_t contiguous_available() const;
// Iteration
@@ -133,8 +137,6 @@ class TenuredGeneration: public Generation {
bool no_allocs_since_save_marks();
- inline bool block_is_obj(const HeapWord* addr) const;
-
virtual void collect(bool full,
bool clear_all_soft_refs,
size_t size,
@@ -142,8 +144,8 @@ class TenuredGeneration: public Generation {
HeapWord* expand_and_allocate(size_t size, bool is_tlab);
- virtual void gc_prologue(bool full);
- virtual void gc_epilogue(bool full);
+ void gc_prologue();
+ void gc_epilogue();
bool should_collect(bool full,
size_t word_size,
@@ -152,13 +154,25 @@ class TenuredGeneration: public Generation {
// Performance Counter support
void update_counters();
- virtual void record_spaces_top();
+ void record_spaces_top();
// Statistics
virtual void update_gc_stats(Generation* current_generation, bool full);
- virtual bool promotion_attempt_is_safe(size_t max_promoted_in_bytes) const;
+ // Returns true if promotions of the specified amount are
+ // likely to succeed without a promotion failure.
+ // Promotion of the full amount is not guaranteed but
+ // might be attempted in the worst case.
+ bool promotion_attempt_is_safe(size_t max_promoted_in_bytes) const;
+
+ // "obj" is the address of an object in young-gen. Allocate space for "obj"
+ // in the old-gen, and copy "obj" into the newly allocated space, if
+ // possible, returning the result (or null if the allocation failed).
+ //
+ // The "obj_size" argument is just obj->size(), passed along so the caller can
+ // avoid repeating the virtual call to retrieve it.
+ oop promote(oop obj, size_t obj_size);
virtual void verify();
virtual void print_on(outputStream* st) const;
diff --git a/src/hotspot/share/gc/serial/tenuredGeneration.inline.hpp b/src/hotspot/share/gc/serial/tenuredGeneration.inline.hpp
index 5d8bf32e2cec3..25d57a7628238 100644
--- a/src/hotspot/share/gc/serial/tenuredGeneration.inline.hpp
+++ b/src/hotspot/share/gc/serial/tenuredGeneration.inline.hpp
@@ -41,18 +41,10 @@ inline size_t TenuredGeneration::free() const {
return space()->free();
}
-inline MemRegion TenuredGeneration::used_region() const {
- return space()->used_region();
-}
-
inline bool TenuredGeneration::is_in(const void* p) const {
return space()->is_in(p);
}
-inline ContiguousSpace* TenuredGeneration::first_compaction_space() const {
- return space();
-}
-
HeapWord* TenuredGeneration::allocate(size_t word_size,
bool is_tlab) {
assert(!is_tlab, "TenuredGeneration does not support TLAB allocation");
@@ -65,15 +57,9 @@ HeapWord* TenuredGeneration::par_allocate(size_t word_size,
return _the_space->par_allocate(word_size);
}
-bool TenuredGeneration::block_is_obj(const HeapWord* addr) const {
- return addr < _the_space ->top();
-}
-
template
void TenuredGeneration::oop_since_save_marks_iterate(OopClosureType* blk) {
_the_space->oop_since_save_marks_iterate(blk);
-
- save_marks();
}
#endif // SHARE_GC_SERIAL_TENUREDGENERATION_INLINE_HPP
diff --git a/src/hotspot/share/gc/serial/vmStructs_serial.hpp b/src/hotspot/share/gc/serial/vmStructs_serial.hpp
index c99b90aa7d66e..9f646424100db 100644
--- a/src/hotspot/share/gc/serial/vmStructs_serial.hpp
+++ b/src/hotspot/share/gc/serial/vmStructs_serial.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2007, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,13 @@
#define VM_STRUCTS_SERIALGC(nonstatic_field, \
volatile_nonstatic_field, \
static_field) \
+ nonstatic_field(Generation, _reserved, MemRegion) \
+ nonstatic_field(Generation, _virtual_space, VirtualSpace) \
+ nonstatic_field(Generation, _stat_record, Generation::StatRecord) \
+ \
+ nonstatic_field(Generation::StatRecord, invocations, int) \
+ nonstatic_field(Generation::StatRecord, accumulated_time, elapsedTimer) \
+ \
nonstatic_field(TenuredGeneration, _rs, CardTableRS*) \
nonstatic_field(TenuredGeneration, _bts, SerialBlockOffsetSharedArray*) \
nonstatic_field(TenuredGeneration, _shrink_factor, size_t) \
@@ -40,7 +47,7 @@
nonstatic_field(TenuredGeneration, _min_heap_delta_bytes, size_t) \
nonstatic_field(TenuredGeneration, _the_space, TenuredSpace*) \
\
- nonstatic_field(DefNewGeneration, _old_gen, Generation*) \
+ nonstatic_field(DefNewGeneration, _old_gen, TenuredGeneration*) \
nonstatic_field(DefNewGeneration, _tenuring_threshold, uint) \
nonstatic_field(DefNewGeneration, _age_table, AgeTable) \
nonstatic_field(DefNewGeneration, _eden_space, ContiguousSpace*) \
@@ -53,12 +60,17 @@
nonstatic_field(SerialBlockOffsetSharedArray, _vs, VirtualSpace) \
nonstatic_field(SerialBlockOffsetSharedArray, _offset_array, u_char*) \
\
- nonstatic_field(TenuredSpace, _offsets, SerialBlockOffsetTable)
+ nonstatic_field(TenuredSpace, _offsets, SerialBlockOffsetTable) \
+ \
+ nonstatic_field(SerialHeap, _young_gen, DefNewGeneration*) \
+ nonstatic_field(SerialHeap, _old_gen, TenuredGeneration*) \
#define VM_TYPES_SERIALGC(declare_type, \
declare_toplevel_type, \
declare_integer_type) \
- declare_type(SerialHeap, GenCollectedHeap) \
+ declare_toplevel_type(Generation) \
+ declare_toplevel_type(Generation::StatRecord) \
+ declare_type(SerialHeap, CollectedHeap) \
declare_type(TenuredGeneration, Generation) \
declare_type(TenuredSpace, ContiguousSpace) \
\
@@ -70,6 +82,8 @@
declare_toplevel_type(SerialBlockOffsetTable)
#define VM_INT_CONSTANTS_SERIALGC(declare_constant, \
- declare_constant_with_value)
+ declare_constant_with_value) \
+ declare_constant(Generation::LogOfGenGrain) \
+ declare_constant(Generation::GenGrain)
#endif // SHARE_GC_SERIAL_VMSTRUCTS_SERIAL_HPP
diff --git a/src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp b/src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp
index 959fb3102589e..a9c11af570cd9 100644
--- a/src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp
+++ b/src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp
@@ -48,12 +48,6 @@ class AdaptiveSizePolicy : public CHeapObj {
virtual GCPolicyKind kind() const { return _gc_adaptive_size_policy; }
enum SizePolicyTrueValues {
- decrease_old_gen_for_throughput_true = -7,
- decrease_young_gen_for_througput_true = -6,
-
- increase_young_gen_for_min_pauses_true = -2,
- increase_old_gen_for_maj_pauses_true = -1,
-
decrease_young_gen_for_min_pauses_true = 1,
decrease_old_gen_for_maj_pauses_true = 2,
diff --git a/src/hotspot/share/gc/shared/ageTable.cpp b/src/hotspot/share/gc/shared/ageTable.cpp
index 0c2bcda73eddb..34e4bb91887a0 100644
--- a/src/hotspot/share/gc/shared/ageTable.cpp
+++ b/src/hotspot/share/gc/shared/ageTable.cpp
@@ -71,6 +71,15 @@ void AgeTable::clear() {
}
}
+#ifndef PRODUCT
+bool AgeTable::is_clear() const {
+ for (const size_t* p = sizes; p < sizes + table_size; ++p) {
+ if (*p != 0) return false;
+ }
+ return true;
+}
+#endif // !PRODUCT
+
void AgeTable::merge(const AgeTable* subTable) {
for (int i = 0; i < table_size; i++) {
sizes[i]+= subTable->sizes[i];
diff --git a/src/hotspot/share/gc/shared/ageTable.hpp b/src/hotspot/share/gc/shared/ageTable.hpp
index 9f0c10ec31203..decade6ee8581 100644
--- a/src/hotspot/share/gc/shared/ageTable.hpp
+++ b/src/hotspot/share/gc/shared/ageTable.hpp
@@ -25,6 +25,7 @@
#ifndef SHARE_GC_SHARED_AGETABLE_HPP
#define SHARE_GC_SHARED_AGETABLE_HPP
+#include "memory/allocation.hpp"
#include "oops/markWord.hpp"
#include "oops/oop.hpp"
#include "runtime/perfDataTypes.hpp"
@@ -36,7 +37,7 @@
//
// Note: all sizes are in oops
-class AgeTable {
+class AgeTable: public CHeapObj {
friend class VMStructs;
public:
@@ -53,16 +54,20 @@ class AgeTable {
// clear table
void clear();
+#ifndef PRODUCT
+ // check whether it's clear
+ bool is_clear() const;
+#endif // !PRODUCT
+
// add entry
inline void add(oop p, size_t oop_size);
void add(uint age, size_t oop_size) {
- assert(age > 0 && age < table_size, "invalid age of object");
+ assert(age < table_size, "invalid age of object");
sizes[age] += oop_size;
}
- // Merge another age table with the current one. Used
- // for parallel young generation gc.
+ // Merge another age table with the current one.
void merge(const AgeTable* subTable);
// Calculate new tenuring threshold based on age information.
diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
index 600512c456b02..bef2fc5607efc 100644
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
@@ -278,6 +278,9 @@ class BarrierSetC2: public CHeapObj {
virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { return false; }
virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return false; }
virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return false; }
+ // Estimated size of the node barrier in number of C2 Ideal nodes.
+ // This is used to guide heuristics in C2, e.g. whether to unroll a loop.
+ virtual uint estimated_barrier_size(const Node* node) const { return 0; }
enum CompilePhase {
BeforeOptimize,
diff --git a/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp b/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp
index b7d706df6e395..c4f318561d64b 100644
--- a/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp
+++ b/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -97,7 +97,7 @@ void CardTableBarrierSet::print_on(outputStream* st) const {
// to a newly allocated object along the fast-path. We
// compensate for such elided card-marks as follows:
// (a) Generational, non-concurrent collectors, such as
-// GenCollectedHeap(DefNew,Tenured) and
+// SerialHeap(DefNew,Tenured) and
// ParallelScavengeHeap(ParallelGC, ParallelOldGC)
// need the card-mark if and only if the region is
// in the old gen, and do not care if the card-mark
diff --git a/src/hotspot/share/gc/shared/collectedHeap.cpp b/src/hotspot/share/gc/shared/collectedHeap.cpp
index 1b65ee90b5d7a..02d20b92b5ce7 100644
--- a/src/hotspot/share/gc/shared/collectedHeap.cpp
+++ b/src/hotspot/share/gc/shared/collectedHeap.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -242,6 +242,7 @@ bool CollectedHeap::is_oop(oop object) const {
CollectedHeap::CollectedHeap() :
_capacity_at_last_gc(0),
_used_at_last_gc(0),
+ _soft_ref_policy(),
_is_gc_active(false),
_last_whole_heap_examined_time_ns(os::javaTimeNanos()),
_total_collections(0),
@@ -401,6 +402,13 @@ void CollectedHeap::set_gc_cause(GCCause::Cause v) {
_gc_cause = v;
}
+// Returns the header size in words aligned to the requirements of the
+// array object type.
+static int int_array_header_size() {
+ size_t typesize_in_bytes = arrayOopDesc::header_size_in_bytes();
+ return (int)align_up(typesize_in_bytes, HeapWordSize)/HeapWordSize;
+}
+
size_t CollectedHeap::max_tlab_size() const {
// TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE].
// This restriction could be removed by enabling filling with multiple arrays.
@@ -410,14 +418,14 @@ size_t CollectedHeap::max_tlab_size() const {
// We actually lose a little by dividing first,
// but that just makes the TLAB somewhat smaller than the biggest array,
// which is fine, since we'll be able to fill that.
- size_t max_int_size = typeArrayOopDesc::header_size(T_INT) +
+ size_t max_int_size = int_array_header_size() +
sizeof(jint) *
((juint) max_jint / (size_t) HeapWordSize);
return align_down(max_int_size, MinObjAlignment);
}
size_t CollectedHeap::filler_array_hdr_size() {
- return align_object_offset(arrayOopDesc::header_size(T_INT)); // align to Long
+ return align_object_offset(int_array_header_size()); // align to Long
}
size_t CollectedHeap::filler_array_min_size() {
diff --git a/src/hotspot/share/gc/shared/collectedHeap.hpp b/src/hotspot/share/gc/shared/collectedHeap.hpp
index cccc7b168832a..1562a4b43ad04 100644
--- a/src/hotspot/share/gc/shared/collectedHeap.hpp
+++ b/src/hotspot/share/gc/shared/collectedHeap.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
#include "gc/shared/gcCause.hpp"
#include "gc/shared/gcWhen.hpp"
+#include "gc/shared/softRefPolicy.hpp"
#include "gc/shared/verifyOption.hpp"
#include "memory/allocation.hpp"
#include "memory/metaspace.hpp"
@@ -45,9 +46,6 @@
// class defines the functions that a heap must implement, and contains
// infrastructure common to all heaps.
-class WorkerTask;
-class AdaptiveSizePolicy;
-class BarrierSet;
class GCHeapLog;
class GCHeapSummary;
class GCTimer;
@@ -56,7 +54,6 @@ class GCMemoryManager;
class MemoryPool;
class MetaspaceSummary;
class ReservedHeapSpace;
-class SoftRefPolicy;
class Thread;
class ThreadClosure;
class VirtualSpaceSummary;
@@ -84,8 +81,7 @@ class ParallelObjectIterator : public StackObj {
//
// CollectedHeap
-// GenCollectedHeap
-// SerialHeap
+// SerialHeap
// G1CollectedHeap
// ParallelScavengeHeap
// ShenandoahHeap
@@ -106,6 +102,8 @@ class CollectedHeap : public CHeapObj {
size_t _capacity_at_last_gc;
size_t _used_at_last_gc;
+ SoftRefPolicy _soft_ref_policy;
+
// First, set it to java_lang_Object.
// Then, set it to FillerObject after the FillerObject_klass loading is complete.
static Klass* _filler_object_klass;
@@ -404,7 +402,7 @@ class CollectedHeap : public CHeapObj {
void increment_total_full_collections() { _total_full_collections++; }
// Return the SoftRefPolicy for the heap;
- virtual SoftRefPolicy* soft_ref_policy() = 0;
+ SoftRefPolicy* soft_ref_policy() { return &_soft_ref_policy; }
virtual MemoryUsage memory_usage();
virtual GrowableArray memory_managers() = 0;
diff --git a/src/hotspot/share/gc/shared/gcId.cpp b/src/hotspot/share/gc/shared/gcId.cpp
index 646de8339104c..8d123386180cb 100644
--- a/src/hotspot/share/gc/shared/gcId.cpp
+++ b/src/hotspot/share/gc/shared/gcId.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -43,7 +43,7 @@ void GCId::set_printer(GCIdPrinter* printer) {
_printer = printer;
}
-NamedThread* currentNamedthread() {
+static NamedThread* currentNamedthread() {
assert(Thread::current()->is_Named_thread(), "This thread must be NamedThread");
return (NamedThread*)Thread::current();
}
diff --git a/src/hotspot/share/gc/shared/gcVMOperations.cpp b/src/hotspot/share/gc/shared/gcVMOperations.cpp
index 80358f38e2d60..5f17c828bd7a6 100644
--- a/src/hotspot/share/gc/shared/gcVMOperations.cpp
+++ b/src/hotspot/share/gc/shared/gcVMOperations.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,7 @@
#include "gc/shared/gcLocker.hpp"
#include "gc/shared/gcVMOperations.hpp"
#include "gc/shared/gc_globals.hpp"
-#include "gc/shared/genCollectedHeap.hpp"
+#include "gc/shared/softRefPolicy.hpp"
#include "interpreter/oopMapCache.hpp"
#include "logging/log.hpp"
#include "memory/classLoaderMetaspace.hpp"
@@ -194,28 +194,6 @@ void VM_GC_HeapInspection::doit() {
}
}
-
-void VM_GenCollectForAllocation::doit() {
- SvcGCMarker sgcm(SvcGCMarker::MINOR);
-
- GenCollectedHeap* gch = GenCollectedHeap::heap();
- GCCauseSetter gccs(gch, _gc_cause);
- _result = gch->satisfy_failed_allocation(_word_size, _tlab);
- assert(_result == nullptr || gch->is_in_reserved(_result), "result not in heap");
-
- if (_result == nullptr && GCLocker::is_active_and_needs_gc()) {
- set_gc_locked();
- }
-}
-
-void VM_GenCollectFull::doit() {
- SvcGCMarker sgcm(SvcGCMarker::FULL);
-
- GenCollectedHeap* gch = GenCollectedHeap::heap();
- GCCauseSetter gccs(gch, _gc_cause);
- gch->do_full_collection(gch->must_clear_all_soft_refs(), _max_generation);
-}
-
VM_CollectForMetadataAllocation::VM_CollectForMetadataAllocation(ClassLoaderData* loader_data,
size_t size,
Metaspace::MetadataType mdtype,
diff --git a/src/hotspot/share/gc/shared/gcVMOperations.hpp b/src/hotspot/share/gc/shared/gcVMOperations.hpp
index f4bc933331267..c7a2edb135509 100644
--- a/src/hotspot/share/gc/shared/gcVMOperations.hpp
+++ b/src/hotspot/share/gc/shared/gcVMOperations.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
#define SHARE_GC_SHARED_GCVMOPERATIONS_HPP
#include "gc/shared/collectedHeap.hpp"
-#include "gc/shared/genCollectedHeap.hpp"
+#include "gc/shared/collectorCounters.hpp"
#include "memory/metaspace.hpp"
#include "prims/jvmtiExport.hpp"
#include "runtime/handles.hpp"
@@ -192,40 +192,6 @@ class VM_CollectForAllocation : public VM_GC_Operation {
}
};
-class VM_GenCollectForAllocation : public VM_CollectForAllocation {
- private:
- bool _tlab; // alloc is of a tlab.
- public:
- VM_GenCollectForAllocation(size_t word_size,
- bool tlab,
- uint gc_count_before)
- : VM_CollectForAllocation(word_size, gc_count_before, GCCause::_allocation_failure),
- _tlab(tlab) {
- assert(word_size != 0, "An allocation should always be requested with this operation.");
- }
- ~VM_GenCollectForAllocation() {}
- virtual VMOp_Type type() const { return VMOp_GenCollectForAllocation; }
- virtual void doit();
-};
-
-// VM operation to invoke a collection of the heap as a
-// GenCollectedHeap heap.
-class VM_GenCollectFull: public VM_GC_Operation {
- private:
- GenCollectedHeap::GenerationType _max_generation;
- public:
- VM_GenCollectFull(uint gc_count_before,
- uint full_gc_count_before,
- GCCause::Cause gc_cause,
- GenCollectedHeap::GenerationType max_generation)
- : VM_GC_Operation(gc_count_before, gc_cause, full_gc_count_before,
- max_generation != GenCollectedHeap::YoungGen /* full */),
- _max_generation(max_generation) { }
- ~VM_GenCollectFull() {}
- virtual VMOp_Type type() const { return VMOp_GenCollectFull; }
- virtual void doit();
-};
-
class VM_CollectForMetadataAllocation: public VM_GC_Operation {
private:
MetaWord* _result;
diff --git a/src/hotspot/share/gc/shared/gc_globals.hpp b/src/hotspot/share/gc/shared/gc_globals.hpp
index ae8674c84a9d0..25e581cae6784 100644
--- a/src/hotspot/share/gc/shared/gc_globals.hpp
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -213,13 +213,13 @@
\
/* where does the range max value of (max_jint - 1) come from? */ \
product(size_t, MarkStackSizeMax, NOT_LP64(4*M) LP64_ONLY(512*M), \
- "Maximum size of marking stack") \
- range(1, (INT_MAX - 1)) \
+ "Maximum size of marking stack in bytes.") \
+ range(1, (INT_MAX - 1)) \
\
product(size_t, MarkStackSize, NOT_LP64(64*K) LP64_ONLY(4*M), \
- "Size of marking stack") \
+ "Size of marking stack in bytes.") \
constraint(MarkStackSizeConstraintFunc,AfterErgo) \
- range(1, (INT_MAX - 1)) \
+ range(1, (INT_MAX - 1)) \
\
product(bool, ParallelRefProcEnabled, false, \
"Enable parallel reference processing whenever possible") \
@@ -309,23 +309,6 @@
"MaxRAM * MaxRAMPercentage / 100") \
range(0, max_uintx) \
\
- product(uintx, MaxRAMFraction, 4, \
- "Maximum fraction (1/n) of real memory used for maximum heap " \
- "size. " \
- "Deprecated, use MaxRAMPercentage instead") \
- range(1, max_uintx) \
- \
- product(uintx, MinRAMFraction, 2, \
- "Minimum fraction (1/n) of real memory used for maximum heap " \
- "size on systems with small physical memory size. " \
- "Deprecated, use MinRAMPercentage instead") \
- range(1, max_uintx) \
- \
- product(uintx, InitialRAMFraction, 64, \
- "Fraction (1/n) of real memory used for initial heap size. " \
- "Deprecated, use InitialRAMPercentage instead") \
- range(1, max_uintx) \
- \
product(double, MaxRAMPercentage, 25.0, \
"Maximum percentage of real memory used for maximum heap size") \
range(0.0, 100.0) \
@@ -362,10 +345,6 @@
product(bool, UseAdaptiveSizePolicyWithSystemGC, false, \
"Include statistics from System.gc() for adaptive size policy") \
\
- develop(intx, PSAdaptiveSizePolicyResizeVirtualSpaceAlot, -1, \
- "Resize the virtual spaces of the young or old generations") \
- range(-1, 1) \
- \
product(uint, AdaptiveSizeThroughPutPolicy, 0, \
"Policy for changing generation size for throughput goals") \
range(0, 1) \
@@ -442,11 +421,6 @@
"Time slice for MMU specification") \
constraint(GCPauseIntervalMillisConstraintFunc,AfterErgo) \
\
- product(uintx, MaxGCMinorPauseMillis, max_uintx, \
- "Adaptive size policy maximum GC minor pause time goal " \
- "in millisecond") \
- range(0, max_uintx) \
- \
product(uint, GCTimeRatio, 99, \
"Adaptive size policy application time to GC time ratio") \
range(0, UINT_MAX) \
@@ -639,10 +613,6 @@
"GC invoke count where +VerifyBefore/AfterGC kicks in") \
range(0, max_uintx) \
\
- product(int, VerifyGCLevel, 0, DIAGNOSTIC, \
- "Generation level at which to start +VerifyBefore/AfterGC") \
- range(0, 1) \
- \
product(uint, MaxTenuringThreshold, 15, \
"Maximum value for tenuring threshold") \
range(0, markWord::max_age + 1) \
diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.cpp b/src/hotspot/share/gc/shared/genCollectedHeap.cpp
deleted file mode 100644
index cf101caae38b3..0000000000000
--- a/src/hotspot/share/gc/shared/genCollectedHeap.cpp
+++ /dev/null
@@ -1,1035 +0,0 @@
-/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "classfile/classLoaderDataGraph.hpp"
-#include "classfile/stringTable.hpp"
-#include "classfile/symbolTable.hpp"
-#include "classfile/vmSymbols.hpp"
-#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
-#include "compiler/oopMap.hpp"
-#include "gc/serial/cardTableRS.hpp"
-#include "gc/serial/defNewGeneration.hpp"
-#include "gc/serial/genMarkSweep.hpp"
-#include "gc/serial/markSweep.hpp"
-#include "gc/serial/tenuredGeneration.hpp"
-#include "gc/shared/cardTableBarrierSet.hpp"
-#include "gc/shared/classUnloadingContext.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "gc/shared/collectorCounters.hpp"
-#include "gc/shared/continuationGCSupport.inline.hpp"
-#include "gc/shared/gcId.hpp"
-#include "gc/shared/gcInitLogger.hpp"
-#include "gc/shared/gcLocker.hpp"
-#include "gc/shared/gcPolicyCounters.hpp"
-#include "gc/shared/gcTrace.hpp"
-#include "gc/shared/gcTraceTime.inline.hpp"
-#include "gc/shared/gcVMOperations.hpp"
-#include "gc/shared/genArguments.hpp"
-#include "gc/shared/genCollectedHeap.hpp"
-#include "gc/shared/locationPrinter.inline.hpp"
-#include "gc/shared/oopStorage.inline.hpp"
-#include "gc/shared/oopStorageParState.inline.hpp"
-#include "gc/shared/oopStorageSet.inline.hpp"
-#include "gc/shared/scavengableNMethods.hpp"
-#include "gc/shared/space.hpp"
-#include "gc/shared/strongRootsScope.hpp"
-#include "gc/shared/weakProcessor.hpp"
-#include "gc/shared/workerThread.hpp"
-#include "memory/iterator.hpp"
-#include "memory/metaspaceCounters.hpp"
-#include "memory/metaspaceUtils.hpp"
-#include "memory/resourceArea.hpp"
-#include "memory/universe.hpp"
-#include "oops/oop.inline.hpp"
-#include "runtime/handles.hpp"
-#include "runtime/handles.inline.hpp"
-#include "runtime/java.hpp"
-#include "runtime/threads.hpp"
-#include "runtime/vmThread.hpp"
-#include "services/memoryService.hpp"
-#include "utilities/autoRestore.hpp"
-#include "utilities/debug.hpp"
-#include "utilities/formatBuffer.hpp"
-#include "utilities/macros.hpp"
-#include "utilities/stack.inline.hpp"
-#include "utilities/vmError.hpp"
-#if INCLUDE_JVMCI
-#include "jvmci/jvmci.hpp"
-#endif
-
-GenCollectedHeap::GenCollectedHeap(Generation::Name young,
- Generation::Name old,
- const char* policy_counters_name) :
- CollectedHeap(),
- _young_gen(nullptr),
- _old_gen(nullptr),
- _rem_set(nullptr),
- _soft_ref_policy(),
- _gc_policy_counters(new GCPolicyCounters(policy_counters_name, 2, 2)),
- _incremental_collection_failed(false),
- _full_collections_completed(0),
- _young_manager(nullptr),
- _old_manager(nullptr) {
-}
-
-jint GenCollectedHeap::initialize() {
- // Allocate space for the heap.
-
- ReservedHeapSpace heap_rs = allocate(HeapAlignment);
-
- if (!heap_rs.is_reserved()) {
- vm_shutdown_during_initialization(
- "Could not reserve enough space for object heap");
- return JNI_ENOMEM;
- }
-
- initialize_reserved_region(heap_rs);
-
- ReservedSpace young_rs = heap_rs.first_part(MaxNewSize);
- ReservedSpace old_rs = heap_rs.last_part(MaxNewSize);
-
- _rem_set = create_rem_set(heap_rs.region());
- _rem_set->initialize(young_rs.base(), old_rs.base());
-
- CardTableBarrierSet *bs = new CardTableBarrierSet(_rem_set);
- bs->initialize();
- BarrierSet::set_barrier_set(bs);
-
- _young_gen = new DefNewGeneration(young_rs, NewSize, MinNewSize, MaxNewSize);
- _old_gen = new TenuredGeneration(old_rs, OldSize, MinOldSize, MaxOldSize, rem_set());
-
- GCInitLogger::print();
-
- return JNI_OK;
-}
-
-CardTableRS* GenCollectedHeap::create_rem_set(const MemRegion& reserved_region) {
- return new CardTableRS(reserved_region);
-}
-
-ReservedHeapSpace GenCollectedHeap::allocate(size_t alignment) {
- // Now figure out the total size.
- const size_t pageSize = UseLargePages ? os::large_page_size() : os::vm_page_size();
- assert(alignment % pageSize == 0, "Must be");
-
- // Check for overflow.
- size_t total_reserved = MaxNewSize + MaxOldSize;
- if (total_reserved < MaxNewSize) {
- vm_exit_during_initialization("The size of the object heap + VM data exceeds "
- "the maximum representable size");
- }
- assert(total_reserved % alignment == 0,
- "Gen size; total_reserved=" SIZE_FORMAT ", alignment="
- SIZE_FORMAT, total_reserved, alignment);
-
- ReservedHeapSpace heap_rs = Universe::reserve_heap(total_reserved, alignment);
- size_t used_page_size = heap_rs.page_size();
-
- os::trace_page_sizes("Heap",
- MinHeapSize,
- total_reserved,
- heap_rs.base(),
- heap_rs.size(),
- used_page_size);
-
- return heap_rs;
-}
-
-class GenIsScavengable : public BoolObjectClosure {
-public:
- bool do_object_b(oop obj) {
- return GenCollectedHeap::heap()->is_in_young(obj);
- }
-};
-
-static GenIsScavengable _is_scavengable;
-
-void GenCollectedHeap::post_initialize() {
- CollectedHeap::post_initialize();
-
- DefNewGeneration* def_new_gen = (DefNewGeneration*)_young_gen;
-
- def_new_gen->ref_processor_init();
-
- MarkSweep::initialize();
-
- ScavengableNMethods::initialize(&_is_scavengable);
-}
-
-PreGenGCValues GenCollectedHeap::get_pre_gc_values() const {
- const DefNewGeneration* const def_new_gen = (DefNewGeneration*) young_gen();
-
- return PreGenGCValues(def_new_gen->used(),
- def_new_gen->capacity(),
- def_new_gen->eden()->used(),
- def_new_gen->eden()->capacity(),
- def_new_gen->from()->used(),
- def_new_gen->from()->capacity(),
- old_gen()->used(),
- old_gen()->capacity());
-}
-
-size_t GenCollectedHeap::capacity() const {
- return _young_gen->capacity() + _old_gen->capacity();
-}
-
-size_t GenCollectedHeap::used() const {
- return _young_gen->used() + _old_gen->used();
-}
-
-void GenCollectedHeap::save_used_regions() {
- _old_gen->save_used_region();
- _young_gen->save_used_region();
-}
-
-size_t GenCollectedHeap::max_capacity() const {
- return _young_gen->max_capacity() + _old_gen->max_capacity();
-}
-
-// Update the _full_collections_completed counter
-// at the end of a stop-world full GC.
-unsigned int GenCollectedHeap::update_full_collections_completed() {
- assert(_full_collections_completed <= _total_full_collections,
- "Can't complete more collections than were started");
- _full_collections_completed = _total_full_collections;
- return _full_collections_completed;
-}
-
-// Return true if any of the following is true:
-// . the allocation won't fit into the current young gen heap
-// . gc locker is occupied (jni critical section)
-// . heap memory is tight -- the most recent previous collection
-// was a full collection because a partial collection (would
-// have) failed and is likely to fail again
-bool GenCollectedHeap::should_try_older_generation_allocation(size_t word_size) const {
- size_t young_capacity = _young_gen->capacity_before_gc();
- return (word_size > heap_word_size(young_capacity))
- || GCLocker::is_active_and_needs_gc()
- || incremental_collection_failed();
-}
-
-HeapWord* GenCollectedHeap::expand_heap_and_allocate(size_t size, bool is_tlab) {
- HeapWord* result = nullptr;
- if (_old_gen->should_allocate(size, is_tlab)) {
- result = _old_gen->expand_and_allocate(size, is_tlab);
- }
- if (result == nullptr) {
- if (_young_gen->should_allocate(size, is_tlab)) {
- result = _young_gen->expand_and_allocate(size, is_tlab);
- }
- }
- assert(result == nullptr || is_in_reserved(result), "result not in heap");
- return result;
-}
-
-HeapWord* GenCollectedHeap::mem_allocate_work(size_t size,
- bool is_tlab) {
-
- HeapWord* result = nullptr;
-
- // Loop until the allocation is satisfied, or unsatisfied after GC.
- for (uint try_count = 1, gclocker_stalled_count = 0; /* return or throw */; try_count += 1) {
-
- // First allocation attempt is lock-free.
- Generation *young = _young_gen;
- if (young->should_allocate(size, is_tlab)) {
- result = young->par_allocate(size, is_tlab);
- if (result != nullptr) {
- assert(is_in_reserved(result), "result not in heap");
- return result;
- }
- }
- uint gc_count_before; // Read inside the Heap_lock locked region.
- {
- MutexLocker ml(Heap_lock);
- log_trace(gc, alloc)("GenCollectedHeap::mem_allocate_work: attempting locked slow path allocation");
- // Note that only large objects get a shot at being
- // allocated in later generations.
- bool first_only = !should_try_older_generation_allocation(size);
-
- result = attempt_allocation(size, is_tlab, first_only);
- if (result != nullptr) {
- assert(is_in_reserved(result), "result not in heap");
- return result;
- }
-
- if (GCLocker::is_active_and_needs_gc()) {
- if (is_tlab) {
- return nullptr; // Caller will retry allocating individual object.
- }
- if (!is_maximal_no_gc()) {
- // Try and expand heap to satisfy request.
- result = expand_heap_and_allocate(size, is_tlab);
- // Result could be null if we are out of space.
- if (result != nullptr) {
- return result;
- }
- }
-
- if (gclocker_stalled_count > GCLockerRetryAllocationCount) {
- return nullptr; // We didn't get to do a GC and we didn't get any memory.
- }
-
- // If this thread is not in a jni critical section, we stall
- // the requestor until the critical section has cleared and
- // GC allowed. When the critical section clears, a GC is
- // initiated by the last thread exiting the critical section; so
- // we retry the allocation sequence from the beginning of the loop,
- // rather than causing more, now probably unnecessary, GC attempts.
- JavaThread* jthr = JavaThread::current();
- if (!jthr->in_critical()) {
- MutexUnlocker mul(Heap_lock);
- // Wait for JNI critical section to be exited
- GCLocker::stall_until_clear();
- gclocker_stalled_count += 1;
- continue;
- } else {
- if (CheckJNICalls) {
- fatal("Possible deadlock due to allocating while"
- " in jni critical section");
- }
- return nullptr;
- }
- }
-
- // Read the gc count while the heap lock is held.
- gc_count_before = total_collections();
- }
-
- VM_GenCollectForAllocation op(size, is_tlab, gc_count_before);
- VMThread::execute(&op);
- if (op.prologue_succeeded()) {
- result = op.result();
- if (op.gc_locked()) {
- assert(result == nullptr, "must be null if gc_locked() is true");
- continue; // Retry and/or stall as necessary.
- }
-
- assert(result == nullptr || is_in_reserved(result),
- "result not in heap");
- return result;
- }
-
- // Give a warning if we seem to be looping forever.
- if ((QueuedAllocationWarningCount > 0) &&
- (try_count % QueuedAllocationWarningCount == 0)) {
- log_warning(gc, ergo)("GenCollectedHeap::mem_allocate_work retries %d times,"
- " size=" SIZE_FORMAT " %s", try_count, size, is_tlab ? "(TLAB)" : "");
- }
- }
-}
-
-HeapWord* GenCollectedHeap::attempt_allocation(size_t size,
- bool is_tlab,
- bool first_only) {
- HeapWord* res = nullptr;
-
- if (_young_gen->should_allocate(size, is_tlab)) {
- res = _young_gen->allocate(size, is_tlab);
- if (res != nullptr || first_only) {
- return res;
- }
- }
-
- if (_old_gen->should_allocate(size, is_tlab)) {
- res = _old_gen->allocate(size, is_tlab);
- }
-
- return res;
-}
-
-HeapWord* GenCollectedHeap::mem_allocate(size_t size,
- bool* gc_overhead_limit_was_exceeded) {
- return mem_allocate_work(size,
- false /* is_tlab */);
-}
-
-bool GenCollectedHeap::must_clear_all_soft_refs() {
- return _gc_cause == GCCause::_metadata_GC_clear_soft_refs ||
- _gc_cause == GCCause::_wb_full_gc;
-}
-
-void GenCollectedHeap::collect_generation(Generation* gen, bool full, size_t size,
- bool is_tlab, bool run_verification, bool clear_soft_refs) {
- FormatBuffer<> title("Collect gen: %s", gen->short_name());
- GCTraceTime(Trace, gc, phases) t1(title);
- TraceCollectorStats tcs(gen->counters());
- TraceMemoryManagerStats tmms(gen->gc_manager(), gc_cause(), heap()->is_young_gen(gen) ? "end of minor GC" : "end of major GC");
-
- gen->stat_record()->invocations++;
- gen->stat_record()->accumulated_time.start();
-
- // Must be done anew before each collection because
- // a previous collection will do mangling and will
- // change top of some spaces.
- record_gen_tops_before_GC();
-
- log_trace(gc)("%s invoke=%d size=" SIZE_FORMAT, heap()->is_young_gen(gen) ? "Young" : "Old", gen->stat_record()->invocations, size * HeapWordSize);
-
- if (run_verification && VerifyBeforeGC) {
- Universe::verify("Before GC");
- }
- COMPILER2_OR_JVMCI_PRESENT(DerivedPointerTable::clear());
-
- // Do collection work
- {
- save_marks(); // save marks for all gens
-
- gen->collect(full, clear_soft_refs, size, is_tlab);
- }
-
- COMPILER2_OR_JVMCI_PRESENT(DerivedPointerTable::update_pointers());
-
- gen->stat_record()->accumulated_time.stop();
-
- update_gc_stats(gen, full);
-
- if (run_verification && VerifyAfterGC) {
- Universe::verify("After GC");
- }
-}
-
-void GenCollectedHeap::do_collection(bool full,
- bool clear_all_soft_refs,
- size_t size,
- bool is_tlab,
- GenerationType max_generation) {
- ResourceMark rm;
- DEBUG_ONLY(Thread* my_thread = Thread::current();)
-
- assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
- assert(my_thread->is_VM_thread(), "only VM thread");
- assert(Heap_lock->is_locked(),
- "the requesting thread should have the Heap_lock");
- guarantee(!is_gc_active(), "collection is not reentrant");
-
- if (GCLocker::check_active_before_gc()) {
- return; // GC is disabled (e.g. JNI GetXXXCritical operation)
- }
-
- const bool do_clear_all_soft_refs = clear_all_soft_refs ||
- soft_ref_policy()->should_clear_all_soft_refs();
-
- ClearedAllSoftRefs casr(do_clear_all_soft_refs, soft_ref_policy());
-
- AutoModifyRestore temporarily(_is_gc_active, true);
-
- bool complete = full && (max_generation == OldGen);
- bool old_collects_young = complete && !ScavengeBeforeFullGC;
- bool do_young_collection = !old_collects_young && _young_gen->should_collect(full, size, is_tlab);
-
- const PreGenGCValues pre_gc_values = get_pre_gc_values();
-
- bool run_verification = total_collections() >= VerifyGCStartAt;
- bool prepared_for_verification = false;
- bool do_full_collection = false;
-
- if (do_young_collection) {
- GCIdMark gc_id_mark;
- GCTraceCPUTime tcpu(((DefNewGeneration*)_young_gen)->gc_tracer());
- GCTraceTime(Info, gc) t("Pause Young", nullptr, gc_cause(), true);
-
- print_heap_before_gc();
-
- if (run_verification && VerifyGCLevel <= 0 && VerifyBeforeGC) {
- prepare_for_verify();
- prepared_for_verification = true;
- }
-
- gc_prologue(complete);
- increment_total_collections(complete);
-
- collect_generation(_young_gen,
- full,
- size,
- is_tlab,
- run_verification && VerifyGCLevel <= 0,
- do_clear_all_soft_refs);
-
- if (size > 0 && (!is_tlab || _young_gen->supports_tlab_allocation()) &&
- size * HeapWordSize <= _young_gen->unsafe_max_alloc_nogc()) {
- // Allocation request was met by young GC.
- size = 0;
- }
-
- // Ask if young collection is enough. If so, do the final steps for young collection,
- // and fallthrough to the end.
- do_full_collection = should_do_full_collection(size, full, is_tlab, max_generation);
- if (!do_full_collection) {
- // Adjust generation sizes.
- _young_gen->compute_new_size();
-
- print_heap_change(pre_gc_values);
-
- // Track memory usage and detect low memory after GC finishes
- MemoryService::track_memory_usage();
-
- gc_epilogue(complete);
- }
-
- print_heap_after_gc();
-
- } else {
- // No young collection, ask if we need to perform Full collection.
- do_full_collection = should_do_full_collection(size, full, is_tlab, max_generation);
- }
-
- if (do_full_collection) {
- GCIdMark gc_id_mark;
- GCTraceCPUTime tcpu(GenMarkSweep::gc_tracer());
- GCTraceTime(Info, gc) t("Pause Full", nullptr, gc_cause(), true);
-
- print_heap_before_gc();
-
- if (!prepared_for_verification && run_verification &&
- VerifyGCLevel <= 1 && VerifyBeforeGC) {
- prepare_for_verify();
- }
-
- if (!do_young_collection) {
- gc_prologue(complete);
- increment_total_collections(complete);
- }
-
- // Accounting quirk: total full collections would be incremented when "complete"
- // is set, by calling increment_total_collections above. However, we also need to
- // account Full collections that had "complete" unset.
- if (!complete) {
- increment_total_full_collections();
- }
-
- CodeCache::on_gc_marking_cycle_start();
-
- ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
- false /* unregister_nmethods_during_purge */,
- false /* lock_codeblob_free_separately */);
-
- collect_generation(_old_gen,
- full,
- size,
- is_tlab,
- run_verification && VerifyGCLevel <= 1,
- do_clear_all_soft_refs);
-
- CodeCache::on_gc_marking_cycle_finish();
- CodeCache::arm_all_nmethods();
-
- // Adjust generation sizes.
- _old_gen->compute_new_size();
- _young_gen->compute_new_size();
-
- // Delete metaspaces for unloaded class loaders and clean up loader_data graph
- ClassLoaderDataGraph::purge(true /* at_safepoint */);
- DEBUG_ONLY(MetaspaceUtils::verify();)
-
- // Need to clear claim bits for the next mark.
- ClassLoaderDataGraph::clear_claimed_marks();
-
- // Resize the metaspace capacity after full collections
- MetaspaceGC::compute_new_size();
- update_full_collections_completed();
-
- print_heap_change(pre_gc_values);
-
- // Track memory usage and detect low memory after GC finishes
- MemoryService::track_memory_usage();
-
- // Need to tell the epilogue code we are done with Full GC, regardless what was
- // the initial value for "complete" flag.
- gc_epilogue(true);
-
- print_heap_after_gc();
- }
-}
-
-bool GenCollectedHeap::should_do_full_collection(size_t size, bool full, bool is_tlab,
- GenCollectedHeap::GenerationType max_gen) const {
- return max_gen == OldGen && _old_gen->should_collect(full, size, is_tlab);
-}
-
-void GenCollectedHeap::register_nmethod(nmethod* nm) {
- ScavengableNMethods::register_nmethod(nm);
-}
-
-void GenCollectedHeap::unregister_nmethod(nmethod* nm) {
- ScavengableNMethods::unregister_nmethod(nm);
-}
-
-void GenCollectedHeap::verify_nmethod(nmethod* nm) {
- ScavengableNMethods::verify_nmethod(nm);
-}
-
-void GenCollectedHeap::prune_scavengable_nmethods() {
- ScavengableNMethods::prune_nmethods_not_into_young();
-}
-
-void GenCollectedHeap::prune_unlinked_nmethods() {
- ScavengableNMethods::prune_unlinked_nmethods();
-}
-
-HeapWord* GenCollectedHeap::satisfy_failed_allocation(size_t size, bool is_tlab) {
- GCCauseSetter x(this, GCCause::_allocation_failure);
- HeapWord* result = nullptr;
-
- assert(size != 0, "Precondition violated");
- if (GCLocker::is_active_and_needs_gc()) {
- // GC locker is active; instead of a collection we will attempt
- // to expand the heap, if there's room for expansion.
- if (!is_maximal_no_gc()) {
- result = expand_heap_and_allocate(size, is_tlab);
- }
- return result; // Could be null if we are out of space.
- } else if (!incremental_collection_will_fail(false /* don't consult_young */)) {
- // Do an incremental collection.
- do_collection(false, // full
- false, // clear_all_soft_refs
- size, // size
- is_tlab, // is_tlab
- GenCollectedHeap::OldGen); // max_generation
- } else {
- log_trace(gc)(" :: Trying full because partial may fail :: ");
- // Try a full collection; see delta for bug id 6266275
- // for the original code and why this has been simplified
- // with from-space allocation criteria modified and
- // such allocation moved out of the safepoint path.
- do_collection(true, // full
- false, // clear_all_soft_refs
- size, // size
- is_tlab, // is_tlab
- GenCollectedHeap::OldGen); // max_generation
- }
-
- result = attempt_allocation(size, is_tlab, false /*first_only*/);
-
- if (result != nullptr) {
- assert(is_in_reserved(result), "result not in heap");
- return result;
- }
-
- // OK, collection failed, try expansion.
- result = expand_heap_and_allocate(size, is_tlab);
- if (result != nullptr) {
- return result;
- }
-
- // If we reach this point, we're really out of memory. Try every trick
- // we can to reclaim memory. Force collection of soft references. Force
- // a complete compaction of the heap. Any additional methods for finding
- // free memory should be here, especially if they are expensive. If this
- // attempt fails, an OOM exception will be thrown.
- {
- UIntFlagSetting flag_change(MarkSweepAlwaysCompactCount, 1); // Make sure the heap is fully compacted
-
- do_collection(true, // full
- true, // clear_all_soft_refs
- size, // size
- is_tlab, // is_tlab
- GenCollectedHeap::OldGen); // max_generation
- }
-
- result = attempt_allocation(size, is_tlab, false /* first_only */);
- if (result != nullptr) {
- assert(is_in_reserved(result), "result not in heap");
- return result;
- }
-
- assert(!soft_ref_policy()->should_clear_all_soft_refs(),
- "Flag should have been handled and cleared prior to this point");
-
- // What else? We might try synchronous finalization later. If the total
- // space available is large enough for the allocation, then a more
- // complete compaction phase than we've tried so far might be
- // appropriate.
- return nullptr;
-}
-
-#ifdef ASSERT
-class AssertNonScavengableClosure: public OopClosure {
-public:
- virtual void do_oop(oop* p) {
- assert(!GenCollectedHeap::heap()->is_in_partial_collection(*p),
- "Referent should not be scavengable."); }
- virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); }
-};
-static AssertNonScavengableClosure assert_is_non_scavengable_closure;
-#endif
-
-void GenCollectedHeap::process_roots(ScanningOption so,
- OopClosure* strong_roots,
- CLDClosure* strong_cld_closure,
- CLDClosure* weak_cld_closure,
- CodeBlobToOopClosure* code_roots) {
- // General roots.
- assert(code_roots != nullptr, "code root closure should always be set");
-
- ClassLoaderDataGraph::roots_cld_do(strong_cld_closure, weak_cld_closure);
-
- // Only process code roots from thread stacks if we aren't visiting the entire CodeCache anyway
- CodeBlobToOopClosure* roots_from_code_p = (so & SO_AllCodeCache) ? nullptr : code_roots;
-
- Threads::oops_do(strong_roots, roots_from_code_p);
-
- OopStorageSet::strong_oops_do(strong_roots);
-
- if (so & SO_ScavengeCodeCache) {
- assert(code_roots != nullptr, "must supply closure for code cache");
-
- // We only visit parts of the CodeCache when scavenging.
- ScavengableNMethods::nmethods_do(code_roots);
- }
- if (so & SO_AllCodeCache) {
- assert(code_roots != nullptr, "must supply closure for code cache");
-
- // CMSCollector uses this to do intermediate-strength collections.
- // We scan the entire code cache, since CodeCache::do_unloading is not called.
- CodeCache::blobs_do(code_roots);
- }
- // Verify that the code cache contents are not subject to
- // movement by a scavenging collection.
- DEBUG_ONLY(CodeBlobToOopClosure assert_code_is_non_scavengable(&assert_is_non_scavengable_closure, !CodeBlobToOopClosure::FixRelocations));
- DEBUG_ONLY(ScavengableNMethods::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable));
-}
-
-bool GenCollectedHeap::no_allocs_since_save_marks() {
- return _young_gen->no_allocs_since_save_marks() &&
- _old_gen->no_allocs_since_save_marks();
-}
-
-// public collection interfaces
-void GenCollectedHeap::collect(GCCause::Cause cause) {
- // The caller doesn't have the Heap_lock
- assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");
-
- unsigned int gc_count_before;
- unsigned int full_gc_count_before;
-
- {
- MutexLocker ml(Heap_lock);
- // Read the GC count while holding the Heap_lock
- gc_count_before = total_collections();
- full_gc_count_before = total_full_collections();
- }
-
- if (GCLocker::should_discard(cause, gc_count_before)) {
- return;
- }
-
- bool should_run_young_gc = (cause == GCCause::_wb_young_gc)
- || (cause == GCCause::_gc_locker)
- DEBUG_ONLY(|| (cause == GCCause::_scavenge_alot));
-
- const GenerationType max_generation = should_run_young_gc
- ? YoungGen
- : OldGen;
-
- while (true) {
- VM_GenCollectFull op(gc_count_before, full_gc_count_before,
- cause, max_generation);
- VMThread::execute(&op);
-
- if (!GCCause::is_explicit_full_gc(cause)) {
- return;
- }
-
- {
- MutexLocker ml(Heap_lock);
- // Read the GC count while holding the Heap_lock
- if (full_gc_count_before != total_full_collections()) {
- return;
- }
- }
-
- if (GCLocker::is_active_and_needs_gc()) {
- // If GCLocker is active, wait until clear before retrying.
- GCLocker::stall_until_clear();
- }
- }
-}
-
-void GenCollectedHeap::do_full_collection(bool clear_all_soft_refs) {
- do_full_collection(clear_all_soft_refs, OldGen);
-}
-
-void GenCollectedHeap::do_full_collection(bool clear_all_soft_refs,
- GenerationType last_generation) {
- do_collection(true, // full
- clear_all_soft_refs, // clear_all_soft_refs
- 0, // size
- false, // is_tlab
- last_generation); // last_generation
- // Hack XXX FIX ME !!!
- // A scavenge may not have been attempted, or may have
- // been attempted and failed, because the old gen was too full
- if (gc_cause() == GCCause::_gc_locker && incremental_collection_failed()) {
- log_debug(gc, jni)("GC locker: Trying a full collection because scavenge failed");
- // This time allow the old gen to be collected as well
- do_collection(true, // full
- clear_all_soft_refs, // clear_all_soft_refs
- 0, // size
- false, // is_tlab
- OldGen); // last_generation
- }
-}
-
-bool GenCollectedHeap::is_in_young(const void* p) const {
- bool result = p < _old_gen->reserved().start();
- assert(result == _young_gen->is_in_reserved(p),
- "incorrect test - result=%d, p=" PTR_FORMAT, result, p2i(p));
- return result;
-}
-
-bool GenCollectedHeap::requires_barriers(stackChunkOop obj) const {
- return !is_in_young(obj);
-}
-
-// Returns "TRUE" iff "p" points into the committed areas of the heap.
-bool GenCollectedHeap::is_in(const void* p) const {
- return _young_gen->is_in(p) || _old_gen->is_in(p);
-}
-
-#ifdef ASSERT
-// Don't implement this by using is_in_young(). This method is used
-// in some cases to check that is_in_young() is correct.
-bool GenCollectedHeap::is_in_partial_collection(const void* p) {
- assert(is_in_reserved(p) || p == nullptr,
- "Does not work if address is non-null and outside of the heap");
- return p < _young_gen->reserved().end() && p != nullptr;
-}
-#endif
-
-void GenCollectedHeap::object_iterate(ObjectClosure* cl) {
- _young_gen->object_iterate(cl);
- _old_gen->object_iterate(cl);
-}
-
-HeapWord* GenCollectedHeap::block_start(const void* addr) const {
- assert(is_in_reserved(addr), "block_start of address outside of heap");
- if (_young_gen->is_in_reserved(addr)) {
- assert(_young_gen->is_in(addr), "addr should be in allocated part of generation");
- return _young_gen->block_start(addr);
- }
-
- assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address");
- assert(_old_gen->is_in(addr), "addr should be in allocated part of generation");
- return _old_gen->block_start(addr);
-}
-
-bool GenCollectedHeap::block_is_obj(const HeapWord* addr) const {
- assert(is_in_reserved(addr), "block_is_obj of address outside of heap");
- assert(block_start(addr) == addr, "addr must be a block start");
- if (_young_gen->is_in_reserved(addr)) {
- return _young_gen->block_is_obj(addr);
- }
-
- assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address");
- return _old_gen->block_is_obj(addr);
-}
-
-size_t GenCollectedHeap::tlab_capacity(Thread* thr) const {
- assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!");
- assert(_young_gen->supports_tlab_allocation(), "Young gen doesn't support TLAB allocation?!");
- return _young_gen->tlab_capacity();
-}
-
-size_t GenCollectedHeap::tlab_used(Thread* thr) const {
- assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!");
- assert(_young_gen->supports_tlab_allocation(), "Young gen doesn't support TLAB allocation?!");
- return _young_gen->tlab_used();
-}
-
-size_t GenCollectedHeap::unsafe_max_tlab_alloc(Thread* thr) const {
- assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!");
- assert(_young_gen->supports_tlab_allocation(), "Young gen doesn't support TLAB allocation?!");
- return _young_gen->unsafe_max_tlab_alloc();
-}
-
-HeapWord* GenCollectedHeap::allocate_new_tlab(size_t min_size,
- size_t requested_size,
- size_t* actual_size) {
- HeapWord* result = mem_allocate_work(requested_size /* size */,
- true /* is_tlab */);
- if (result != nullptr) {
- *actual_size = requested_size;
- }
-
- return result;
-}
-
-void GenCollectedHeap::prepare_for_verify() {
- ensure_parsability(false); // no need to retire TLABs
-}
-
-void GenCollectedHeap::generation_iterate(GenClosure* cl,
- bool old_to_young) {
- if (old_to_young) {
- cl->do_generation(_old_gen);
- cl->do_generation(_young_gen);
- } else {
- cl->do_generation(_young_gen);
- cl->do_generation(_old_gen);
- }
-}
-
-bool GenCollectedHeap::is_maximal_no_gc() const {
- return _young_gen->is_maximal_no_gc() && _old_gen->is_maximal_no_gc();
-}
-
-void GenCollectedHeap::save_marks() {
- _young_gen->save_marks();
- _old_gen->save_marks();
-}
-
-GenCollectedHeap* GenCollectedHeap::heap() {
- // SerialHeap is the only subtype of GenCollectedHeap.
- return named_heap(CollectedHeap::Serial);
-}
-
-void GenCollectedHeap::verify(VerifyOption option /* ignored */) {
- log_debug(gc, verify)("%s", _old_gen->name());
- _old_gen->verify();
-
- log_debug(gc, verify)("%s", _young_gen->name());
- _young_gen->verify();
-
- log_debug(gc, verify)("RemSet");
- rem_set()->verify();
-}
-
-void GenCollectedHeap::print_on(outputStream* st) const {
- if (_young_gen != nullptr) {
- _young_gen->print_on(st);
- }
- if (_old_gen != nullptr) {
- _old_gen->print_on(st);
- }
- MetaspaceUtils::print_on(st);
-}
-
-void GenCollectedHeap::gc_threads_do(ThreadClosure* tc) const {
-}
-
-bool GenCollectedHeap::print_location(outputStream* st, void* addr) const {
- return BlockLocationPrinter::print_location(st, addr);
-}
-
-void GenCollectedHeap::print_tracing_info() const {
- if (log_is_enabled(Debug, gc, heap, exit)) {
- LogStreamHandle(Debug, gc, heap, exit) lsh;
- _young_gen->print_summary_info_on(&lsh);
- _old_gen->print_summary_info_on(&lsh);
- }
-}
-
-void GenCollectedHeap::print_heap_change(const PreGenGCValues& pre_gc_values) const {
- const DefNewGeneration* const def_new_gen = (DefNewGeneration*) young_gen();
-
- log_info(gc, heap)(HEAP_CHANGE_FORMAT" "
- HEAP_CHANGE_FORMAT" "
- HEAP_CHANGE_FORMAT,
- HEAP_CHANGE_FORMAT_ARGS(def_new_gen->short_name(),
- pre_gc_values.young_gen_used(),
- pre_gc_values.young_gen_capacity(),
- def_new_gen->used(),
- def_new_gen->capacity()),
- HEAP_CHANGE_FORMAT_ARGS("Eden",
- pre_gc_values.eden_used(),
- pre_gc_values.eden_capacity(),
- def_new_gen->eden()->used(),
- def_new_gen->eden()->capacity()),
- HEAP_CHANGE_FORMAT_ARGS("From",
- pre_gc_values.from_used(),
- pre_gc_values.from_capacity(),
- def_new_gen->from()->used(),
- def_new_gen->from()->capacity()));
- log_info(gc, heap)(HEAP_CHANGE_FORMAT,
- HEAP_CHANGE_FORMAT_ARGS(old_gen()->short_name(),
- pre_gc_values.old_gen_used(),
- pre_gc_values.old_gen_capacity(),
- old_gen()->used(),
- old_gen()->capacity()));
- MetaspaceUtils::print_metaspace_change(pre_gc_values.metaspace_sizes());
-}
-
-class GenGCPrologueClosure: public GenCollectedHeap::GenClosure {
- private:
- bool _full;
- public:
- void do_generation(Generation* gen) {
- gen->gc_prologue(_full);
- }
- GenGCPrologueClosure(bool full) : _full(full) {};
-};
-
-void GenCollectedHeap::gc_prologue(bool full) {
- assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
-
- // Fill TLAB's and such
- ensure_parsability(true); // retire TLABs
-
- // Walk generations
- GenGCPrologueClosure blk(full);
- generation_iterate(&blk, false); // not old-to-young.
-};
-
-class GenGCEpilogueClosure: public GenCollectedHeap::GenClosure {
- private:
- bool _full;
- public:
- void do_generation(Generation* gen) {
- gen->gc_epilogue(_full);
- }
- GenGCEpilogueClosure(bool full) : _full(full) {};
-};
-
-void GenCollectedHeap::gc_epilogue(bool full) {
-#if COMPILER2_OR_JVMCI
- assert(DerivedPointerTable::is_empty(), "derived pointer present");
-#endif // COMPILER2_OR_JVMCI
-
- resize_all_tlabs();
-
- GenGCEpilogueClosure blk(full);
- generation_iterate(&blk, false); // not old-to-young.
-
- MetaspaceCounters::update_performance_counters();
-};
-
-#ifndef PRODUCT
-class GenGCSaveTopsBeforeGCClosure: public GenCollectedHeap::GenClosure {
- private:
- public:
- void do_generation(Generation* gen) {
- gen->record_spaces_top();
- }
-};
-
-void GenCollectedHeap::record_gen_tops_before_GC() {
- if (ZapUnusedHeapArea) {
- GenGCSaveTopsBeforeGCClosure blk;
- generation_iterate(&blk, false); // not old-to-young.
- }
-}
-#endif // not PRODUCT
diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.hpp b/src/hotspot/share/gc/shared/genCollectedHeap.hpp
deleted file mode 100644
index 1c1759f1422be..0000000000000
--- a/src/hotspot/share/gc/shared/genCollectedHeap.hpp
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_SHARED_GENCOLLECTEDHEAP_HPP
-#define SHARE_GC_SHARED_GENCOLLECTEDHEAP_HPP
-
-#include "gc/serial/generation.hpp"
-#include "gc/shared/collectedHeap.hpp"
-#include "gc/shared/oopStorageParState.hpp"
-#include "gc/shared/preGCValues.hpp"
-#include "gc/shared/softRefPolicy.hpp"
-
-class CardTableRS;
-class GCPolicyCounters;
-
-// A "GenCollectedHeap" is a CollectedHeap that uses generational
-// collection. It has two generations, young and old.
-class GenCollectedHeap : public CollectedHeap {
- friend class Generation;
- friend class DefNewGeneration;
- friend class TenuredGeneration;
- friend class GenMarkSweep;
- friend class VM_GenCollectForAllocation;
- friend class VM_GenCollectFull;
- friend class VM_GC_HeapInspection;
- friend class VM_HeapDumper;
- friend class HeapInspection;
- friend class GCCauseSetter;
- friend class VMStructs;
-public:
- friend class VM_PopulateDumpSharedSpace;
-
- enum GenerationType {
- YoungGen,
- OldGen
- };
-
-protected:
- Generation* _young_gen;
- Generation* _old_gen;
-
-private:
- // The singleton CardTable Remembered Set.
- CardTableRS* _rem_set;
-
- SoftRefPolicy _soft_ref_policy;
-
- GCPolicyCounters* _gc_policy_counters;
-
- // Indicates that the most recent previous incremental collection failed.
- // The flag is cleared when an action is taken that might clear the
- // condition that caused that incremental collection to fail.
- bool _incremental_collection_failed;
-
- // In support of ExplicitGCInvokesConcurrent functionality
- unsigned int _full_collections_completed;
-
- // Collects the given generation.
- void collect_generation(Generation* gen, bool full, size_t size, bool is_tlab,
- bool run_verification, bool clear_soft_refs);
-
- // Reserve aligned space for the heap as needed by the contained generations.
- ReservedHeapSpace allocate(size_t alignment);
-
- PreGenGCValues get_pre_gc_values() const;
-
-protected:
-
- GCMemoryManager* _young_manager;
- GCMemoryManager* _old_manager;
-
- // Helper functions for allocation
- HeapWord* attempt_allocation(size_t size,
- bool is_tlab,
- bool first_only);
-
- // Helper function for two callbacks below.
- // Considers collection of the first max_level+1 generations.
- void do_collection(bool full,
- bool clear_all_soft_refs,
- size_t size,
- bool is_tlab,
- GenerationType max_generation);
-
- // Callback from VM_GenCollectForAllocation operation.
- // This function does everything necessary/possible to satisfy an
- // allocation request that failed in the youngest generation that should
- // have handled it (including collection, expansion, etc.)
- HeapWord* satisfy_failed_allocation(size_t size, bool is_tlab);
-
- // Callback from VM_GenCollectFull operation.
- // Perform a full collection of the first max_level+1 generations.
- void do_full_collection(bool clear_all_soft_refs) override;
- void do_full_collection(bool clear_all_soft_refs, GenerationType max_generation);
-
- // Does the "cause" of GC indicate that
- // we absolutely __must__ clear soft refs?
- bool must_clear_all_soft_refs();
-
- GenCollectedHeap(Generation::Name young,
- Generation::Name old,
- const char* policy_counters_name);
-
-public:
-
- // Returns JNI_OK on success
- jint initialize() override;
- virtual CardTableRS* create_rem_set(const MemRegion& reserved_region);
-
- // Does operations required after initialization has been done.
- void post_initialize() override;
-
- Generation* young_gen() const { return _young_gen; }
- Generation* old_gen() const { return _old_gen; }
-
- bool is_young_gen(const Generation* gen) const { return gen == _young_gen; }
- bool is_old_gen(const Generation* gen) const { return gen == _old_gen; }
-
- MemRegion reserved_region() const { return _reserved; }
- bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); }
-
- SoftRefPolicy* soft_ref_policy() override { return &_soft_ref_policy; }
-
- // Performance Counter support
- GCPolicyCounters* counters() { return _gc_policy_counters; }
-
- size_t capacity() const override;
- size_t used() const override;
-
- // Save the "used_region" for both generations.
- void save_used_regions();
-
- size_t max_capacity() const override;
-
- HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) override;
-
- // Perform a full collection of the heap; intended for use in implementing
- // "System.gc". This implies as full a collection as the CollectedHeap
- // supports. Caller does not hold the Heap_lock on entry.
- void collect(GCCause::Cause cause) override;
-
- // Returns "TRUE" iff "p" points into the committed areas of the heap.
- // The methods is_in() and is_in_youngest() may be expensive to compute
- // in general, so, to prevent their inadvertent use in product jvm's, we
- // restrict their use to assertion checking or verification only.
- bool is_in(const void* p) const override;
-
- // Returns true if p points into the reserved space for the young generation.
- // Assumes the young gen address range is less than that of the old gen.
- bool is_in_young(const void* p) const;
-
- bool requires_barriers(stackChunkOop obj) const override;
-
-#ifdef ASSERT
- bool is_in_partial_collection(const void* p);
-#endif
-
- // Optimized nmethod scanning support routines
- void register_nmethod(nmethod* nm) override;
- void unregister_nmethod(nmethod* nm) override;
- void verify_nmethod(nmethod* nm) override;
-
- void prune_scavengable_nmethods();
- void prune_unlinked_nmethods();
-
- // Iteration functions.
- void object_iterate(ObjectClosure* cl) override;
-
- // A CollectedHeap is divided into a dense sequence of "blocks"; that is,
- // each address in the (reserved) heap is a member of exactly
- // one block. The defining characteristic of a block is that it is
- // possible to find its size, and thus to progress forward to the next
- // block. (Blocks may be of different sizes.) Thus, blocks may
- // represent Java objects, or they might be free blocks in a
- // free-list-based heap (or subheap), as long as the two kinds are
- // distinguishable and the size of each is determinable.
-
- // Returns the address of the start of the "block" that contains the
- // address "addr". We say "blocks" instead of "object" since some heaps
- // may not pack objects densely; a chunk may either be an object or a
- // non-object.
- HeapWord* block_start(const void* addr) const;
-
- // Requires "addr" to be the start of a block, and returns "TRUE" iff
- // the block is an object. Assumes (and verifies in non-product
- // builds) that addr is in the allocated part of the heap and is
- // the start of a chunk.
- bool block_is_obj(const HeapWord* addr) const;
-
- // Section on TLAB's.
- size_t tlab_capacity(Thread* thr) const override;
- size_t tlab_used(Thread* thr) const override;
- size_t unsafe_max_tlab_alloc(Thread* thr) const override;
- HeapWord* allocate_new_tlab(size_t min_size,
- size_t requested_size,
- size_t* actual_size) override;
-
- // Total number of full collections completed.
- unsigned int total_full_collections_completed() {
- assert(_full_collections_completed <= _total_full_collections,
- "Can't complete more collections than were started");
- return _full_collections_completed;
- }
-
- // Update above counter, as appropriate, at the end of a stop-world GC cycle
- unsigned int update_full_collections_completed();
-
- // Update the gc statistics for each generation.
- void update_gc_stats(Generation* current_generation, bool full) {
- _old_gen->update_gc_stats(current_generation, full);
- }
-
- bool no_gc_in_progress() { return !is_gc_active(); }
-
- void prepare_for_verify() override;
- void verify(VerifyOption option) override;
-
- void print_on(outputStream* st) const override;
- void gc_threads_do(ThreadClosure* tc) const override;
- void print_tracing_info() const override;
-
- // Used to print information about locations in the hs_err file.
- bool print_location(outputStream* st, void* addr) const override;
-
- void print_heap_change(const PreGenGCValues& pre_gc_values) const;
-
- // The functions below are helper functions that a subclass of
- // "CollectedHeap" can use in the implementation of its virtual
- // functions.
-
- class GenClosure : public StackObj {
- public:
- virtual void do_generation(Generation* gen) = 0;
- };
-
- // Apply "cl.do_generation" to all generations in the heap
- // If "old_to_young" determines the order.
- void generation_iterate(GenClosure* cl, bool old_to_young);
-
- // Return "true" if all generations have reached the
- // maximal committed limit that they can reach, without a garbage
- // collection.
- virtual bool is_maximal_no_gc() const override;
-
- // This function returns the CardTableRS object that allows us to scan
- // generations in a fully generational heap.
- CardTableRS* rem_set() { return _rem_set; }
-
- // Convenience function to be used in situations where the heap type can be
- // asserted to be this type.
- static GenCollectedHeap* heap();
-
- // The ScanningOption determines which of the roots
- // the closure is applied to:
- // "SO_None" does none;
- enum ScanningOption {
- SO_None = 0x0,
- SO_AllCodeCache = 0x8,
- SO_ScavengeCodeCache = 0x10
- };
-
- protected:
- virtual void gc_prologue(bool full);
- virtual void gc_epilogue(bool full);
-
- public:
- // Apply closures on various roots in Young GC or marking/adjust phases of Full GC.
- void process_roots(ScanningOption so,
- OopClosure* strong_roots,
- CLDClosure* strong_cld_closure,
- CLDClosure* weak_cld_closure,
- CodeBlobToOopClosure* code_roots);
-
- // Set the saved marks of generations, if that makes sense.
- // In particular, if any generation might iterate over the oops
- // in other generations, it should call this method.
- void save_marks();
-
- // Returns "true" iff no allocations have occurred since the last
- // call to "save_marks".
- bool no_allocs_since_save_marks();
-
- // Returns true if an incremental collection is likely to fail.
- // We optionally consult the young gen, if asked to do so;
- // otherwise we base our answer on whether the previous incremental
- // collection attempt failed with no corrective action as of yet.
- bool incremental_collection_will_fail(bool consult_young) {
- // The first disjunct remembers if an incremental collection failed, even
- // when we thought (second disjunct) that it would not.
- return incremental_collection_failed() ||
- (consult_young && !_young_gen->collection_attempt_is_safe());
- }
-
- // If a generation bails out of an incremental collection,
- // it sets this flag.
- bool incremental_collection_failed() const {
- return _incremental_collection_failed;
- }
- void set_incremental_collection_failed() {
- _incremental_collection_failed = true;
- }
- void clear_incremental_collection_failed() {
- _incremental_collection_failed = false;
- }
-
-private:
- // Return true if an allocation should be attempted in the older generation
- // if it fails in the younger generation. Return false, otherwise.
- bool should_try_older_generation_allocation(size_t word_size) const;
-
- // Try to allocate space by expanding the heap.
- HeapWord* expand_heap_and_allocate(size_t size, bool is_tlab);
-
- HeapWord* mem_allocate_work(size_t size,
- bool is_tlab);
-
- // Save the tops of the spaces in all generations
- void record_gen_tops_before_GC() PRODUCT_RETURN;
-
- // Return true if we need to perform full collection.
- bool should_do_full_collection(size_t size, bool full,
- bool is_tlab, GenerationType max_gen) const;
-};
-
-#endif // SHARE_GC_SHARED_GENCOLLECTEDHEAP_HPP
diff --git a/src/hotspot/share/gc/shared/memAllocator.cpp b/src/hotspot/share/gc/shared/memAllocator.cpp
index e4b91413537bd..dc94b83c13998 100644
--- a/src/hotspot/share/gc/shared/memAllocator.cpp
+++ b/src/hotspot/share/gc/shared/memAllocator.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,6 +41,7 @@
#include "services/lowMemoryDetector.hpp"
#include "utilities/align.hpp"
#include "utilities/copy.hpp"
+#include "utilities/globalDefinitions.hpp"
class MemAllocator::Allocation: StackObj {
friend class MemAllocator;
@@ -408,11 +409,30 @@ oop ObjArrayAllocator::initialize(HeapWord* mem) const {
assert(_length >= 0, "length should be non-negative");
if (_do_zero) {
mem_clear(mem);
+ mem_zap_end_padding(mem);
}
arrayOopDesc::set_length(mem, _length);
return finish(mem);
}
+#ifndef PRODUCT
+void ObjArrayAllocator::mem_zap_end_padding(HeapWord* mem) const {
+ const size_t length_in_bytes = static_cast(_length) << ArrayKlass::cast(_klass)->log2_element_size();
+ const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
+ const size_t base_offset_in_bytes = arrayOopDesc::base_offset_in_bytes(element_type);
+ const size_t size_in_bytes = _word_size * BytesPerWord;
+
+ const address obj_end = reinterpret_cast(mem) + size_in_bytes;
+ const address base = reinterpret_cast(mem) + base_offset_in_bytes;
+ const address elements_end = base + length_in_bytes;
+ assert(elements_end <= obj_end, "payload must fit in object");
+ if (elements_end < obj_end) {
+ const size_t padding_in_bytes = obj_end - elements_end;
+ Copy::fill_to_bytes(elements_end, padding_in_bytes, heapPaddingByteVal);
+ }
+}
+#endif
+
oop ClassAllocator::initialize(HeapWord* mem) const {
// Set oop_size field before setting the _klass field because a
// non-null _klass field indicates that the object is parsable by
diff --git a/src/hotspot/share/gc/shared/memAllocator.hpp b/src/hotspot/share/gc/shared/memAllocator.hpp
index 93ed10dce0555..bd4ae04babfcd 100644
--- a/src/hotspot/share/gc/shared/memAllocator.hpp
+++ b/src/hotspot/share/gc/shared/memAllocator.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -98,6 +98,8 @@ class ObjArrayAllocator: public MemAllocator {
const int _length;
const bool _do_zero;
+ void mem_zap_end_padding(HeapWord* mem) const PRODUCT_RETURN;
+
public:
ObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero,
Thread* thread = Thread::current())
diff --git a/src/hotspot/share/gc/shared/parallelCleaning.cpp b/src/hotspot/share/gc/shared/parallelCleaning.cpp
index 396f7e6c31e1f..ef82c8b676ef1 100644
--- a/src/hotspot/share/gc/shared/parallelCleaning.cpp
+++ b/src/hotspot/share/gc/shared/parallelCleaning.cpp
@@ -47,7 +47,6 @@ CodeCacheUnloadingTask::CodeCacheUnloadingTask(uint num_workers, bool unloading_
CodeCacheUnloadingTask::~CodeCacheUnloadingTask() {
CodeCache::verify_clean_inline_caches();
- CodeCache::verify_icholder_relocations();
}
void CodeCacheUnloadingTask::claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods) {
diff --git a/src/hotspot/share/gc/shared/pretouchTask.cpp b/src/hotspot/share/gc/shared/pretouchTask.cpp
index 5e35baf03dede..aba7a715fd326 100644
--- a/src/hotspot/share/gc/shared/pretouchTask.cpp
+++ b/src/hotspot/share/gc/shared/pretouchTask.cpp
@@ -68,12 +68,6 @@ void PretouchTask::pretouch(const char* task_name, char* start_address, char* en
// Page-align the chunk size, so if start_address is also page-aligned (as
// is common) then there won't be any pages shared by multiple chunks.
size_t chunk_size = align_down_bounded(PretouchTask::chunk_size(), page_size);
-#ifdef LINUX
- // When using THP we need to always pre-touch using small pages as the OS will
- // initially always use small pages.
- page_size = UseTransparentHugePages ? (size_t)os::vm_page_size() : page_size;
-#endif
-
PretouchTask task(task_name, start_address, end_address, page_size, chunk_size);
size_t total_bytes = pointer_delta(end_address, start_address, sizeof(char));
diff --git a/src/hotspot/share/gc/shared/space.cpp b/src/hotspot/share/gc/shared/space.cpp
index 67d3ad5338388..7d141710d47e1 100644
--- a/src/hotspot/share/gc/shared/space.cpp
+++ b/src/hotspot/share/gc/shared/space.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,6 @@
#include "classfile/vmClasses.hpp"
#include "classfile/vmSymbols.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
-#include "gc/shared/genCollectedHeap.hpp"
#include "gc/shared/space.hpp"
#include "gc/shared/space.inline.hpp"
#include "gc/shared/spaceDecorator.inline.hpp"
@@ -41,7 +40,9 @@
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
-ContiguousSpace::ContiguousSpace(): Space(),
+ContiguousSpace::ContiguousSpace():
+ _bottom(nullptr),
+ _end(nullptr),
_next_compaction_space(nullptr),
_top(nullptr) {
_mangler = new GenSpaceMangler(this);
@@ -76,9 +77,6 @@ void ContiguousSpace::clear(bool mangle_space) {
#ifndef PRODUCT
-void ContiguousSpace::set_top_for_allocations(HeapWord* v) {
- mangler()->set_top_for_allocations(v);
-}
void ContiguousSpace::set_top_for_allocations() {
mangler()->set_top_for_allocations(top());
}
@@ -101,36 +99,14 @@ void ContiguousSpace::mangle_unused_area_complete() {
}
#endif // NOT_PRODUCT
-
-void Space::print_short() const { print_short_on(tty); }
-
-void Space::print_short_on(outputStream* st) const {
- st->print(" space " SIZE_FORMAT "K, %3d%% used", capacity() / K,
- (int) ((double) used() * 100 / capacity()));
-}
-
-void Space::print() const { print_on(tty); }
-
-void Space::print_on(outputStream* st) const {
- print_short_on(st);
- st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ")",
- p2i(bottom()), p2i(end()));
-}
+void ContiguousSpace::print() const { print_on(tty); }
void ContiguousSpace::print_on(outputStream* st) const {
- print_short_on(st);
- st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
- p2i(bottom()), p2i(top()), p2i(end()));
+ st->print_cr(" space %zuK, %3d%% used [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
+ capacity() / K, (int) ((double) used() * 100 / capacity()),
+ p2i(bottom()), p2i(top()), p2i(end()));
}
-#if INCLUDE_SERIALGC
-void TenuredSpace::print_on(outputStream* st) const {
- print_short_on(st);
- st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
- p2i(bottom()), p2i(top()), p2i(end()));
-}
-#endif
-
void ContiguousSpace::verify() const {
HeapWord* p = bottom();
HeapWord* t = top();
@@ -141,20 +117,12 @@ void ContiguousSpace::verify() const {
guarantee(p == top(), "end of last object must match end of space");
}
-bool Space::obj_is_alive(const HeapWord* p) const {
- assert (block_is_obj(p), "The address should point to an object");
- return true;
-}
-
void ContiguousSpace::object_iterate(ObjectClosure* blk) {
- if (is_empty()) return;
- object_iterate_from(bottom(), blk);
-}
-
-void ContiguousSpace::object_iterate_from(HeapWord* mark, ObjectClosure* blk) {
- while (mark < top()) {
- blk->do_object(cast_to_oop(mark));
- mark += cast_to_oop(mark)->size();
+ HeapWord* addr = bottom();
+ while (addr < top()) {
+ oop obj = cast_to_oop(addr);
+ blk->do_object(obj);
+ addr += obj->size();
}
}
@@ -177,26 +145,6 @@ HeapWord* ContiguousSpace::block_start_const(const void* p) const {
}
}
-size_t ContiguousSpace::block_size(const HeapWord* p) const {
- assert(MemRegion(bottom(), end()).contains(p),
- "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
- p2i(p), p2i(bottom()), p2i(end()));
- HeapWord* current_top = top();
- assert(p <= current_top,
- "p > current top - p: " PTR_FORMAT ", current top: " PTR_FORMAT,
- p2i(p), p2i(current_top));
- assert(p == current_top || oopDesc::is_oop(cast_to_oop(p)),
- "p (" PTR_FORMAT ") is not a block start - "
- "current_top: " PTR_FORMAT ", is_oop: %s",
- p2i(p), p2i(current_top), BOOL_TO_STR(oopDesc::is_oop(cast_to_oop(p))));
- if (p < current_top) {
- return cast_to_oop(p)->size();
- } else {
- assert(p == current_top, "just checking");
- return pointer_delta(end(), (HeapWord*) p);
- }
-}
-
// This version requires locking.
inline HeapWord* ContiguousSpace::allocate_impl(size_t size) {
assert(Heap_lock->owned_by_self() ||
@@ -206,7 +154,7 @@ inline HeapWord* ContiguousSpace::allocate_impl(size_t size) {
if (pointer_delta(end(), obj) >= size) {
HeapWord* new_top = obj + size;
set_top(new_top);
- assert(is_aligned(obj) && is_aligned(new_top), "checking alignment");
+ assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment");
return obj;
} else {
return nullptr;
@@ -224,7 +172,7 @@ inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size) {
// the old top value: the exchange succeeded
// otherwise: the new value of the top is returned.
if (result == obj) {
- assert(is_aligned(obj) && is_aligned(new_top), "checking alignment");
+ assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment");
return obj;
}
} else {
diff --git a/src/hotspot/share/gc/shared/space.hpp b/src/hotspot/share/gc/shared/space.hpp
index 85f870ba1e2af..6eb8fe9818c6b 100644
--- a/src/hotspot/share/gc/shared/space.hpp
+++ b/src/hotspot/share/gc/shared/space.hpp
@@ -45,49 +45,53 @@
// for iterating over objects and free blocks, etc.
// Forward decls.
-class Space;
class ContiguousSpace;
class Generation;
class ContiguousSpace;
class CardTableRS;
class DirtyCardToOopClosure;
+class GenSpaceMangler;
-// A Space describes a heap area. Class Space is an abstract
-// base class.
-//
-// Space supports allocation, size computation and GC support is provided.
+// A space in which the free area is contiguous. It therefore supports
+// faster allocation, and compaction.
//
// Invariant: bottom() and end() are on page_size boundaries and
// bottom() <= top() <= end()
// top() is inclusive and end() is exclusive.
-
-class Space: public CHeapObj {
+class ContiguousSpace: public CHeapObj {
friend class VMStructs;
- protected:
+
+private:
HeapWord* _bottom;
HeapWord* _end;
// Used in support of save_marks()
HeapWord* _saved_mark_word;
- Space():
- _bottom(nullptr), _end(nullptr) { }
+ ContiguousSpace* _next_compaction_space;
+
+ HeapWord* _top;
+ // A helper for mangling the unused area of the space in debug builds.
+ GenSpaceMangler* _mangler;
+
+ GenSpaceMangler* mangler() { return _mangler; }
+
+ // Allocation helpers (return null if full).
+ inline HeapWord* allocate_impl(size_t word_size);
+ inline HeapWord* par_allocate_impl(size_t word_size);
+
+public:
+ ContiguousSpace();
+ ~ContiguousSpace();
- public:
// Accessors
HeapWord* bottom() const { return _bottom; }
HeapWord* end() const { return _end; }
- virtual void set_bottom(HeapWord* value) { _bottom = value; }
- virtual void set_end(HeapWord* value) { _end = value; }
+ void set_bottom(HeapWord* value) { _bottom = value; }
+ void set_end(HeapWord* value) { _end = value; }
HeapWord* saved_mark_word() const { return _saved_mark_word; }
- void set_saved_mark_word(HeapWord* p) { _saved_mark_word = p; }
-
- // Returns a subregion of the space containing only the allocated objects in
- // the space.
- virtual MemRegion used_region() const = 0;
-
// Returns a region that is guaranteed to contain (at least) all objects
// allocated at the time of the last call to "save_marks". If the space
// initializes its DirtyCardToOopClosure's specifying the "contig" option
@@ -100,13 +104,6 @@ class Space: public CHeapObj {
return MemRegion(bottom(), saved_mark_word());
}
- // For detecting GC bugs. Should only be called at GC boundaries, since
- // some unused space may be used as scratch space during GC's.
- // We also call this when expanding a space to satisfy an allocation
- // request. See bug #4668531
- virtual void mangle_unused_area() = 0;
- virtual void mangle_unused_area_complete() = 0;
-
// Testers
bool is_empty() const { return used() == 0; }
@@ -119,91 +116,18 @@ class Space: public CHeapObj {
bool is_in(const void* p) const {
return used_region().contains(p);
}
- bool is_in(oop obj) const {
- return is_in((void*)obj);
- }
// Returns true iff the given reserved memory of the space contains the
// given address.
bool is_in_reserved(const void* p) const { return _bottom <= p && p < _end; }
- // Test whether p is double-aligned
- static bool is_aligned(void* p) {
- return ::is_aligned(p, sizeof(double));
- }
-
// Size computations. Sizes are in bytes.
- size_t capacity() const { return byte_size(bottom(), end()); }
- virtual size_t used() const = 0;
- virtual size_t free() const = 0;
-
- // Iterate over all objects in the space, calling "cl.do_object" on
- // each. Objects allocated by applications of the closure are not
- // included in the iteration.
- virtual void object_iterate(ObjectClosure* blk) = 0;
-
- // If "p" is in the space, returns the address of the start of the
- // "block" that contains "p". We say "block" instead of "object" since
- // some heaps may not pack objects densely; a chunk may either be an
- // object or a non-object. If "p" is not in the space, return null.
- virtual HeapWord* block_start_const(const void* p) const = 0;
-
- // The non-const version may have benevolent side effects on the data
- // structure supporting these calls, possibly speeding up future calls.
- // The default implementation, however, is simply to call the const
- // version.
- HeapWord* block_start(const void* p);
-
- // Requires "addr" to be the start of a chunk, and returns its size.
- // "addr + size" is required to be the start of a new chunk, or the end
- // of the active area of the heap.
- virtual size_t block_size(const HeapWord* addr) const = 0;
-
- // Requires "addr" to be the start of a block, and returns "TRUE" iff
- // the block is an object.
- virtual bool block_is_obj(const HeapWord* addr) const = 0;
-
- // Requires "addr" to be the start of a block, and returns "TRUE" iff
- // the block is an object and the object is alive.
- bool obj_is_alive(const HeapWord* addr) const;
-
- // Allocation (return null if full). Assumes the caller has established
- // mutually exclusive access to the space.
- virtual HeapWord* allocate(size_t word_size) = 0;
-
- // Allocation (return null if full). Enforces mutual exclusion internally.
- virtual HeapWord* par_allocate(size_t word_size) = 0;
+ size_t capacity() const { return byte_size(bottom(), end()); }
+ size_t used() const { return byte_size(bottom(), top()); }
+ size_t free() const { return byte_size(top(), end()); }
void print() const;
- virtual void print_on(outputStream* st) const;
- void print_short() const;
- void print_short_on(outputStream* st) const;
-};
-
-class GenSpaceMangler;
-
-// A space in which the free area is contiguous. It therefore supports
-// faster allocation, and compaction.
-class ContiguousSpace: public Space {
- friend class VMStructs;
-
-private:
- ContiguousSpace* _next_compaction_space;
-
-protected:
- HeapWord* _top;
- // A helper for mangling the unused area of the space in debug builds.
- GenSpaceMangler* _mangler;
-
- GenSpaceMangler* mangler() { return _mangler; }
-
- // Allocation helpers (return null if full).
- inline HeapWord* allocate_impl(size_t word_size);
- inline HeapWord* par_allocate_impl(size_t word_size);
-
- public:
- ContiguousSpace();
- ~ContiguousSpace();
+ void print_on(outputStream* st) const;
// Initialization.
// "initialize" should be called once on a space, before it is used for
@@ -240,19 +164,18 @@ class ContiguousSpace: public Space {
bool saved_mark_at_top() const { return saved_mark_word() == top(); }
- // In debug mode mangle (write it with a particular bit
- // pattern) the unused part of a space.
-
- // Used to save the address in a space for later use during mangling.
- void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN;
// Used to save the space's current top for later use during mangling.
void set_top_for_allocations() PRODUCT_RETURN;
+ // For detecting GC bugs. Should only be called at GC boundaries, since
+ // some unused space may be used as scratch space during GC's.
+ // We also call this when expanding a space to satisfy an allocation
+ // request. See bug #4668531
// Mangle regions in the space from the current top up to the
// previously mangled part of the space.
- void mangle_unused_area() override PRODUCT_RETURN;
+ void mangle_unused_area() PRODUCT_RETURN;
// Mangle [top, end)
- void mangle_unused_area_complete() override PRODUCT_RETURN;
+ void mangle_unused_area_complete() PRODUCT_RETURN;
// Do some sparse checking on the area that should have been mangled.
void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN;
@@ -260,20 +183,16 @@ class ContiguousSpace: public Space {
// This code may be null depending on the macro DEBUG_MANGLING.
void check_mangled_unused_area_complete() PRODUCT_RETURN;
- // Size computations: sizes in bytes.
- size_t used() const override { return byte_size(bottom(), top()); }
- size_t free() const override { return byte_size(top(), end()); }
+ MemRegion used_region() const { return MemRegion(bottom(), top()); }
- // In a contiguous space we have a more obvious bound on what parts
- // contain objects.
- MemRegion used_region() const override { return MemRegion(bottom(), top()); }
-
- // Allocation (return null if full)
- HeapWord* allocate(size_t word_size) override;
- HeapWord* par_allocate(size_t word_size) override;
+ // Allocation (return null if full). Assumes the caller has established
+ // mutually exclusive access to the space.
+ virtual HeapWord* allocate(size_t word_size);
+ // Allocation (return null if full). Enforces mutual exclusion internally.
+ virtual HeapWord* par_allocate(size_t word_size);
// Iteration
- void object_iterate(ObjectClosure* blk) override;
+ void object_iterate(ObjectClosure* blk);
// Apply "blk->do_oop" to the addresses of all reference fields in objects
// starting with the _saved_mark_word, which was noted during a generation's
@@ -285,22 +204,15 @@ class ContiguousSpace: public Space {
template
void oop_since_save_marks_iterate(OopClosureType* blk);
- // Same as object_iterate, but starting from "mark", which is required
- // to denote the start of an object. Objects allocated by
- // applications of the closure *are* included in the iteration.
- virtual void object_iterate_from(HeapWord* mark, ObjectClosure* blk);
-
- // Very inefficient implementation.
- HeapWord* block_start_const(const void* p) const override;
- size_t block_size(const HeapWord* p) const override;
- // If a block is in the allocated area, it is an object.
- bool block_is_obj(const HeapWord* p) const override { return p < top(); }
+ // If "p" is in the space, returns the address of the start of the
+ // "block" that contains "p". We say "block" instead of "object" since
+ // some heaps may not pack objects densely; a chunk may either be an
+ // object or a non-object. If "p" is not in the space, return null.
+ virtual HeapWord* block_start_const(const void* p) const;
// Addresses for inlined allocation
HeapWord** top_addr() { return &_top; }
- void print_on(outputStream* st) const override;
-
// Debugging
void verify() const;
};
@@ -329,8 +241,6 @@ class TenuredSpace: public ContiguousSpace {
inline HeapWord* par_allocate(size_t word_size) override;
inline void update_for_block(HeapWord* start, HeapWord* end);
-
- void print_on(outputStream* st) const override;
};
#endif //INCLUDE_SERIALGC
diff --git a/src/hotspot/share/gc/shared/space.inline.hpp b/src/hotspot/share/gc/shared/space.inline.hpp
index bfaf84e8fac01..e34f833775aab 100644
--- a/src/hotspot/share/gc/shared/space.inline.hpp
+++ b/src/hotspot/share/gc/shared/space.inline.hpp
@@ -34,10 +34,6 @@
#include "runtime/prefetch.inline.hpp"
#include "runtime/safepoint.hpp"
-inline HeapWord* Space::block_start(const void* p) {
- return block_start_const(p);
-}
-
#if INCLUDE_SERIALGC
inline HeapWord* TenuredSpace::allocate(size_t size) {
HeapWord* res = ContiguousSpace::allocate(size);
@@ -71,13 +67,12 @@ void ContiguousSpace::oop_since_save_marks_iterate(OopClosureType* blk) {
t = top();
while (p < t) {
Prefetch::write(p, interval);
- debug_only(HeapWord* prev = p);
oop m = cast_to_oop(p);
p += m->oop_iterate_size(blk);
}
} while (t < top());
- set_saved_mark_word(p);
+ set_saved_mark();
}
#endif // SHARE_GC_SHARED_SPACE_INLINE_HPP
diff --git a/src/hotspot/share/gc/shared/spaceDecorator.hpp b/src/hotspot/share/gc/shared/spaceDecorator.hpp
index 14ee14352d151..688be70bada39 100644
--- a/src/hotspot/share/gc/shared/spaceDecorator.hpp
+++ b/src/hotspot/share/gc/shared/spaceDecorator.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -65,7 +65,7 @@ class SpaceDecorator: public AllStatic {
// area and provides the methods for doing the piece meal mangling.
// Methods for doing spaces and full checking of the mangling are
// included. The full checking is done if DEBUG_MANGLING is defined.
-// GenSpaceMangler is used with the GenCollectedHeap collectors and
+// GenSpaceMangler is used with the SerialHeap collectors and
// MutableSpaceMangler is used with the ParallelScavengeHeap collectors.
// These subclasses abstract the differences in the types of spaces used
// by each heap.
@@ -122,7 +122,7 @@ class SpaceMangler: public CHeapObj {
class ContiguousSpace;
class MutableSpace;
-// For use with GenCollectedHeap's
+// For use with SerialHeap's
class GenSpaceMangler: public SpaceMangler {
ContiguousSpace* _sp;
diff --git a/src/hotspot/share/gc/shared/tlab_globals.hpp b/src/hotspot/share/gc/shared/tlab_globals.hpp
index 0b047b4b9e8ac..0c135d490ec2e 100644
--- a/src/hotspot/share/gc/shared/tlab_globals.hpp
+++ b/src/hotspot/share/gc/shared/tlab_globals.hpp
@@ -46,9 +46,6 @@
product(bool, ZeroTLAB, false, \
"Zero out the newly created TLAB") \
\
- product(bool, TLABStats, true, \
- "Provide more detailed and expensive TLAB statistics.") \
- \
product(size_t, MinTLABSize, 2*K, \
"Minimum allowed TLAB size (in bytes)") \
range(1, max_uintx/2) \
diff --git a/src/hotspot/share/gc/shared/vmStructs_gc.hpp b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
index a5d671a3b9295..ae9843c31eb6f 100644
--- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp
+++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,6 @@
#include "gc/shared/ageTable.hpp"
#include "gc/shared/cardTable.hpp"
#include "gc/shared/collectedHeap.hpp"
-#include "gc/shared/genCollectedHeap.hpp"
#include "gc/shared/oopStorage.hpp"
#include "gc/shared/space.hpp"
#if INCLUDE_EPSILONGC
@@ -99,24 +98,13 @@
nonstatic_field(CollectedHeap, _is_gc_active, bool) \
nonstatic_field(CollectedHeap, _total_collections, unsigned int) \
\
+ nonstatic_field(ContiguousSpace, _bottom, HeapWord*) \
+ nonstatic_field(ContiguousSpace, _end, HeapWord*) \
nonstatic_field(ContiguousSpace, _top, HeapWord*) \
nonstatic_field(ContiguousSpace, _saved_mark_word, HeapWord*) \
\
- nonstatic_field(Generation, _reserved, MemRegion) \
- nonstatic_field(Generation, _virtual_space, VirtualSpace) \
- nonstatic_field(Generation, _stat_record, Generation::StatRecord) \
- \
- nonstatic_field(Generation::StatRecord, invocations, int) \
- nonstatic_field(Generation::StatRecord, accumulated_time, elapsedTimer) \
- \
- nonstatic_field(GenCollectedHeap, _young_gen, Generation*) \
- nonstatic_field(GenCollectedHeap, _old_gen, Generation*) \
- \
nonstatic_field(MemRegion, _start, HeapWord*) \
- nonstatic_field(MemRegion, _word_size, size_t) \
- \
- nonstatic_field(Space, _bottom, HeapWord*) \
- nonstatic_field(Space, _end, HeapWord*)
+ nonstatic_field(MemRegion, _word_size, size_t)
#define VM_TYPES_GC(declare_type, \
declare_toplevel_type, \
@@ -146,10 +134,7 @@
/******************************************/ \
\
declare_toplevel_type(CollectedHeap) \
- declare_type(GenCollectedHeap, CollectedHeap) \
- declare_toplevel_type(Generation) \
- declare_toplevel_type(Space) \
- declare_type(ContiguousSpace, Space) \
+ declare_toplevel_type(ContiguousSpace) \
declare_toplevel_type(BarrierSet) \
declare_type(ModRefBarrierSet, BarrierSet) \
declare_type(CardTableBarrierSet, ModRefBarrierSet) \
@@ -160,7 +145,6 @@
\
declare_toplevel_type(AgeTable) \
declare_toplevel_type(CardTable::CardValue) \
- declare_toplevel_type(Generation::StatRecord) \
declare_toplevel_type(HeapWord) \
declare_toplevel_type(MemRegion) \
declare_toplevel_type(ThreadLocalAllocBuffer) \
@@ -175,13 +159,9 @@
declare_toplevel_type(CardTableBarrierSet**) \
declare_toplevel_type(CollectedHeap*) \
declare_toplevel_type(ContiguousSpace*) \
- declare_toplevel_type(DefNewGeneration*) \
- declare_toplevel_type(GenCollectedHeap*) \
- declare_toplevel_type(Generation*) \
declare_toplevel_type(HeapWord*) \
declare_toplevel_type(HeapWord* volatile) \
declare_toplevel_type(MemRegion*) \
- declare_toplevel_type(Space*) \
declare_toplevel_type(ThreadLocalAllocBuffer*) \
\
declare_toplevel_type(BarrierSet::FakeRtti)
@@ -220,15 +200,6 @@
declare_constant(CollectedHeap::Serial) \
declare_constant(CollectedHeap::Parallel) \
declare_constant(CollectedHeap::G1) \
- \
- /* constants from Generation::Name enum */ \
- \
- declare_constant(Generation::DefNew) \
- declare_constant(Generation::MarkSweepCompact) \
- declare_constant(Generation::Other) \
- \
- declare_constant(Generation::LogOfGenGrain) \
- declare_constant(Generation::GenGrain) \
#define VM_LONG_CONSTANTS_GC(declare_constant) \
ZGC_ONLY(VM_LONG_CONSTANTS_Z_SHARED(declare_constant))
diff --git a/src/hotspot/share/gc/shared/weakProcessor.cpp b/src/hotspot/share/gc/shared/weakProcessor.cpp
index 02ef9bbe88da8..8adf3fab7d5c1 100644
--- a/src/hotspot/share/gc/shared/weakProcessor.cpp
+++ b/src/hotspot/share/gc/shared/weakProcessor.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,7 +41,7 @@
#include "prims/jvmtiTagMap.hpp"
#endif // INCLUDE_JVMTI
-void notify_jvmti_tagmaps() {
+static void notify_jvmti_tagmaps() {
#if INCLUDE_JVMTI
// Notify JVMTI tagmaps that a STW weak reference processing might be
// clearing entries, so the tagmaps need cleaning. Doing this here allows
diff --git a/src/hotspot/share/gc/shared/workerThread.cpp b/src/hotspot/share/gc/shared/workerThread.cpp
index b64c5050a2230..49e43c284fad9 100644
--- a/src/hotspot/share/gc/shared/workerThread.cpp
+++ b/src/hotspot/share/gc/shared/workerThread.cpp
@@ -31,6 +31,7 @@
#include "runtime/init.hpp"
#include "runtime/java.hpp"
#include "runtime/os.hpp"
+#include "runtime/safepoint.hpp"
WorkerTaskDispatcher::WorkerTaskDispatcher() :
_task(nullptr),
@@ -141,40 +142,44 @@ void WorkerThreads::threads_do(ThreadClosure* tc) const {
}
}
-void WorkerThreads::set_indirectly_suspendible_threads() {
+template
+void WorkerThreads::threads_do_f(Function function) const {
+ for (uint i = 0; i < _created_workers; i++) {
+ function(_workers[i]);
+ }
+}
+
+void WorkerThreads::set_indirect_states() {
#ifdef ASSERT
- class SetIndirectlySuspendibleThreadClosure : public ThreadClosure {
- virtual void do_thread(Thread* thread) {
+ const bool is_suspendible = Thread::current()->is_suspendible_thread();
+ const bool is_safepointed = Thread::current()->is_VM_thread() && SafepointSynchronize::is_at_safepoint();
+
+ threads_do_f([&](Thread* thread) {
+ assert(!thread->is_indirectly_suspendible_thread(), "Unexpected");
+ assert(!thread->is_indirectly_safepoint_thread(), "Unexpected");
+ if (is_suspendible) {
thread->set_indirectly_suspendible_thread();
}
- };
-
- if (Thread::current()->is_suspendible_thread()) {
- SetIndirectlySuspendibleThreadClosure cl;
- threads_do(&cl);
- }
+ if (is_safepointed) {
+ thread->set_indirectly_safepoint_thread();
+ }
+ });
#endif
}
-void WorkerThreads::clear_indirectly_suspendible_threads() {
+void WorkerThreads::clear_indirect_states() {
#ifdef ASSERT
- class ClearIndirectlySuspendibleThreadClosure : public ThreadClosure {
- virtual void do_thread(Thread* thread) {
- thread->clear_indirectly_suspendible_thread();
- }
- };
-
- if (Thread::current()->is_suspendible_thread()) {
- ClearIndirectlySuspendibleThreadClosure cl;
- threads_do(&cl);
- }
+ threads_do_f([&](Thread* thread) {
+ thread->clear_indirectly_suspendible_thread();
+ thread->clear_indirectly_safepoint_thread();
+ });
#endif
}
void WorkerThreads::run_task(WorkerTask* task) {
- set_indirectly_suspendible_threads();
+ set_indirect_states();
_dispatcher.coordinator_distribute_task(task, _active_workers);
- clear_indirectly_suspendible_threads();
+ clear_indirect_states();
}
void WorkerThreads::run_task(WorkerTask* task, uint num_workers) {
diff --git a/src/hotspot/share/gc/shared/workerThread.hpp b/src/hotspot/share/gc/shared/workerThread.hpp
index 39687e2581c25..642c8d93f5926 100644
--- a/src/hotspot/share/gc/shared/workerThread.hpp
+++ b/src/hotspot/share/gc/shared/workerThread.hpp
@@ -93,8 +93,8 @@ class WorkerThreads : public CHeapObj {
WorkerThread* create_worker(uint name_suffix);
- void set_indirectly_suspendible_threads();
- void clear_indirectly_suspendible_threads();
+ void set_indirect_states();
+ void clear_indirect_states();
protected:
virtual void on_create_worker(WorkerThread* worker) {}
@@ -111,6 +111,8 @@ class WorkerThreads : public CHeapObj {
uint set_active_workers(uint num_workers);
void threads_do(ThreadClosure* tc) const;
+ template
+ void threads_do_f(Function function) const;
const char* name() const { return _name; }
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
index c9c566ad611b6..14fb038a6c6a8 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
@@ -1734,11 +1734,26 @@ bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoo
return true;
}
+bool ShenandoahBarrierC2Support::merge_point_safe(Node* region) {
+ for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
+ Node* n = region->fast_out(i);
+ if (n->is_LoadStore()) {
+ // Splitting a LoadStore node through phi, causes it to lose its SCMemProj: the split if code doesn't have support
+ // for a LoadStore at the region the if is split through because that's not expected to happen (LoadStore nodes
+ // should be between barrier nodes). It does however happen with Shenandoah though because barriers can get
+ // expanded around a LoadStore node.
+ return false;
+ }
+ }
+ return true;
+}
+
+
void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
assert(is_heap_stable_test(n), "no other tests");
if (identical_backtoback_ifs(n, phase)) {
Node* n_ctrl = n->in(0);
- if (phase->can_split_if(n_ctrl)) {
+ if (phase->can_split_if(n_ctrl) && merge_point_safe(n_ctrl)) {
IfNode* dom_if = phase->idom(n_ctrl)->as_If();
if (is_heap_stable_test(n)) {
Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
index 032f338aa8895..7a6ed74f56393 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
@@ -65,6 +65,7 @@ class ShenandoahBarrierC2Support : public AllStatic {
static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase);
static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase);
static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase);
+ static bool merge_point_safe(Node* region);
static bool identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase);
static void fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase);
static IfNode* find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase);
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
index c2620a228115f..2d5af892c809f 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
@@ -45,8 +45,6 @@ int ShenandoahHeuristics::compare_by_garbage(RegionData a, RegionData b) {
ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahSpaceInfo* space_info) :
_space_info(space_info),
_region_data(nullptr),
- _degenerated_cycles_in_a_row(0),
- _successful_cycles_in_a_row(0),
_cycle_start(os::elapsedTime()),
_last_cycle_end(0),
_gc_times_learned(0),
@@ -199,7 +197,7 @@ bool ShenandoahHeuristics::should_start_gc() {
}
bool ShenandoahHeuristics::should_degenerate_cycle() {
- return _degenerated_cycles_in_a_row <= ShenandoahFullGCThreshold;
+ return ShenandoahHeap::heap()->shenandoah_policy()->consecutive_degenerated_gc_count() <= ShenandoahFullGCThreshold;
}
void ShenandoahHeuristics::adjust_penalty(intx step) {
@@ -220,9 +218,6 @@ void ShenandoahHeuristics::adjust_penalty(intx step) {
}
void ShenandoahHeuristics::record_success_concurrent() {
- _degenerated_cycles_in_a_row = 0;
- _successful_cycles_in_a_row++;
-
_gc_time_history->add(time_since_last_gc());
_gc_times_learned++;
@@ -230,16 +225,10 @@ void ShenandoahHeuristics::record_success_concurrent() {
}
void ShenandoahHeuristics::record_success_degenerated() {
- _degenerated_cycles_in_a_row++;
- _successful_cycles_in_a_row = 0;
-
adjust_penalty(Degenerated_Penalty);
}
void ShenandoahHeuristics::record_success_full() {
- _degenerated_cycles_in_a_row = 0;
- _successful_cycles_in_a_row++;
-
adjust_penalty(Full_Penalty);
}
@@ -254,8 +243,7 @@ void ShenandoahHeuristics::record_requested_gc() {
}
bool ShenandoahHeuristics::can_unload_classes() {
- if (!ClassUnloading) return false;
- return true;
+ return ClassUnloading;
}
bool ShenandoahHeuristics::should_unload_classes() {
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
index bc6d2dc40c54c..c4bfaed400d43 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
@@ -80,9 +80,6 @@ class ShenandoahHeuristics : public CHeapObj {
RegionData* _region_data;
- uint _degenerated_cycles_in_a_row;
- uint _successful_cycles_in_a_row;
-
double _cycle_start;
double _last_cycle_end;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
index c5b6d787b95a8..c6352b2749c78 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
@@ -24,7 +24,6 @@
#include "precompiled.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
@@ -128,26 +127,17 @@ void ShenandoahCodeRoots::disarm_nmethods() {
class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
private:
bool _unloading_occurred;
- volatile bool _failed;
ShenandoahHeap* const _heap;
BarrierSetNMethod* const _bs;
- void set_failed() {
- Atomic::store(&_failed, true);
- }
-
public:
ShenandoahNMethodUnlinkClosure(bool unloading_occurred) :
_unloading_occurred(unloading_occurred),
- _failed(false),
_heap(ShenandoahHeap::heap()),
_bs(ShenandoahBarrierSet::barrier_set()->barrier_set_nmethod()) {}
virtual void do_nmethod(nmethod* nm) {
assert(_heap->is_concurrent_weak_root_in_progress(), "Only this phase");
- if (failed()) {
- return;
- }
ShenandoahNMethod* nm_data = ShenandoahNMethod::gc_data(nm);
assert(!nm_data->is_unregistered(), "Should not see unregistered entry");
@@ -170,27 +160,19 @@ class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
}
// Clear compiled ICs and exception caches
- if (!nm->unload_nmethod_caches(_unloading_occurred)) {
- set_failed();
- }
- }
-
- bool failed() const {
- return Atomic::load(&_failed);
+ nm->unload_nmethod_caches(_unloading_occurred);
}
};
class ShenandoahUnlinkTask : public WorkerTask {
private:
ShenandoahNMethodUnlinkClosure _cl;
- ICRefillVerifier* _verifier;
ShenandoahConcurrentNMethodIterator _iterator;
public:
- ShenandoahUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :
+ ShenandoahUnlinkTask(bool unloading_occurred) :
WorkerTask("Shenandoah Unlink NMethods"),
_cl(unloading_occurred),
- _verifier(verifier),
_iterator(ShenandoahCodeRoots::table()) {
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
_iterator.nmethods_do_begin();
@@ -202,35 +184,15 @@ class ShenandoahUnlinkTask : public WorkerTask {
}
virtual void work(uint worker_id) {
- ICRefillVerifierMark mark(_verifier);
_iterator.nmethods_do(&_cl);
}
-
- bool success() const {
- return !_cl.failed();
- }
};
void ShenandoahCodeRoots::unlink(WorkerThreads* workers, bool unloading_occurred) {
assert(ShenandoahHeap::heap()->unload_classes(), "Only when running concurrent class unloading");
- for (;;) {
- ICRefillVerifier verifier;
-
- {
- ShenandoahUnlinkTask task(unloading_occurred, &verifier);
- workers->run_task(&task);
- if (task.success()) {
- return;
- }
- }
-
- // Cleaning failed because we ran out of transitional IC stubs,
- // so we have to refill and try again. Refilling requires taking
- // a safepoint, so we temporarily leave the suspendible thread set.
- SuspendibleThreadSetLeaver sts;
- InlineCacheBuffer::refill_ic_stubs();
- }
+ ShenandoahUnlinkTask task(unloading_occurred);
+ workers->run_task(&task);
}
void ShenandoahCodeRoots::purge() {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
index 7a034e709367f..d14bcc39f4591 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
@@ -31,37 +31,24 @@
ShenandoahCollectorPolicy::ShenandoahCollectorPolicy() :
_success_concurrent_gcs(0),
+ _abbreviated_concurrent_gcs(0),
_success_degenerated_gcs(0),
+ _abbreviated_degenerated_gcs(0),
_success_full_gcs(0),
+ _consecutive_degenerated_gcs(0),
_alloc_failure_degenerated(0),
_alloc_failure_degenerated_upgrade_to_full(0),
- _alloc_failure_full(0),
- _explicit_concurrent(0),
- _explicit_full(0),
- _implicit_concurrent(0),
- _implicit_full(0),
- _cycle_counter(0) {
+ _alloc_failure_full(0) {
- Copy::zero_to_bytes(_degen_points, sizeof(size_t) * ShenandoahGC::_DEGENERATED_LIMIT);
+ Copy::zero_to_bytes(_degen_point_counts, sizeof(size_t) * ShenandoahGC::_DEGENERATED_LIMIT);
+ Copy::zero_to_bytes(_collection_cause_counts, sizeof(size_t) * GCCause::_last_gc_cause);
_tracer = new ShenandoahTracer();
-
-}
-
-void ShenandoahCollectorPolicy::record_explicit_to_concurrent() {
- _explicit_concurrent++;
-}
-
-void ShenandoahCollectorPolicy::record_explicit_to_full() {
- _explicit_full++;
}
-void ShenandoahCollectorPolicy::record_implicit_to_concurrent() {
- _implicit_concurrent++;
-}
-
-void ShenandoahCollectorPolicy::record_implicit_to_full() {
- _implicit_full++;
+void ShenandoahCollectorPolicy::record_collection_cause(GCCause::Cause cause) {
+ assert(cause < GCCause::_last_gc_cause, "Invalid GCCause");
+ _collection_cause_counts[cause]++;
}
void ShenandoahCollectorPolicy::record_alloc_failure_to_full() {
@@ -71,33 +58,35 @@ void ShenandoahCollectorPolicy::record_alloc_failure_to_full() {
void ShenandoahCollectorPolicy::record_alloc_failure_to_degenerated(ShenandoahGC::ShenandoahDegenPoint point) {
assert(point < ShenandoahGC::_DEGENERATED_LIMIT, "sanity");
_alloc_failure_degenerated++;
- _degen_points[point]++;
+ _degen_point_counts[point]++;
}
void ShenandoahCollectorPolicy::record_degenerated_upgrade_to_full() {
+ _consecutive_degenerated_gcs = 0;
_alloc_failure_degenerated_upgrade_to_full++;
}
-void ShenandoahCollectorPolicy::record_success_concurrent() {
+void ShenandoahCollectorPolicy::record_success_concurrent(bool is_abbreviated) {
+ _consecutive_degenerated_gcs = 0;
_success_concurrent_gcs++;
+ if (is_abbreviated) {
+ _abbreviated_concurrent_gcs++;
+ }
}
-void ShenandoahCollectorPolicy::record_success_degenerated() {
+void ShenandoahCollectorPolicy::record_success_degenerated(bool is_abbreviated) {
_success_degenerated_gcs++;
+ _consecutive_degenerated_gcs++;
+ if (is_abbreviated) {
+ _abbreviated_degenerated_gcs++;
+ }
}
void ShenandoahCollectorPolicy::record_success_full() {
+ _consecutive_degenerated_gcs = 0;
_success_full_gcs++;
}
-size_t ShenandoahCollectorPolicy::cycle_counter() const {
- return _cycle_counter;
-}
-
-void ShenandoahCollectorPolicy::record_cycle_start() {
- _cycle_counter++;
-}
-
void ShenandoahCollectorPolicy::record_shutdown() {
_in_shutdown.set();
}
@@ -106,32 +95,102 @@ bool ShenandoahCollectorPolicy::is_at_shutdown() {
return _in_shutdown.is_set();
}
+bool is_explicit_gc(GCCause::Cause cause) {
+ return GCCause::is_user_requested_gc(cause)
+ || GCCause::is_serviceability_requested_gc(cause);
+}
+
+bool is_implicit_gc(GCCause::Cause cause) {
+ return cause != GCCause::_allocation_failure
+ && cause != GCCause::_shenandoah_concurrent_gc
+ && !is_explicit_gc(cause);
+}
+
+#ifdef ASSERT
+bool is_valid_request(GCCause::Cause cause) {
+ return is_explicit_gc(cause)
+ || cause == GCCause::_metadata_GC_clear_soft_refs
+ || cause == GCCause::_codecache_GC_aggressive
+ || cause == GCCause::_codecache_GC_threshold
+ || cause == GCCause::_full_gc_alot
+ || cause == GCCause::_wb_young_gc
+ || cause == GCCause::_wb_full_gc
+ || cause == GCCause::_wb_breakpoint
+ || cause == GCCause::_scavenge_alot;
+}
+#endif
+
+bool ShenandoahCollectorPolicy::should_run_full_gc(GCCause::Cause cause) {
+ return is_explicit_gc(cause) ? !ExplicitGCInvokesConcurrent : !ShenandoahImplicitGCInvokesConcurrent;
+}
+
+bool ShenandoahCollectorPolicy::should_handle_requested_gc(GCCause::Cause cause) {
+ assert(is_valid_request(cause), "only requested GCs here: %s", GCCause::to_string(cause));
+
+ if (DisableExplicitGC) {
+ return !is_explicit_gc(cause);
+ }
+ return true;
+}
+
void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const {
out->print_cr("Under allocation pressure, concurrent cycles may cancel, and either continue cycle");
out->print_cr("under stop-the-world pause or result in stop-the-world Full GC. Increase heap size,");
out->print_cr("tune GC heuristics, set more aggressive pacing delay, or lower allocation rate");
- out->print_cr("to avoid Degenerated and Full GC cycles.");
+ out->print_cr("to avoid Degenerated and Full GC cycles. Abbreviated cycles are those which found");
+ out->print_cr("enough regions with no live objects to skip evacuation.");
out->cr();
- out->print_cr(SIZE_FORMAT_W(5) " successful concurrent GCs", _success_concurrent_gcs);
- out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly", _explicit_concurrent);
- out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly", _implicit_concurrent);
+ size_t completed_gcs = _success_full_gcs + _success_degenerated_gcs + _success_concurrent_gcs;
+ out->print_cr(SIZE_FORMAT_W(5) " Completed GCs", completed_gcs);
+
+ size_t explicit_requests = 0;
+ size_t implicit_requests = 0;
+ for (int c = 0; c < GCCause::_last_gc_cause; c++) {
+ size_t cause_count = _collection_cause_counts[c];
+ if (cause_count > 0) {
+ auto cause = (GCCause::Cause) c;
+ if (is_explicit_gc(cause)) {
+ explicit_requests += cause_count;
+ } else if (is_implicit_gc(cause)) {
+ implicit_requests += cause_count;
+ }
+ const char* desc = GCCause::to_string(cause);
+ out->print_cr(" " SIZE_FORMAT_W(5) " caused by %s (%.2f%%)", cause_count, desc, percent_of(cause_count, completed_gcs));
+ }
+ }
+
+ out->cr();
+ out->print_cr(SIZE_FORMAT_W(5) " Successful Concurrent GCs (%.2f%%)", _success_concurrent_gcs, percent_of(_success_concurrent_gcs, completed_gcs));
+ if (ExplicitGCInvokesConcurrent) {
+ out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly (%.2f%%)", explicit_requests, percent_of(explicit_requests, _success_concurrent_gcs));
+ }
+ if (ShenandoahImplicitGCInvokesConcurrent) {
+ out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly (%.2f%%)", implicit_requests, percent_of(implicit_requests, _success_concurrent_gcs));
+ }
+ out->print_cr(" " SIZE_FORMAT_W(5) " abbreviated (%.2f%%)", _abbreviated_concurrent_gcs, percent_of(_abbreviated_concurrent_gcs, _success_concurrent_gcs));
out->cr();
- out->print_cr(SIZE_FORMAT_W(5) " Degenerated GCs", _success_degenerated_gcs);
- out->print_cr(" " SIZE_FORMAT_W(5) " caused by allocation failure", _alloc_failure_degenerated);
+ size_t degenerated_gcs = _alloc_failure_degenerated_upgrade_to_full + _success_degenerated_gcs;
+ out->print_cr(SIZE_FORMAT_W(5) " Degenerated GCs (%.2f%%)", degenerated_gcs, percent_of(degenerated_gcs, completed_gcs));
+ out->print_cr(" " SIZE_FORMAT_W(5) " upgraded to Full GC (%.2f%%)", _alloc_failure_degenerated_upgrade_to_full, percent_of(_alloc_failure_degenerated_upgrade_to_full, degenerated_gcs));
+ out->print_cr(" " SIZE_FORMAT_W(5) " caused by allocation failure (%.2f%%)", _alloc_failure_degenerated, percent_of(_alloc_failure_degenerated, degenerated_gcs));
+ out->print_cr(" " SIZE_FORMAT_W(5) " abbreviated (%.2f%%)", _abbreviated_degenerated_gcs, percent_of(_abbreviated_degenerated_gcs, degenerated_gcs));
for (int c = 0; c < ShenandoahGC::_DEGENERATED_LIMIT; c++) {
- if (_degen_points[c] > 0) {
+ if (_degen_point_counts[c] > 0) {
const char* desc = ShenandoahGC::degen_point_to_string((ShenandoahGC::ShenandoahDegenPoint)c);
- out->print_cr(" " SIZE_FORMAT_W(5) " happened at %s", _degen_points[c], desc);
+ out->print_cr(" " SIZE_FORMAT_W(5) " happened at %s", _degen_point_counts[c], desc);
}
}
- out->print_cr(" " SIZE_FORMAT_W(5) " upgraded to Full GC", _alloc_failure_degenerated_upgrade_to_full);
out->cr();
- out->print_cr(SIZE_FORMAT_W(5) " Full GCs", _success_full_gcs + _alloc_failure_degenerated_upgrade_to_full);
- out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly", _explicit_full);
- out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly", _implicit_full);
- out->print_cr(" " SIZE_FORMAT_W(5) " caused by allocation failure", _alloc_failure_full);
- out->print_cr(" " SIZE_FORMAT_W(5) " upgraded from Degenerated GC", _alloc_failure_degenerated_upgrade_to_full);
+ out->print_cr(SIZE_FORMAT_W(5) " Full GCs (%.2f%%)", _success_full_gcs, percent_of(_success_full_gcs, completed_gcs));
+ if (!ExplicitGCInvokesConcurrent) {
+ out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly (%.2f%%)", explicit_requests, percent_of(explicit_requests, _success_concurrent_gcs));
+ }
+ if (!ShenandoahImplicitGCInvokesConcurrent) {
+ out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly (%.2f%%)", implicit_requests, percent_of(implicit_requests, _success_concurrent_gcs));
+ }
+ out->print_cr(" " SIZE_FORMAT_W(5) " caused by allocation failure (%.2f%%)", _alloc_failure_full, percent_of(_alloc_failure_full, _success_full_gcs));
+ out->print_cr(" " SIZE_FORMAT_W(5) " upgraded from Degenerated GC (%.2f%%)", _alloc_failure_degenerated_upgrade_to_full, percent_of(_alloc_failure_degenerated_upgrade_to_full, _success_full_gcs));
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
index a6ea6e976ae4d..638acce14561f 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
@@ -39,54 +39,58 @@ class ShenandoahTracer : public GCTracer, public CHeapObj {
class ShenandoahCollectorPolicy : public CHeapObj {
private:
size_t _success_concurrent_gcs;
+ size_t _abbreviated_concurrent_gcs;
size_t _success_degenerated_gcs;
+ size_t _abbreviated_degenerated_gcs;
// Written by control thread, read by mutators
volatile size_t _success_full_gcs;
+ uint _consecutive_degenerated_gcs;
size_t _alloc_failure_degenerated;
size_t _alloc_failure_degenerated_upgrade_to_full;
size_t _alloc_failure_full;
- size_t _explicit_concurrent;
- size_t _explicit_full;
- size_t _implicit_concurrent;
- size_t _implicit_full;
- size_t _degen_points[ShenandoahGC::_DEGENERATED_LIMIT];
+ size_t _collection_cause_counts[GCCause::_last_gc_cause];
+ size_t _degen_point_counts[ShenandoahGC::_DEGENERATED_LIMIT];
ShenandoahSharedFlag _in_shutdown;
-
ShenandoahTracer* _tracer;
- size_t _cycle_counter;
public:
ShenandoahCollectorPolicy();
- // TODO: This is different from gc_end: that one encompasses one VM operation.
- // These two encompass the entire cycle.
- void record_cycle_start();
-
- void record_success_concurrent();
- void record_success_degenerated();
+ // A collection cycle may be "abbreviated" if Shenandoah finds a sufficient percentage
+ // of regions that contain no live objects (ShenandoahImmediateThreshold). These cycles
+ // end after final mark, skipping the evacuation and reference-updating phases. Such
+ // cycles are very efficient and are worth tracking. Note that both degenerated and
+ // concurrent cycles can be abbreviated.
+ void record_success_concurrent(bool is_abbreviated);
+ void record_success_degenerated(bool is_abbreviated);
void record_success_full();
void record_alloc_failure_to_degenerated(ShenandoahGC::ShenandoahDegenPoint point);
void record_alloc_failure_to_full();
void record_degenerated_upgrade_to_full();
- void record_explicit_to_concurrent();
- void record_explicit_to_full();
- void record_implicit_to_concurrent();
- void record_implicit_to_full();
+ void record_collection_cause(GCCause::Cause cause);
void record_shutdown();
bool is_at_shutdown();
ShenandoahTracer* tracer() {return _tracer;}
- size_t cycle_counter() const;
-
void print_gc_stats(outputStream* out) const;
size_t full_gc_count() const {
return _success_full_gcs + _alloc_failure_degenerated_upgrade_to_full;
}
+
+ // If the heuristics find that the number of consecutive degenerated cycles is above
+ // ShenandoahFullGCThreshold, then they will initiate a Full GC upon an allocation
+ // failure.
+ inline size_t consecutive_degenerated_gc_count() const {
+ return _consecutive_degenerated_gcs;
+ }
+
+ static bool should_run_full_gc(GCCause::Cause cause);
+ static bool should_handle_requested_gc(GCCause::Cause cause);
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTORPOLICY_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
index c7284fbceadbd..1e2120d55565a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
@@ -87,7 +87,8 @@ class ShenandoahBreakpointMarkScope : public StackObj {
ShenandoahConcurrentGC::ShenandoahConcurrentGC() :
_mark(),
- _degen_point(ShenandoahDegenPoint::_degenerated_unset) {
+ _degen_point(ShenandoahDegenPoint::_degenerated_unset),
+ _abbreviated(false) {
}
ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
@@ -112,11 +113,15 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
// Concurrent mark roots
entry_mark_roots();
- if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) return false;
+ if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) {
+ return false;
+ }
// Continue concurrent mark
entry_mark();
- if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) return false;
+ if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
+ return false;
+ }
}
// Complete marking under STW, and start evacuation
@@ -161,16 +166,22 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
if (heap->is_evacuation_in_progress()) {
// Concurrently evacuate
entry_evacuate();
- if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) return false;
+ if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
+ return false;
+ }
// Perform update-refs phase.
vmop_entry_init_updaterefs();
entry_updaterefs();
- if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
+ if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) {
+ return false;
+ }
// Concurrent update thread roots
entry_update_thread_roots();
- if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
+ if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) {
+ return false;
+ }
vmop_entry_final_updaterefs();
@@ -178,6 +189,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
entry_cleanup_complete();
} else {
vmop_entry_final_roots();
+ _abbreviated = true;
}
return true;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp
index 1010ffe5da7ed..4cd1a841350f0 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp
@@ -45,12 +45,16 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
private:
ShenandoahConcurrentMark _mark;
ShenandoahDegenPoint _degen_point;
+ bool _abbreviated;
public:
ShenandoahConcurrentGC();
bool collect(GCCause::Cause cause);
ShenandoahDegenPoint degen_point() const;
+ // Return true if this cycle found enough immediate garbage to skip evacuation
+ bool abbreviated() const { return _abbreviated; }
+
// Cancel ongoing concurrent GC
static void cancel();
private:
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
index f3c72dfc0b10a..fa37fcbcfd300 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
@@ -44,6 +44,7 @@
#include "runtime/continuation.hpp"
#include "runtime/threads.hpp"
+template
class ShenandoahConcurrentMarkingTask : public WorkerTask {
private:
ShenandoahConcurrentMark* const _cm;
@@ -61,8 +62,7 @@ class ShenandoahConcurrentMarkingTask : public WorkerTask {
ShenandoahReferenceProcessor* rp = heap->ref_processor();
assert(rp != nullptr, "need reference processor");
StringDedup::Requests requests;
- _cm->mark_loop(worker_id, _terminator, rp,
- true /*cancellable*/,
+ _cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/,
ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
&requests);
}
@@ -90,6 +90,7 @@ class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure {
}
};
+template
class ShenandoahFinalMarkingTask : public WorkerTask {
private:
ShenandoahConcurrentMark* _cm;
@@ -112,18 +113,17 @@ class ShenandoahFinalMarkingTask : public WorkerTask {
{
ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
- ShenandoahSATBBufferClosure cl(q);
+ ShenandoahSATBBufferClosure cl(q);
SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
assert(!heap->has_forwarded_objects(), "Not expected");
- ShenandoahMarkRefsClosure mark_cl(q, rp);
+ ShenandoahMarkRefsClosure mark_cl(q, rp);
ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set,
ShenandoahIUBarrier ? &mark_cl : nullptr);
Threads::possibly_parallel_threads_do(true /* is_par */, &tc);
}
- _cm->mark_loop(worker_id, _terminator, rp,
- false /*not cancellable*/,
+ _cm->mark_loop(worker_id, _terminator, rp, GENERATION, false /*not cancellable*/,
_dedup_string ? ENQUEUE_DEDUP : NO_DEDUP,
&requests);
assert(_cm->task_queues()->is_empty(), "Should be empty");
@@ -134,6 +134,7 @@ ShenandoahConcurrentMark::ShenandoahConcurrentMark() :
ShenandoahMark() {}
// Mark concurrent roots during concurrent phases
+template
class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
private:
SuspendibleThreadSetJoiner _sts_joiner;
@@ -149,7 +150,8 @@ class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
void work(uint worker_id);
};
-ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
+template
+ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
ShenandoahReferenceProcessor* rp,
ShenandoahPhaseTimings::Phase phase,
uint nworkers) :
@@ -160,10 +162,11 @@ ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahO
assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
}
-void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) {
+template
+void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
- ShenandoahMarkRefsClosure cl(q, _rp);
+ ShenandoahMarkRefsClosure cl(q, _rp);
_root_scanner.roots_do(&cl, worker_id);
}
@@ -176,7 +179,7 @@ void ShenandoahConcurrentMark::mark_concurrent_roots() {
WorkerThreads* workers = heap->workers();
ShenandoahReferenceProcessor* rp = heap->ref_processor();
task_queues()->reserve(workers->active_workers());
- ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+ ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
workers->run_task(&task);
}
@@ -204,7 +207,7 @@ void ShenandoahConcurrentMark::concurrent_mark() {
ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
TaskTerminator terminator(nworkers, task_queues());
- ShenandoahConcurrentMarkingTask task(this, &terminator);
+ ShenandoahConcurrentMarkingTask task(this, &terminator);
workers->run_task(&task);
if (heap->cancelled_gc()) {
@@ -254,7 +257,7 @@ void ShenandoahConcurrentMark::finish_mark_work() {
StrongRootsScope scope(nworkers);
TaskTerminator terminator(nworkers, task_queues());
- ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
heap->workers()->run_task(&task);
assert(task_queues()->is_empty(), "Should be empty");
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp
index fbcd075117aa0..b658e42dfb86a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp
@@ -25,14 +25,17 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP
+#include "gc/shenandoah/shenandoahGenerationType.hpp"
#include "gc/shenandoah/shenandoahMark.hpp"
+template
class ShenandoahConcurrentMarkingTask;
+template
class ShenandoahFinalMarkingTask;
class ShenandoahConcurrentMark: public ShenandoahMark {
- friend class ShenandoahConcurrentMarkingTask;
- friend class ShenandoahFinalMarkingTask;
+ template friend class ShenandoahConcurrentMarkingTask;
+ template friend class ShenandoahFinalMarkingTask;
public:
ShenandoahConcurrentMark();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
index 80c9189e93814..bdf1e5b912857 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
@@ -30,58 +30,33 @@
#include "gc/shenandoah/shenandoahDegeneratedGC.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahFullGC.hpp"
-#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
-#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
-#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
+#include "gc/shenandoah/shenandoahPacer.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
-#include "gc/shenandoah/shenandoahVMOperations.hpp"
-#include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
-#include "memory/iterator.hpp"
#include "memory/metaspaceUtils.hpp"
#include "memory/metaspaceStats.hpp"
-#include "memory/universe.hpp"
+#include "memory/resourceArea.hpp"
#include "runtime/atomic.hpp"
ShenandoahControlThread::ShenandoahControlThread() :
ConcurrentGCThread(),
_alloc_failure_waiters_lock(Mutex::safepoint-2, "ShenandoahAllocFailureGC_lock", true),
_gc_waiters_lock(Mutex::safepoint-2, "ShenandoahRequestedGC_lock", true),
- _periodic_task(this),
_requested_gc_cause(GCCause::_no_cause_specified),
_degen_point(ShenandoahGC::_degenerated_outside_cycle),
_allocs_seen(0) {
set_name("Shenandoah Control Thread");
reset_gc_id();
create_and_start();
- _periodic_task.enroll();
- if (ShenandoahPacing) {
- _periodic_pacer_notify_task.enroll();
- }
-}
-
-ShenandoahControlThread::~ShenandoahControlThread() {
- // This is here so that super is called.
-}
-
-void ShenandoahPeriodicTask::task() {
- _thread->handle_force_counters_update();
- _thread->handle_counters_update();
-}
-
-void ShenandoahPeriodicPacerNotify::task() {
- assert(ShenandoahPacing, "Should not be here otherwise");
- ShenandoahHeap::heap()->pacer()->notify_waiters();
}
void ShenandoahControlThread::run_service() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
- GCMode default_mode = concurrent_normal;
- GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc;
+ const GCMode default_mode = concurrent_normal;
+ const GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc;
int sleep = ShenandoahControlIntervalMin;
double last_shrink_time = os::elapsedTime();
@@ -91,23 +66,21 @@ void ShenandoahControlThread::run_service() {
// Having a period 10x lower than the delay would mean we hit the
// shrinking with lag of less than 1/10-th of true delay.
// ShenandoahUncommitDelay is in msecs, but shrink_period is in seconds.
- double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10;
+ const double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10;
- ShenandoahCollectorPolicy* policy = heap->shenandoah_policy();
- ShenandoahHeuristics* heuristics = heap->heuristics();
+ ShenandoahCollectorPolicy* const policy = heap->shenandoah_policy();
+ ShenandoahHeuristics* const heuristics = heap->heuristics();
while (!in_graceful_shutdown() && !should_terminate()) {
// Figure out if we have pending requests.
- bool alloc_failure_pending = _alloc_failure_gc.is_set();
- bool is_gc_requested = _gc_requested.is_set();
- GCCause::Cause requested_gc_cause = _requested_gc_cause;
- bool explicit_gc_requested = is_gc_requested && is_explicit_gc(requested_gc_cause);
- bool implicit_gc_requested = is_gc_requested && !is_explicit_gc(requested_gc_cause);
+ const bool alloc_failure_pending = _alloc_failure_gc.is_set();
+ const bool is_gc_requested = _gc_requested.is_set();
+ const GCCause::Cause requested_gc_cause = _requested_gc_cause;
- // This control loop iteration have seen this much allocations.
- size_t allocs_seen = Atomic::xchg(&_allocs_seen, (size_t)0, memory_order_relaxed);
+ // This control loop iteration has seen this much allocation.
+ const size_t allocs_seen = Atomic::xchg(&_allocs_seen, (size_t)0, memory_order_relaxed);
// Check if we have seen a new target for soft max heap size.
- bool soft_max_changed = check_soft_max_changed();
+ const bool soft_max_changed = heap->check_soft_max_changed();
// Choose which GC mode to run in. The block below should select a single mode.
GCMode mode = none;
@@ -134,36 +107,17 @@ void ShenandoahControlThread::run_service() {
mode = stw_full;
}
- } else if (explicit_gc_requested) {
+ } else if (is_gc_requested) {
cause = requested_gc_cause;
- log_info(gc)("Trigger: Explicit GC request (%s)", GCCause::to_string(cause));
-
+ log_info(gc)("Trigger: GC request (%s)", GCCause::to_string(cause));
heuristics->record_requested_gc();
- if (ExplicitGCInvokesConcurrent) {
- policy->record_explicit_to_concurrent();
- mode = default_mode;
- // Unload and clean up everything
- heap->set_unload_classes(heuristics->can_unload_classes());
- } else {
- policy->record_explicit_to_full();
+ if (ShenandoahCollectorPolicy::should_run_full_gc(cause)) {
mode = stw_full;
- }
- } else if (implicit_gc_requested) {
- cause = requested_gc_cause;
- log_info(gc)("Trigger: Implicit GC request (%s)", GCCause::to_string(cause));
-
- heuristics->record_requested_gc();
-
- if (ShenandoahImplicitGCInvokesConcurrent) {
- policy->record_implicit_to_concurrent();
+ } else {
mode = default_mode;
-
// Unload and clean up everything
heap->set_unload_classes(heuristics->can_unload_classes());
- } else {
- policy->record_implicit_to_full();
- mode = stw_full;
}
} else {
// Potential normal cycle: ask heuristics if it wants to act
@@ -178,11 +132,11 @@ void ShenandoahControlThread::run_service() {
// Blow all soft references on this cycle, if handling allocation failure,
// either implicit or explicit GC request, or we are requested to do so unconditionally.
- if (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs) {
+ if (alloc_failure_pending || is_gc_requested || ShenandoahAlwaysClearSoftRefs) {
heap->soft_ref_policy()->set_should_clear_all_soft_refs(true);
}
- bool gc_requested = (mode != none);
+ const bool gc_requested = (mode != none);
assert (!gc_requested || cause != GCCause::_last_gc_cause, "GC cause should be set");
if (gc_requested) {
@@ -195,7 +149,7 @@ void ShenandoahControlThread::run_service() {
// If GC was requested, we are sampling the counters even without actual triggers
// from allocation machinery. This captures GC phases more accurately.
- set_forced_counters_update(true);
+ heap->set_forced_counters_update(true);
// If GC was requested, we better dump freeset data for performance debugging
{
@@ -218,7 +172,7 @@ void ShenandoahControlThread::run_service() {
}
// If this was the requested GC cycle, notify waiters about it
- if (explicit_gc_requested || implicit_gc_requested) {
+ if (is_gc_requested) {
notify_gc_waiters();
}
@@ -236,16 +190,16 @@ void ShenandoahControlThread::run_service() {
// Notify Universe about new heap usage. This has implications for
// global soft refs policy, and we better report it every time heap
// usage goes down.
- Universe::heap()->update_capacity_and_used_at_gc();
+ heap->update_capacity_and_used_at_gc();
// Signal that we have completed a visit to all live objects.
- Universe::heap()->record_whole_heap_examined_timestamp();
+ heap->record_whole_heap_examined_timestamp();
}
// Disable forced counters update, and update counters one more time
// to capture the state at the end of GC session.
- handle_force_counters_update();
- set_forced_counters_update(false);
+ heap->handle_force_counters_update();
+ heap->set_forced_counters_update(false);
// Retract forceful part of soft refs policy
heap->soft_ref_policy()->set_should_clear_all_soft_refs(false);
@@ -291,14 +245,14 @@ void ShenandoahControlThread::run_service() {
}
}
- double current = os::elapsedTime();
+ const double current = os::elapsedTime();
- if (ShenandoahUncommit && (explicit_gc_requested || soft_max_changed || (current - last_shrink_time > shrink_period))) {
+ if (ShenandoahUncommit && (is_gc_requested || soft_max_changed || (current - last_shrink_time > shrink_period))) {
// Explicit GC tries to uncommit everything down to min capacity.
// Soft max change tries to uncommit everything down to target capacity.
// Periodic uncommit tries to uncommit suitable regions down to min capacity.
- double shrink_before = (explicit_gc_requested || soft_max_changed) ?
+ double shrink_before = (is_gc_requested || soft_max_changed) ?
current :
current - (ShenandoahUncommitDelay / 1000.0);
@@ -306,7 +260,7 @@ void ShenandoahControlThread::run_service() {
heap->soft_max_capacity() :
heap->min_capacity();
- service_uncommit(shrink_before, shrink_until);
+ heap->maybe_uncommit(shrink_before, shrink_until);
heap->phase_timings()->flush_cycle_to_global();
last_shrink_time = current;
}
@@ -314,7 +268,7 @@ void ShenandoahControlThread::run_service() {
// Wait before performing the next action. If allocation happened during this wait,
// we exit sooner, to let heuristics re-evaluate new conditions. If we are at idle,
// back off exponentially.
- if (_heap_changed.try_unset()) {
+ if (heap->has_changed()) {
sleep = ShenandoahControlIntervalMin;
} else if ((current - last_sleep_adjust_time) * 1000 > ShenandoahControlIntervalAdjustPeriod){
sleep = MIN2(ShenandoahControlIntervalMax, MAX2(1, sleep * 2));
@@ -329,25 +283,6 @@ void ShenandoahControlThread::run_service() {
}
}
-bool ShenandoahControlThread::check_soft_max_changed() const {
- ShenandoahHeap* heap = ShenandoahHeap::heap();
- size_t new_soft_max = Atomic::load(&SoftMaxHeapSize);
- size_t old_soft_max = heap->soft_max_capacity();
- if (new_soft_max != old_soft_max) {
- new_soft_max = MAX2(heap->min_capacity(), new_soft_max);
- new_soft_max = MIN2(heap->max_capacity(), new_soft_max);
- if (new_soft_max != old_soft_max) {
- log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s",
- byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
- byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
- );
- heap->set_soft_max_capacity(new_soft_max);
- return true;
- }
- }
- return false;
-}
-
void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cause) {
// Normal cycle goes via all concurrent phases. If allocation failure (af) happens during
// any of the concurrent phases, it first degrades to Degenerated GC and completes GC there.
@@ -396,7 +331,7 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cau
if (gc.collect(cause)) {
// Cycle is complete
heap->heuristics()->record_success_concurrent();
- heap->shenandoah_policy()->record_success_concurrent();
+ heap->shenandoah_policy()->record_success_concurrent(gc.abbreviated());
} else {
assert(heap->cancelled_gc(), "Must have been cancelled");
check_cancellation_or_degen(gc.degen_point());
@@ -427,10 +362,6 @@ void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) {
ShenandoahFullGC gc;
gc.collect(cause);
-
- ShenandoahHeap* const heap = ShenandoahHeap::heap();
- heap->heuristics()->record_success_full();
- heap->shenandoah_policy()->record_success_full();
}
void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point) {
@@ -441,58 +372,10 @@ void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause
ShenandoahDegenGC gc(point);
gc.collect(cause);
-
- ShenandoahHeap* const heap = ShenandoahHeap::heap();
- heap->heuristics()->record_success_degenerated();
- heap->shenandoah_policy()->record_success_degenerated();
-}
-
-void ShenandoahControlThread::service_uncommit(double shrink_before, size_t shrink_until) {
- ShenandoahHeap* heap = ShenandoahHeap::heap();
-
- // Determine if there is work to do. This avoids taking heap lock if there is
- // no work available, avoids spamming logs with superfluous logging messages,
- // and minimises the amount of work while locks are taken.
-
- if (heap->committed() <= shrink_until) return;
-
- bool has_work = false;
- for (size_t i = 0; i < heap->num_regions(); i++) {
- ShenandoahHeapRegion *r = heap->get_region(i);
- if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
- has_work = true;
- break;
- }
- }
-
- if (has_work) {
- heap->entry_uncommit(shrink_before, shrink_until);
- }
-}
-
-bool ShenandoahControlThread::is_explicit_gc(GCCause::Cause cause) const {
- return GCCause::is_user_requested_gc(cause) ||
- GCCause::is_serviceability_requested_gc(cause);
}
void ShenandoahControlThread::request_gc(GCCause::Cause cause) {
- assert(GCCause::is_user_requested_gc(cause) ||
- GCCause::is_serviceability_requested_gc(cause) ||
- cause == GCCause::_metadata_GC_clear_soft_refs ||
- cause == GCCause::_codecache_GC_aggressive ||
- cause == GCCause::_codecache_GC_threshold ||
- cause == GCCause::_full_gc_alot ||
- cause == GCCause::_wb_young_gc ||
- cause == GCCause::_wb_full_gc ||
- cause == GCCause::_wb_breakpoint ||
- cause == GCCause::_scavenge_alot,
- "only requested GCs here: %s", GCCause::to_string(cause));
-
- if (is_explicit_gc(cause)) {
- if (!DisableExplicitGC) {
- handle_requested_gc(cause);
- }
- } else {
+ if (ShenandoahCollectorPolicy::should_handle_requested_gc(cause)) {
handle_requested_gc(cause);
}
}
@@ -581,43 +464,11 @@ void ShenandoahControlThread::notify_gc_waiters() {
ml.notify_all();
}
-void ShenandoahControlThread::handle_counters_update() {
- if (_do_counters_update.is_set()) {
- _do_counters_update.unset();
- ShenandoahHeap::heap()->monitoring_support()->update_counters();
- }
-}
-
-void ShenandoahControlThread::handle_force_counters_update() {
- if (_force_counters_update.is_set()) {
- _do_counters_update.unset(); // reset these too, we do update now!
- ShenandoahHeap::heap()->monitoring_support()->update_counters();
- }
-}
-
-void ShenandoahControlThread::notify_heap_changed() {
- // This is called from allocation path, and thus should be fast.
-
- // Update monitoring counters when we took a new region. This amortizes the
- // update costs on slow path.
- if (_do_counters_update.is_unset()) {
- _do_counters_update.set();
- }
- // Notify that something had changed.
- if (_heap_changed.is_unset()) {
- _heap_changed.set();
- }
-}
-
void ShenandoahControlThread::pacing_notify_alloc(size_t words) {
assert(ShenandoahPacing, "should only call when pacing is enabled");
Atomic::add(&_allocs_seen, words, memory_order_relaxed);
}
-void ShenandoahControlThread::set_forced_counters_update(bool value) {
- _force_counters_update.set_cond(value);
-}
-
void ShenandoahControlThread::reset_gc_id() {
Atomic::store(&_gc_id, (size_t)0);
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp
index 544c504b4fa6c..9da25b1a73ced 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp
@@ -28,30 +28,8 @@
#include "gc/shared/gcCause.hpp"
#include "gc/shared/concurrentGCThread.hpp"
#include "gc/shenandoah/shenandoahGC.hpp"
-#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahSharedVariables.hpp"
-#include "runtime/task.hpp"
-#include "utilities/ostream.hpp"
-
-// Periodic task is useful for doing asynchronous things that do not require (heap) locks,
-// or synchronization with other parts of collector. These could run even when ShenandoahConcurrentThread
-// is busy driving the GC cycle.
-class ShenandoahPeriodicTask : public PeriodicTask {
-private:
- ShenandoahControlThread* _thread;
-public:
- ShenandoahPeriodicTask(ShenandoahControlThread* thread) :
- PeriodicTask(100), _thread(thread) {}
- virtual void task();
-};
-
-// Periodic task to notify blocked paced waiters.
-class ShenandoahPeriodicPacerNotify : public PeriodicTask {
-public:
- ShenandoahPeriodicPacerNotify() : PeriodicTask(PeriodicTask::min_interval) {}
- virtual void task();
-};
class ShenandoahControlThread: public ConcurrentGCThread {
friend class VMStructs;
@@ -69,8 +47,6 @@ class ShenandoahControlThread: public ConcurrentGCThread {
// to make complete explicit cycle for for demanding customers.
Monitor _alloc_failure_waiters_lock;
Monitor _gc_waiters_lock;
- ShenandoahPeriodicTask _periodic_task;
- ShenandoahPeriodicPacerNotify _periodic_pacer_notify_task;
public:
void run_service();
@@ -80,9 +56,6 @@ class ShenandoahControlThread: public ConcurrentGCThread {
ShenandoahSharedFlag _gc_requested;
ShenandoahSharedFlag _alloc_failure_gc;
ShenandoahSharedFlag _graceful_shutdown;
- ShenandoahSharedFlag _heap_changed;
- ShenandoahSharedFlag _do_counters_update;
- ShenandoahSharedFlag _force_counters_update;
GCCause::Cause _requested_gc_cause;
ShenandoahGC::ShenandoahDegenPoint _degen_point;
@@ -96,7 +69,6 @@ class ShenandoahControlThread: public ConcurrentGCThread {
void service_concurrent_normal_cycle(GCCause::Cause cause);
void service_stw_full_cycle(GCCause::Cause cause);
void service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point);
- void service_uncommit(double shrink_before, size_t shrink_until);
bool try_set_alloc_failure_gc();
void notify_alloc_failure_waiters();
@@ -112,14 +84,9 @@ class ShenandoahControlThread: public ConcurrentGCThread {
// Blocks until GC is over.
void handle_requested_gc(GCCause::Cause cause);
- bool is_explicit_gc(GCCause::Cause cause) const;
-
- bool check_soft_max_changed() const;
-
public:
// Constructor
ShenandoahControlThread();
- ~ShenandoahControlThread();
// Handle allocation failure from a mutator allocation.
// Optionally blocks while collector is handling the failure. If the GC
@@ -132,12 +99,6 @@ class ShenandoahControlThread: public ConcurrentGCThread {
void request_gc(GCCause::Cause cause);
- void handle_counters_update();
- void handle_force_counters_update();
- void set_forced_counters_update(bool value);
-
- void notify_heap_changed();
-
void pacing_notify_alloc(size_t words);
void start();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
index e7cf402a52785..fb5283bb1d88d 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
@@ -44,7 +44,8 @@
ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point) :
ShenandoahGC(),
- _degen_point(degen_point) {
+ _degen_point(degen_point),
+ _abbreviated(false) {
}
bool ShenandoahDegenGC::collect(GCCause::Cause cause) {
@@ -193,6 +194,8 @@ void ShenandoahDegenGC::op_degenerated() {
if (heap->has_forwarded_objects()) {
op_init_updaterefs();
assert(!heap->cancelled_gc(), "STW reference update can not OOM");
+ } else {
+ _abbreviated = true;
}
case _degenerated_updaterefs:
@@ -230,6 +233,8 @@ void ShenandoahDegenGC::op_degenerated() {
op_degenerated_futile();
} else {
heap->notify_gc_progress();
+ heap->shenandoah_policy()->record_success_degenerated(_abbreviated);
+ heap->heuristics()->record_success_degenerated();
}
}
@@ -343,17 +348,11 @@ void ShenandoahDegenGC::op_cleanup_complete() {
}
void ShenandoahDegenGC::op_degenerated_fail() {
- log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
- ShenandoahHeap::heap()->shenandoah_policy()->record_degenerated_upgrade_to_full();
-
- ShenandoahFullGC full_gc;
- full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc);
+ upgrade_to_full();
}
void ShenandoahDegenGC::op_degenerated_futile() {
- ShenandoahHeap::heap()->shenandoah_policy()->record_degenerated_upgrade_to_full();
- ShenandoahFullGC full_gc;
- full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc);
+ upgrade_to_full();
}
const char* ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point) const {
@@ -373,3 +372,10 @@ const char* ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point) c
return "ERROR";
}
}
+
+void ShenandoahDegenGC::upgrade_to_full() {
+ log_info(gc)("Degenerated GC upgrading to Full GC");
+ ShenandoahHeap::heap()->shenandoah_policy()->record_degenerated_upgrade_to_full();
+ ShenandoahFullGC full_gc;
+ full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc);
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp
index 8f6f71d52c253..633c3f1ce73e9 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp
@@ -33,6 +33,7 @@ class ShenandoahDegenGC : public ShenandoahGC {
friend class VM_ShenandoahDegeneratedGC;
private:
const ShenandoahDegenPoint _degen_point;
+ bool _abbreviated;
public:
ShenandoahDegenGC(ShenandoahDegenPoint degen_point);
@@ -48,6 +49,7 @@ class ShenandoahDegenGC : public ShenandoahGC {
void op_finish_mark();
void op_prepare_evacuation();
void op_cleanup_early();
+
void op_evacuate();
void op_init_updaterefs();
void op_updaterefs();
@@ -58,6 +60,9 @@ class ShenandoahDegenGC : public ShenandoahGC {
void op_degenerated_futile();
void op_degenerated_fail();
+ // Turns this degenerated cycle into a full gc without leaving the safepoint
+ void upgrade_to_full();
+
const char* degen_event_message(ShenandoahDegenPoint point) const;
};
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
index 4cef5378d30bc..d66bf052da4f1 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
@@ -31,6 +31,7 @@
#include "gc/shared/tlab_globals.hpp"
#include "gc/shared/workerThread.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahConcurrentGC.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
@@ -105,15 +106,21 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) {
// Perform full GC
do_it(cause);
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
+
metrics.snap_after();
if (metrics.is_good_progress()) {
- ShenandoahHeap::heap()->notify_gc_progress();
+ heap->notify_gc_progress();
} else {
// Nothing to do. Tell the allocation path that we have failed to make
// progress, and it can finally fail.
- ShenandoahHeap::heap()->notify_gc_no_progress();
+ heap->notify_gc_no_progress();
}
+
+ // Regardless if progress was made, we record that we completed a "successful" full GC.
+ heap->heuristics()->record_success_full();
+ heap->shenandoah_policy()->record_success_full();
}
void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
@@ -228,12 +235,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
phase3_update_references();
phase4_compact_objects(worker_slices);
- }
- {
- // Epilogue
- _preserved_marks->restore(heap->workers());
- _preserved_marks->reclaim();
+ phase5_epilog();
}
// Resize metaspace
@@ -273,6 +276,8 @@ class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
_ctx->capture_top_at_mark_start(r);
r->clear_live_data();
}
+
+ bool is_thread_safe() { return true; }
};
void ShenandoahFullGC::phase1_mark_heap() {
@@ -282,7 +287,7 @@ void ShenandoahFullGC::phase1_mark_heap() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahPrepareForMarkClosure cl;
- heap->heap_region_iterate(&cl);
+ heap->parallel_heap_region_iterate(&cl);
heap->set_unload_classes(heap->heuristics()->can_unload_classes());
@@ -321,7 +326,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
_from_region = from_region;
}
- void finish_region() {
+ void finish() {
assert(_to_region != nullptr, "should not happen");
_to_region->set_new_top(_compact_point);
}
@@ -341,7 +346,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
size_t obj_size = p->size();
if (_compact_point + obj_size > _to_region->end()) {
- finish_region();
+ finish();
// Object doesn't fit. Pick next empty region and start compacting there.
ShenandoahHeapRegion* new_to_region;
@@ -393,47 +398,61 @@ class ShenandoahPrepareForCompactionTask : public WorkerTask {
return r->is_stw_move_allowed() && !r->is_humongous();
}
- void work(uint worker_id) {
- ShenandoahParallelWorkerSession worker_session(worker_id);
- ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
- ShenandoahHeapRegionSetIterator it(slice);
- ShenandoahHeapRegion* from_region = it.next();
- // No work?
- if (from_region == nullptr) {
- return;
- }
-
- // Sliding compaction. Walk all regions in the slice, and compact them.
- // Remember empty regions and reuse them as needed.
- ResourceMark rm;
+ void work(uint worker_id) override;
+private:
+ template
+ void prepare_for_compaction(ClosureType& cl,
+ GrowableArray& empty_regions,
+ ShenandoahHeapRegionSetIterator& it,
+ ShenandoahHeapRegion* from_region);
+};
- GrowableArray empty_regions((int)_heap->num_regions());
+void ShenandoahPrepareForCompactionTask::work(uint worker_id) {
+ ShenandoahParallelWorkerSession worker_session(worker_id);
+ ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
+ ShenandoahHeapRegionSetIterator it(slice);
+ ShenandoahHeapRegion* from_region = it.next();
+ // No work?
+ if (from_region == nullptr) {
+ return;
+ }
- ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
+ // Sliding compaction. Walk all regions in the slice, and compact them.
+ // Remember empty regions and reuse them as needed.
+ ResourceMark rm;
- while (from_region != nullptr) {
- assert(is_candidate_region(from_region), "Sanity");
+ GrowableArray empty_regions((int)_heap->num_regions());
- cl.set_from_region(from_region);
- if (from_region->has_live()) {
- _heap->marked_object_iterate(from_region, &cl);
- }
+ ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
+ prepare_for_compaction(cl, empty_regions, it, from_region);
+}
- // Compacted the region to somewhere else? From-region is empty then.
- if (!cl.is_compact_same_region()) {
- empty_regions.append(from_region);
- }
- from_region = it.next();
+template
+void ShenandoahPrepareForCompactionTask::prepare_for_compaction(ClosureType& cl,
+ GrowableArray& empty_regions,
+ ShenandoahHeapRegionSetIterator& it,
+ ShenandoahHeapRegion* from_region) {
+ while (from_region != nullptr) {
+ assert(is_candidate_region(from_region), "Sanity");
+ cl.set_from_region(from_region);
+ if (from_region->has_live()) {
+ _heap->marked_object_iterate(from_region, &cl);
}
- cl.finish_region();
- // Mark all remaining regions as empty
- for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
- ShenandoahHeapRegion* r = empty_regions.at(pos);
- r->set_new_top(r->bottom());
+ // Compacted the region to somewhere else? From-region is empty then.
+ if (!cl.is_compact_same_region()) {
+ empty_regions.append(from_region);
}
+ from_region = it.next();
}
-};
+ cl.finish();
+
+ // Mark all remaining regions as empty
+ for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
+ ShenandoahHeapRegion* r = empty_regions.at(pos);
+ r->set_new_top(r->bottom());
+ }
+}
void ShenandoahFullGC::calculate_target_humongous_objects() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
@@ -941,7 +960,7 @@ void ShenandoahFullGC::compact_humongous_objects() {
//
// This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
// humongous regions are already compacted, and do not require further moves, which alleviates
- // sliding costs. We may consider doing this in parallel in future.
+ // sliding costs. We may consider doing this in parallel in the future.
ShenandoahHeap* heap = ShenandoahHeap::heap();
@@ -1047,6 +1066,11 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
compact_humongous_objects();
}
+}
+
+void ShenandoahFullGC::phase5_epilog() {
+ GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer);
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
// Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
// and must ensure the bitmap is in sync.
@@ -1059,14 +1083,15 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s
// Bring regions in proper states after the collection, and set heap properties.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
-
ShenandoahPostCompactClosure post_compact;
heap->heap_region_iterate(&post_compact);
heap->set_used(post_compact.get_live());
heap->collection_set()->clear();
heap->free_set()->rebuild();
+ heap->clear_cancelled_gc();
}
- heap->clear_cancelled_gc();
+ _preserved_marks->restore(heap->workers());
+ _preserved_marks->reclaim();
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp
index 1c1653e59ec56..6687116b21f78 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp
@@ -81,6 +81,7 @@ class ShenandoahFullGC : public ShenandoahGC {
void phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices);
void phase3_update_references();
void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices);
+ void phase5_epilog();
void distribute_slices(ShenandoahHeapRegionSet** worker_slices);
void calculate_target_humongous_objects();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp
new file mode 100644
index 0000000000000..1132fd5eb4d9a
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONTYPE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONTYPE_HPP
+
+enum ShenandoahGenerationType {
+ NON_GEN // non-generational
+};
+
+inline const char* shenandoah_generation_name(ShenandoahGenerationType mode) {
+ switch (mode) {
+ case NON_GEN:
+ return "Non-Generational";
+ default:
+ ShouldNotReachHere();
+ return "Unknown";
+ }
+}
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONTYPE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
index 12eb2aad5e803..f0a2e53a12b1e 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -432,8 +432,6 @@ jint ShenandoahHeap::initialize() {
if (ShenandoahPacing) {
_pacer = new ShenandoahPacer(this);
_pacer->setup_for_idle();
- } else {
- _pacer = nullptr;
}
_control_thread = new ShenandoahControlThread();
@@ -519,7 +517,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
_stw_memory_manager("Shenandoah Pauses"),
_cycle_memory_manager("Shenandoah Cycles"),
_gc_timer(new ConcurrentGCTimer()),
- _soft_ref_policy(),
_log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
_ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
_marking_context(nullptr),
@@ -663,7 +660,7 @@ void ShenandoahHeap::post_initialize() {
_heuristics->initialize();
- JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
+ JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers();)
}
size_t ShenandoahHeap::used() const {
@@ -756,6 +753,33 @@ bool ShenandoahHeap::is_in(const void* p) const {
return p >= heap_base && p < last_region_end;
}
+void ShenandoahHeap::maybe_uncommit(double shrink_before, size_t shrink_until) {
+ assert (ShenandoahUncommit, "should be enabled");
+
+ // Determine if there is work to do. This avoids taking heap lock if there is
+ // no work available, avoids spamming logs with superfluous logging messages,
+ // and minimises the amount of work while locks are taken.
+
+ if (committed() <= shrink_until) return;
+
+ bool has_work = false;
+ for (size_t i = 0; i < num_regions(); i++) {
+ ShenandoahHeapRegion* r = get_region(i);
+ if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
+ has_work = true;
+ break;
+ }
+ }
+
+ if (has_work) {
+ static const char* msg = "Concurrent uncommit";
+ ShenandoahConcurrentPhase gcPhase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
+ EventMark em("%s", msg);
+
+ op_uncommit(shrink_before, shrink_until);
+ }
+}
+
void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
assert (ShenandoahUncommit, "should be enabled");
@@ -782,8 +806,43 @@ void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) {
}
if (count > 0) {
- control_thread()->notify_heap_changed();
+ notify_heap_changed();
+ }
+}
+
+bool ShenandoahHeap::check_soft_max_changed() {
+ size_t new_soft_max = Atomic::load(&SoftMaxHeapSize);
+ size_t old_soft_max = soft_max_capacity();
+ if (new_soft_max != old_soft_max) {
+ new_soft_max = MAX2(min_capacity(), new_soft_max);
+ new_soft_max = MIN2(max_capacity(), new_soft_max);
+ if (new_soft_max != old_soft_max) {
+ log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s",
+ byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
+ byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
+ );
+ set_soft_max_capacity(new_soft_max);
+ return true;
+ }
}
+ return false;
+}
+
+void ShenandoahHeap::notify_heap_changed() {
+ // Update monitoring counters when we took a new region. This amortizes the
+ // update costs on slow path.
+ monitoring_support()->notify_heap_changed();
+
+ // This is called from allocation path, and thus should be fast.
+ _heap_changed.try_set();
+}
+
+void ShenandoahHeap::set_forced_counters_update(bool value) {
+ monitoring_support()->set_forced_counters_update(value);
+}
+
+void ShenandoahHeap::handle_force_counters_update() {
+ monitoring_support()->handle_force_counters_update();
}
HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
@@ -915,7 +974,7 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
}
if (in_new_region) {
- control_thread()->notify_heap_changed();
+ notify_heap_changed();
}
if (result != nullptr) {
@@ -944,7 +1003,11 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
}
HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
- ShenandoahHeapLocker locker(lock());
+ // If we are dealing with mutator allocation, then we may need to block for safepoint.
+ // We cannot block for safepoint for GC allocations, because there is a high chance
+ // we are already running at safepoint or from stack watermark machinery, and we cannot
+ // block again.
+ ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
return _free_set->allocate(req, in_new_region);
}
@@ -2246,14 +2309,6 @@ void ShenandoahHeap::safepoint_synchronize_end() {
SuspendibleThreadSet::desynchronize();
}
-void ShenandoahHeap::entry_uncommit(double shrink_before, size_t shrink_until) {
- static const char *msg = "Concurrent uncommit";
- ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
- EventMark em("%s", msg);
-
- op_uncommit(shrink_before, shrink_until);
-}
-
void ShenandoahHeap::try_inject_alloc_failure() {
if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
_inject_alloc_failure.set();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
index deb0a972167e9..d3bddc8cc76f4 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
@@ -207,6 +207,15 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
void set_soft_max_capacity(size_t v);
+// ---------- Periodic Tasks
+//
+private:
+ void notify_heap_changed();
+
+public:
+ void set_forced_counters_update(bool value);
+ void handle_force_counters_update();
+
// ---------- Workers handling
//
private:
@@ -286,6 +295,9 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
private:
bool _gc_state_changed;
ShenandoahSharedBitmap _gc_state;
+
+ // tracks if new regions have been allocated or retired since last check
+ ShenandoahSharedFlag _heap_changed;
ShenandoahSharedFlag _degenerated_gc_in_progress;
ShenandoahSharedFlag _full_gc_in_progress;
ShenandoahSharedFlag _full_gc_move_in_progress;
@@ -307,6 +319,12 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// a safepoint and that any changes were propagated to java threads after the safepoint.
bool has_gc_state_changed() const { return _gc_state_changed; }
+ // Returns true if allocations have occurred in new regions or if regions have been
+ // uncommitted since the previous calls. This call will reset the flag to false.
+ bool has_changed() {
+ return _heap_changed.try_unset();
+ }
+
void set_concurrent_mark_in_progress(bool in_progress);
void set_evacuation_in_progress(bool in_progress);
void set_update_refs_in_progress(bool in_progress);
@@ -355,10 +373,14 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
void cancel_gc(GCCause::Cause cause);
public:
- // Elastic heap support
- void entry_uncommit(double shrink_before, size_t shrink_until);
+ // These will uncommit empty regions if heap::committed > shrink_until
+ // and there exists at least one region which was made empty before shrink_before.
+ void maybe_uncommit(double shrink_before, size_t shrink_until);
void op_uncommit(double shrink_before, size_t shrink_until);
+ // Returns true if the soft maximum heap has been changed using management APIs.
+ bool check_soft_max_changed();
+
private:
// GC support
// Reset bitmap, prepare regions for new GC cycle
@@ -419,15 +441,12 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
GCMemoryManager _stw_memory_manager;
GCMemoryManager _cycle_memory_manager;
ConcurrentGCTimer* _gc_timer;
- SoftRefPolicy _soft_ref_policy;
-
// For exporting to SA
int _log_min_obj_alignment_in_bytes;
public:
ShenandoahMonitoringSupport* monitoring_support() { return _monitoring_support; }
GCMemoryManager* cycle_memory_manager() { return &_cycle_memory_manager; }
GCMemoryManager* stw_memory_manager() { return &_stw_memory_manager; }
- SoftRefPolicy* soft_ref_policy() override { return &_soft_ref_policy; }
GrowableArray memory_managers() override;
GrowableArray memory_pools() override;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
index d2487705d5a04..c5763608582a1 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
@@ -30,7 +30,6 @@
#include "gc/shenandoah/shenandoahAllocRequest.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
-#include "gc/shenandoah/shenandoahPacer.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "utilities/sizes.hpp"
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp
index 9f0233dd08c14..db5a68d584e90 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,37 +30,26 @@
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "logging/log.hpp"
-#include "runtime/globals.hpp"
#include "utilities/globalDefinitions.hpp"
+void ShenandoahInitLogger::print() {
+ ShenandoahInitLogger init_log;
+ init_log.print_all();
+}
+
void ShenandoahInitLogger::print_heap() {
GCInitLogger::print_heap();
- ShenandoahHeap* heap = ShenandoahHeap::heap();
-
- log_info(gc, init)("Mode: %s",
- heap->mode()->name());
-
- log_info(gc, init)("Heuristics: %s",
- heap->heuristics()->name());
-
- log_info(gc, init)("Heap Region Count: " SIZE_FORMAT,
- ShenandoahHeapRegion::region_count());
-
- log_info(gc, init)("Heap Region Size: " SIZE_FORMAT "%s",
- byte_size_in_exact_unit(ShenandoahHeapRegion::region_size_bytes()),
- exact_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
-
- log_info(gc, init)("TLAB Size Max: " SIZE_FORMAT "%s",
- byte_size_in_exact_unit(ShenandoahHeapRegion::max_tlab_size_bytes()),
- exact_unit_for_byte_size(ShenandoahHeapRegion::max_tlab_size_bytes()));
-
- log_info(gc, init)("Humongous Object Threshold: " SIZE_FORMAT "%s",
- byte_size_in_exact_unit(ShenandoahHeapRegion::humongous_threshold_bytes()),
- exact_unit_for_byte_size(ShenandoahHeapRegion::humongous_threshold_bytes()));
+ log_info(gc, init)("Heap Region Count: " SIZE_FORMAT, ShenandoahHeapRegion::region_count());
+ log_info(gc, init)("Heap Region Size: " EXACTFMT, EXACTFMTARGS(ShenandoahHeapRegion::region_size_bytes()));
+ log_info(gc, init)("TLAB Size Max: " EXACTFMT, EXACTFMTARGS(ShenandoahHeapRegion::max_tlab_size_bytes()));
+ log_info(gc, init)("Humongous Object Threshold: " EXACTFMT, EXACTFMTARGS(ShenandoahHeapRegion::humongous_threshold_bytes()));
}
-void ShenandoahInitLogger::print() {
- ShenandoahInitLogger init_log;
- init_log.print_all();
+void ShenandoahInitLogger::print_gc_specific() {
+ GCInitLogger::print_gc_specific();
+
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ log_info(gc, init)("Mode: %s", heap->mode()->name());
+ log_info(gc, init)("Heuristics: %s", heap->heuristics()->name());
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.hpp b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.hpp
index 98c918c58f741..8c0da41339978 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.hpp
@@ -29,7 +29,8 @@
class ShenandoahInitLogger : public GCInitLogger {
protected:
- virtual void print_heap();
+ void print_heap() override;
+ void print_gc_specific() override;
public:
static void print();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp b/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp
index 3a9b612860b9f..39588d68bc83b 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp
@@ -28,9 +28,47 @@
#include "gc/shenandoah/shenandoahLock.hpp"
#include "runtime/atomic.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/os.inline.hpp"
+// These are inline variants of Thread::SpinAcquire with optional blocking in VM.
+
+class ShenandoahNoBlockOp : public StackObj {
+public:
+ ShenandoahNoBlockOp(JavaThread* java_thread) {
+ assert(java_thread == nullptr, "Should not pass anything");
+ }
+};
+
+void ShenandoahLock::contended_lock(bool allow_block_for_safepoint) {
+ Thread* thread = Thread::current();
+ if (allow_block_for_safepoint && thread->is_Java_thread()) {
+ contended_lock_internal(JavaThread::cast(thread));
+ } else {
+ contended_lock_internal(nullptr);
+ }
+}
+
+template
+void ShenandoahLock::contended_lock_internal(JavaThread* java_thread) {
+ int ctr = 0;
+ int yields = 0;
+ while (Atomic::cmpxchg(&_state, unlocked, locked) != unlocked) {
+ if ((++ctr & 0xFFF) == 0) {
+ BlockOp block(java_thread);
+ if (yields > 5) {
+ os::naked_short_sleep(1);
+ } else {
+ os::naked_yield();
+ yields++;
+ }
+ } else {
+ SpinPause();
+ }
+ }
+}
+
ShenandoahSimpleLock::ShenandoahSimpleLock() {
assert(os::mutex_init_done(), "Too early!");
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp
index 6237637619548..4412680bc8ce6 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp
@@ -35,34 +35,39 @@ class ShenandoahLock {
enum LockState { unlocked = 0, locked = 1 };
shenandoah_padding(0);
- volatile int _state;
+ volatile LockState _state;
shenandoah_padding(1);
volatile Thread* _owner;
shenandoah_padding(2);
+ template
+ void contended_lock_internal(JavaThread* java_thread);
+
public:
ShenandoahLock() : _state(unlocked), _owner(nullptr) {};
- void lock() {
-#ifdef ASSERT
- assert(_owner != Thread::current(), "reentrant locking attempt, would deadlock");
-#endif
- Thread::SpinAcquire(&_state, "Shenandoah Heap Lock");
-#ifdef ASSERT
- assert(_state == locked, "must be locked");
- assert(_owner == nullptr, "must not be owned");
- _owner = Thread::current();
-#endif
+ void lock(bool allow_block_for_safepoint) {
+ assert(Atomic::load(&_owner) != Thread::current(), "reentrant locking attempt, would deadlock");
+
+ // Try to lock fast, or dive into contended lock handling.
+ if (Atomic::cmpxchg(&_state, unlocked, locked) != unlocked) {
+ contended_lock(allow_block_for_safepoint);
+ }
+
+ assert(Atomic::load(&_state) == locked, "must be locked");
+ assert(Atomic::load(&_owner) == nullptr, "must not be owned");
+ DEBUG_ONLY(Atomic::store(&_owner, Thread::current());)
}
void unlock() {
-#ifdef ASSERT
- assert (_owner == Thread::current(), "sanity");
- _owner = nullptr;
-#endif
- Thread::SpinRelease(&_state);
+ assert(Atomic::load(&_owner) == Thread::current(), "sanity");
+ DEBUG_ONLY(Atomic::store(&_owner, (Thread*)nullptr);)
+ OrderAccess::fence();
+ Atomic::store(&_state, unlocked);
}
+ void contended_lock(bool allow_block_for_safepoint);
+
bool owned_by_self() {
#ifdef ASSERT
return _state == locked && _owner == Thread::current();
@@ -77,9 +82,9 @@ class ShenandoahLocker : public StackObj {
private:
ShenandoahLock* const _lock;
public:
- ShenandoahLocker(ShenandoahLock* lock) : _lock(lock) {
+ ShenandoahLocker(ShenandoahLock* lock, bool allow_block_for_safepoint = false) : _lock(lock) {
if (_lock != nullptr) {
- _lock->lock();
+ _lock->lock(allow_block_for_safepoint);
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
index 4725b8c3dfae7..4c45f4a7659b8 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
@@ -66,7 +66,7 @@ void ShenandoahMark::clear() {
ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
}
-template
+template
void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req) {
ShenandoahObjToScanQueue* q = get_queue(w);
@@ -76,48 +76,55 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe
// TODO: We can clean up this if we figure out how to do templated oop closures that
// play nice with specialized_oop_iterators.
if (heap->has_forwarded_objects()) {
- using Closure = ShenandoahMarkUpdateRefsClosure;
+ using Closure = ShenandoahMarkUpdateRefsClosure