diff --git a/.github/actions/upload-bundles/action.yml b/.github/actions/upload-bundles/action.yml index b35ee3a42e9..a3ab7b47414 100644 --- a/.github/actions/upload-bundles/action.yml +++ b/.github/actions/upload-bundles/action.yml @@ -41,10 +41,11 @@ runs: id: bundles run: | # Rename bundles to consistent names - jdk_bundle_zip="$(ls build/*/bundles/jdk-*_bin${{ inputs.debug-suffix }}.zip 2> /dev/null || true)" - jdk_bundle_tar_gz="$(ls build/*/bundles/jdk-*_bin${{ inputs.debug-suffix }}.tar.gz 2> /dev/null || true)" - symbols_bundle="$(ls build/*/bundles/jdk-*_bin${{ inputs.debug-suffix }}-symbols.tar.gz 2> /dev/null || true)" - tests_bundle="$(ls build/*/bundles/jdk-*_bin-tests${{ inputs.debug-suffix }}.tar.gz 2> /dev/null || true)" + # SapMachine 2020-11-04: Adapt bundle names + jdk_bundle_zip="$(ls build/*/bundles/sapmachine-jdk-*_bin${{ inputs.debug-suffix }}.zip 2> /dev/null || true)" + jdk_bundle_tar_gz="$(ls build/*/bundles/sapmachine-jdk-*_bin${{ inputs.debug-suffix }}.tar.gz 2> /dev/null || true)" + symbols_bundle="$(ls build/*/bundles/sapmachine-jdk-*_bin${{ inputs.debug-suffix }}-symbols.tar.gz 2> /dev/null || true)" + tests_bundle="$(ls build/*/bundles/sapmachine-jdk-*_bin-tests${{ inputs.debug-suffix }}.tar.gz 2> /dev/null || true)" mkdir bundles diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000000..ef992a92ada --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,4 @@ +The description of this pull request goes here. + +fixes #Issue + diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 067548b9768..7d2b67ce6bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,14 +23,18 @@ # questions. # -name: 'OpenJDK GHA Sanity Checks' +# SapMachine 2022-06-22: Change the name of the GitHub Action +name: 'SapMachine GHA Sanity Checks' on: push: branches-ignore: - - master - - pr/* - - jdk* + # SapMachine 2020-11-04: Ignore sapmachine branch + - sapmachine + # SapMachine 2020-11-04: Trigger on pull request + pull_request: + branches: + - sapmachine workflow_dispatch: inputs: platforms: @@ -56,6 +60,8 @@ jobs: select: name: 'Select platforms' + # SapMachine 2022-06-23: On 'pull_request' we only want to run GHA if the PR comes from a remote repo. Otherwise we have the run on 'push' already as a check. + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }} runs-on: ubuntu-22.04 outputs: linux-x64: ${{ steps.include.outputs.linux-x64 }} @@ -81,7 +87,8 @@ jobs: function check_platform() { if [[ $GITHUB_EVENT_NAME == workflow_dispatch ]]; then input='${{ github.event.inputs.platforms }}' - elif [[ $GITHUB_EVENT_NAME == push ]]; then + # SapMachine 2022-06-24: Also handle 'pull_request' event. + elif [[ $GITHUB_EVENT_NAME == push ]] || [[ $GITHUB_EVENT_NAME == pull_request ]]; then if [[ '${{ !secrets.JDK_SUBMIT_FILTER || startsWith(github.ref, 'refs/heads/submit/') }}' == 'false' ]]; then # If JDK_SUBMIT_FILTER is set, and this is not a "submit/" branch, don't run anything >&2 echo 'JDK_SUBMIT_FILTER is set and not a "submit/" branch' @@ -344,7 +351,8 @@ jobs: remove-bundles: name: 'Remove bundle artifacts' runs-on: ubuntu-22.04 - if: always() + # SapMachine 2022-06-23: On 'pull_request' we only want to run GHA if the PR comes from a remote repo. Otherwise we have the run on 'push' already as a check. + if: ${{ github.event_name != 'pull_request' || github.repository != 'SAP/SapMachine' }} needs: - build-linux-x64 - build-linux-x86 diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml new file mode 100644 index 00000000000..1f844028e62 --- /dev/null +++ b/.github/workflows/wiki.yml @@ -0,0 +1,33 @@ +# Runs update-wiki action every day at 20:00 UTC + +name: 'Wiki Update' + +on: + workflow_dispatch: + schedule: + - cron: '0 20 * * *' + +jobs: + wiki: + if: ${{ github.event_name != 'schedule' || github.repository == 'SAP/SapMachine' }} + runs-on: ubuntu-latest + steps: + - name: Checkout SapMachine Wiki source + uses: actions/checkout@v4 + with: + repository: 'SAP/SapMachine.wiki.git' + ref: 'master' + - name: Configure git + run: | + git config user.name "SapMachine Github Actions Bot" + git config user.email "sapmachine@sap.com" + git remote set-url origin https://github.com/SAP/SapMachine.wiki.git + - name: Update Wiki + run: | + pip3 install feedparser + python3 scripts/update_blogs.py update + git commit -a -m "Update blogs" || echo "No updates" + - name: Push changes + run: git push origin master + working-directory: . + shell: bash diff --git a/README.md b/README.md index b3f30676b3c..cb4016e8a46 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,50 @@ -# Welcome to the JDK! +[![GitHub release (latest by date)](https://img.shields.io/github/downloads/sap/sapmachine/latest/total?label=Downloads%20of%20Latest%20Release)](https://sap.github.io/SapMachine/#download) [![DockerPulls](https://img.shields.io/docker/pulls/_/sapmachine?label=Docker%20Pulls)](https://hub.docker.com/_/sapmachine) -For build instructions please see the -[online documentation](https://openjdk.org/groups/build/doc/building.html), -or either of these files: + -- [doc/building.html](doc/building.html) (html version) -- [doc/building.md](doc/building.md) (markdown version) +# [](#SapMachine) SapMachine +This project contains a downstream version of the [OpenJDK](http://openjdk.java.net/) project. It is used to build and maintain a SAP supported version of OpenJDK for SAP customers and partners who wish to use OpenJDK to run their applications. -See for more information about the OpenJDK -Community and the JDK and see for JDK issue -tracking. +We want to stress that this is clearly a "*friendly fork*". SAP is committed to ensuring the continued success of the Java platform. SAP is: + +* A member of the [JCP Executive committee](https://jcp.org/en/participation/committee) since 2001 and recently served in the [JSR 379 (Java SE 9)](https://www.jcp.org/en/jsr/detail?id=379), [JSR 383 (Java SE 18.3)](https://www.jcp.org/en/jsr/detail?id=383), [JSR 384 (Java SE 11)](https://www.jcp.org/en/jsr/detail?id=384), [JSR 386 (Java SE 12)](https://www.jcp.org/en/jsr/detail?id=386), [JSR 388 (Java SE 13)](https://www.jcp.org/en/jsr/detail?id=388), [JSR 389 (Java SE 14)](https://www.jcp.org/en/jsr/detail?id=389), [JSR 390 (Java SE 15)](https://www.jcp.org/en/jsr/detail?id=390), [JSR 391 (Java SE 16)](https://www.jcp.org/en/jsr/detail?id=391), [JSR 392 (Java SE 17)](https://www.jcp.org/en/jsr/detail?id=392), [JSR 393 (Java SE 18)](https://www.jcp.org/en/jsr/detail?id=393), [JSR 394 (Java SE 19)](https://www.jcp.org/en/jsr/detail?id=394), [JSR 395 (Java SE 20)](https://www.jcp.org/en/jsr/detail?id=395), [JSR 396 (Java SE 21)](https://www.jcp.org/en/jsr/detail?id=396) and [JSR 397 (Java SE 22)](https://www.jcp.org/en/jsr/detail?id=397) Expert Groups. + +* Among the biggest external contributors to the OpenJDK project (see fix ratio for OpenJDK [11](https://blogs.oracle.com/java-platform-group/building-jdk-11-together), [12](https://blogs.oracle.com/java-platform-group/the-arrival-of-java-12), [13](https://blogs.oracle.com/java-platform-group/the-arrival-of-java-13), [14](https://blogs.oracle.com/java-platform-group/the-arrival-of-java-14), [15](https://blogs.oracle.com/java-platform-group/the-arrival-of-java-15), [16](https://inside.java/2021/03/16/the-arrival-of-java16/), [17](https://inside.java/2021/09/14/the-arrival-of-java17/), [18](https://inside.java/2022/03/22/the-arrival-of-java18/), [19](https://inside.java/2022/09/20/the-arrival-of-java-19/), [20](https://inside.java/2023/03/21/the-arrival-of-java-20/), [21](https://inside.java/2023/09/19/the-arrival-of-java-21/), [22](https://inside.java/2024/03/19/the-arrival-of-java-22/)). + +* Leading the [OpenJDK 17 updates project](https://wiki.openjdk.java.net/display/JDKUpdates/JDK+17u) and heavily supporting the [OpenJDK 11](https://wiki.openjdk.java.net/display/JDKUpdates/JDK11u) and [OpenJDK 21](https://wiki.openjdk.java.net/display/JDKUpdates/JDK+21u) updates projects. + +* Leading the [PowerPC/AIX porting project](http://openjdk.java.net/projects/ppc-aix-port/). + +* Contributing many new features inspired by Java stakeholders within SAP to the OpenJDK project. This ensures such features are available in long reach and for everybody. Rarely we add such features to SapMachine directly to keep the diff of this project as small as possible. + +* Creating tools for developers + * [JFR Event Collection](https://sapmachine.io/jfrevents/): Information on all JFR events for a specific JDK + * [AP-Loader](https://github.com/jvm-profiling-tools/ap-loader): AsyncProfiler in a single cross-platform JAR + + +## Downloads + +Check out the [Download](https://sap.github.io/SapMachine/#download) section on [https://sapmachine.io](https://sapmachine.io). + +## Documentation +Check out our [FAQs](https://github.com/SAP/SapMachine/wiki/Frequently-Asked-Questions) and [wikipages](https://github.com/SAP/SapMachine/wiki) for information about: +* [Installation](https://github.com/SAP/SapMachine/wiki/Installation) and [Docker Images](https://github.com/SAP/SapMachine/wiki/Docker-Images) +* [Certifications and Java Compatibility](https://github.com/SAP/SapMachine/wiki/Certification-and-Java-Compatibility) +* [SapMachine Development Process](https://github.com/SAP/SapMachine/wiki/SapMachine-Development-Process) + +## Have an issue? +If it's SapMachine specific please let us know by filing a [new issue](https://github.com/SAP/SapMachine/issues/new). + +Please notice that the SapMachine [issue tracker](https://github.com/SAP/SapMachine/issues) is mainly used internally by the SapMachine team to organize its work (i.e. sync with upstream, downporting fixes, add SapMachine specific features, etc.). + +General VM/JDK bugs are maintained directly in the [OpenJDK Bug System](https://bugs.openjdk.java.net/). You can open a SapMachine issue with a reference to an open or resolved OpenJDK bug if you want us to resolve the issue or downport the fix to a specific SapMachine version. If you find a general VM/JDK bug in SapMachine and don't have write access to the OpenJDK Bug System you can open an issue here and we'll take care to open a corresponding OpenJDK bug for it. + +Every SapMachine release contains at least all the fixes of the corresponding OpenJDK release it is based on. You can easily find the OpenJDK base version by looking at the [SapMachine version string](https://github.com/SAP/SapMachine/wiki/Differences-between-SapMachine-and-OpenJDK#version-numbers). + +You can find the [Differences between SapMachine and OpenJDK](https://github.com/SAP/SapMachine/wiki/Differences-between-SapMachine-and-OpenJDK) and the [Features Contributed by SAP](https://github.com/SAP/SapMachine/wiki/Features-Contributed-by-SAP) in the [SapMachine Wiki](https://github.com/SAP/SapMachine/wiki). + +## Contributing +We currently do not accept external contributions for this project. If you want to improve the code or fix a bug please consider contributing directly to the upstream [OpenJDK](http://openjdk.java.net/contribute/) project. Our repositories will be regularly synced with the upstream project so any improvements in the upstream OpenJDK project will directly become visible in our project as well. + +## License +This project is run under the same licensing terms as the upstream OpenJDK project. Please see the [LICENSE](LICENSE) file in the top-level directory for more information. diff --git a/make/Bundles.gmk b/make/Bundles.gmk index 0901e415a8a..6927fe607fb 100644 --- a/make/Bundles.gmk +++ b/make/Bundles.gmk @@ -175,8 +175,9 @@ ifeq ($(call isTargetOs, macosx)+$(DEBUG_LEVEL), true+release) else JDK_IMAGE_HOMEDIR := $(JDK_IMAGE_DIR) JRE_IMAGE_HOMEDIR := $(JRE_IMAGE_DIR) - JDK_BUNDLE_SUBDIR := jdk-$(VERSION_NUMBER) - JRE_BUNDLE_SUBDIR := jre-$(VERSION_NUMBER) + # SapMachine 2020-11-04: Adapt bundle names + JDK_BUNDLE_SUBDIR := sapmachine-jdk-$(VERSION_NUMBER) + JRE_BUNDLE_SUBDIR := sapmachine-jre-$(VERSION_NUMBER) ifneq ($(DEBUG_LEVEL), release) JDK_BUNDLE_SUBDIR := $(JDK_BUNDLE_SUBDIR)/$(DEBUG_LEVEL) JRE_BUNDLE_SUBDIR := $(JRE_BUNDLE_SUBDIR)/$(DEBUG_LEVEL) diff --git a/make/Images.gmk b/make/Images.gmk index bfad1ad563c..fb5cb7b4cce 100644 --- a/make/Images.gmk +++ b/make/Images.gmk @@ -46,8 +46,9 @@ ALL_MODULES := $(call FindAllModules) $(EXTRA_MODULES) $(eval $(call ReadImportMetaData)) +# SapMachine 2021-09-24: add jcmd to JRE JRE_MODULES += $(filter $(ALL_MODULES), $(BOOT_MODULES) \ - $(PLATFORM_MODULES) jdk.jdwp.agent) + $(PLATFORM_MODULES) jdk.jdwp.agent jdk.jcmd) JDK_MODULES += $(ALL_MODULES) JRE_MODULES_LIST := $(call CommaList, $(JRE_MODULES)) diff --git a/make/MacBundles.gmk b/make/MacBundles.gmk index 8ed6476c506..448693d9d72 100644 --- a/make/MacBundles.gmk +++ b/make/MacBundles.gmk @@ -40,7 +40,12 @@ ifeq ($(call isTargetOs, macosx), true) MACOSX_PLIST_SRC := $(TOPDIR)/make/data/bundle - BUNDLE_NAME := $(MACOSX_BUNDLE_NAME_BASE) $(VERSION_SHORT) + # SapMachine 2023-06-24: ea bundles should have build number in CFBundleName + ifeq ($(VERSION_PRE), ea) + BUNDLE_NAME := $(MACOSX_BUNDLE_NAME_BASE) $(VERSION_STRING) + else + BUNDLE_NAME := $(MACOSX_BUNDLE_NAME_BASE) $(VERSION_SHORT) + endif BUNDLE_INFO := $(MACOSX_BUNDLE_NAME_BASE) $(VERSION_STRING) ifeq ($(COMPANY_NAME), N/A) BUNDLE_VENDOR := UNDEFINED diff --git a/make/autoconf/spec.gmk.template b/make/autoconf/spec.gmk.template index 79a541dc78c..12c5a9f51f9 100644 --- a/make/autoconf/spec.gmk.template +++ b/make/autoconf/spec.gmk.template @@ -879,16 +879,18 @@ GRAAL_BUILDER_IMAGE_SUBDIR := graal-builder-jdk GRAAL_BUILDER_IMAGE_DIR := $(IMAGES_OUTPUTDIR)/$(GRAAL_BUILDER_IMAGE_SUBDIR) # Macosx bundles directory definitions -JDK_MACOSX_BUNDLE_SUBDIR := jdk-bundle -JRE_MACOSX_BUNDLE_SUBDIR := jre-bundle -JDK_MACOSX_BUNDLE_SUBDIR_SIGNED := jdk-bundle-signed -JRE_MACOSX_BUNDLE_SUBDIR_SIGNED := jre-bundle-signed +# SapMachine 2020-11-04: Adapt bundle names +JDK_MACOSX_BUNDLE_SUBDIR := sapmachine-jdk-bundle +JRE_MACOSX_BUNDLE_SUBDIR := sapmachine-jre-bundle +JDK_MACOSX_BUNDLE_SUBDIR_SIGNED := sapmachine-jdk-bundle-signed +JRE_MACOSX_BUNDLE_SUBDIR_SIGNED := sapmachine-jre-bundle-signed JDK_MACOSX_BUNDLE_DIR = $(IMAGES_OUTPUTDIR)/$(JDK_MACOSX_BUNDLE_SUBDIR) JRE_MACOSX_BUNDLE_DIR = $(IMAGES_OUTPUTDIR)/$(JRE_MACOSX_BUNDLE_SUBDIR) JDK_MACOSX_BUNDLE_DIR_SIGNED = $(IMAGES_OUTPUTDIR)/$(JDK_MACOSX_BUNDLE_SUBDIR_SIGNED) JRE_MACOSX_BUNDLE_DIR_SIGNED = $(IMAGES_OUTPUTDIR)/$(JRE_MACOSX_BUNDLE_SUBDIR_SIGNED) -JDK_MACOSX_BUNDLE_TOP_DIR = jdk-$(VERSION_NUMBER).jdk -JRE_MACOSX_BUNDLE_TOP_DIR = jre-$(VERSION_NUMBER).jre +# SapMachine 2020-11-04: Adapt bundle names +JDK_MACOSX_BUNDLE_TOP_DIR = sapmachine-jdk-$(VERSION_NUMBER).jdk +JRE_MACOSX_BUNDLE_TOP_DIR = sapmachine-jre-$(VERSION_NUMBER).jre JDK_MACOSX_CONTENTS_SUBDIR = $(JDK_MACOSX_BUNDLE_TOP_DIR)/Contents JRE_MACOSX_CONTENTS_SUBDIR = $(JRE_MACOSX_BUNDLE_TOP_DIR)/Contents JDK_MACOSX_CONTENTS_DIR = $(JDK_MACOSX_BUNDLE_DIR)/$(JDK_MACOSX_CONTENTS_SUBDIR) @@ -911,17 +913,18 @@ ifeq ($(OPENJDK_TARGET_OS), windows) else JDK_BUNDLE_EXTENSION := tar.gz endif -JDK_BUNDLE_NAME := jdk-$(BASE_NAME)_bin$(DEBUG_PART).$(JDK_BUNDLE_EXTENSION) -JRE_BUNDLE_NAME := jre-$(BASE_NAME)_bin$(DEBUG_PART).$(JDK_BUNDLE_EXTENSION) -JDK_SYMBOLS_BUNDLE_NAME := jdk-$(BASE_NAME)_bin$(DEBUG_PART)-symbols.tar.gz -TEST_DEMOS_BUNDLE_NAME := jdk-$(BASE_NAME)_bin-tests-demos$(DEBUG_PART).tar.gz -TEST_BUNDLE_NAME := jdk-$(BASE_NAME)_bin-tests$(DEBUG_PART).tar.gz -DOCS_JDK_BUNDLE_NAME := jdk-$(BASE_NAME)_doc-api-spec$(DEBUG_PART).tar.gz +# SapMachine 2020-11-04: Adapt bundle names +JDK_BUNDLE_NAME := sapmachine-jdk-$(BASE_NAME)_bin$(DEBUG_PART).$(JDK_BUNDLE_EXTENSION) +JRE_BUNDLE_NAME := sapmachine-jre-$(BASE_NAME)_bin$(DEBUG_PART).$(JDK_BUNDLE_EXTENSION) +JDK_SYMBOLS_BUNDLE_NAME := sapmachine-jdk-$(BASE_NAME)_bin$(DEBUG_PART)-symbols.tar.gz +TEST_DEMOS_BUNDLE_NAME := sapmachine-jdk-$(BASE_NAME)_bin-tests-demos$(DEBUG_PART).tar.gz +TEST_BUNDLE_NAME := sapmachine-jdk-$(BASE_NAME)_bin-tests$(DEBUG_PART).tar.gz +DOCS_JDK_BUNDLE_NAME := sapmachine-jdk-$(BASE_NAME)_doc-api-spec$(DEBUG_PART).tar.gz DOCS_JAVASE_BUNDLE_NAME := javase-$(BASE_NAME)_doc-api-spec$(DEBUG_PART).tar.gz -DOCS_REFERENCE_BUNDLE_NAME := jdk-reference-$(BASE_NAME)_doc-api-spec$(DEBUG_PART).tar.gz -STATIC_LIBS_BUNDLE_NAME := jdk-$(BASE_NAME)_bin-static-libs$(DEBUG_PART).tar.gz -STATIC_LIBS_GRAAL_BUNDLE_NAME := jdk-$(BASE_NAME)_bin-static-libs-graal$(DEBUG_PART).tar.gz -JCOV_BUNDLE_NAME := jdk-jcov-$(BASE_NAME)_bin$(DEBUG_PART).$(JDK_BUNDLE_EXTENSION) +DOCS_REFERENCE_BUNDLE_NAME := sapmachine-jdk-reference-$(BASE_NAME)_doc-api-spec$(DEBUG_PART).tar.gz +STATIC_LIBS_BUNDLE_NAME := sapmachine-jdk-$(BASE_NAME)_bin-static-libs$(DEBUG_PART).tar.gz +STATIC_LIBS_GRAAL_BUNDLE_NAME := sapmachine-$(BASE_NAME)_bin-static-libs-graal$(DEBUG_PART).tar.gz +JCOV_BUNDLE_NAME := sapmachine-jdk-jcov-$(BASE_NAME)_bin$(DEBUG_PART).$(JDK_BUNDLE_EXTENSION) JDK_BUNDLE := $(BUNDLES_OUTPUTDIR)/$(JDK_BUNDLE_NAME) JRE_BUNDLE := $(BUNDLES_OUTPUTDIR)/$(JRE_BUNDLE_NAME) diff --git a/make/conf/github-actions.conf b/make/conf/github-actions.conf index deb5e36f605..2bbd943591d 100644 --- a/make/conf/github-actions.conf +++ b/make/conf/github-actions.conf @@ -29,17 +29,17 @@ GTEST_VERSION=1.14.0 JTREG_VERSION=7.4+1 LINUX_X64_BOOT_JDK_EXT=tar.gz -LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk22/830ec9fcccef480bb3e73fb7ecafe059/36/GPL/openjdk-22_linux-x64_bin.tar.gz -LINUX_X64_BOOT_JDK_SHA256=4d65cc6ed28711768fd72c2043a7925f7c83f5f51bb64970bd9d52f7791fc6ac +LINUX_X64_BOOT_JDK_URL=https://github.com/SAP/SapMachine/releases/download/sapmachine-22.0.1/sapmachine-jdk-22.0.1_linux-x64_bin.tar.gz +LINUX_X64_BOOT_JDK_SHA256=21794bbccd5bcfd4ef40d5a6b7ae7f43aea5311365f7470239e1899012329888 MACOS_X64_BOOT_JDK_EXT=tar.gz -MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk22/830ec9fcccef480bb3e73fb7ecafe059/36/GPL/openjdk-22_macos-x64_bin.tar.gz -MACOS_X64_BOOT_JDK_SHA256=ae31fe10916429e3fe284266095067a5ce9fecbdc03ff1a079d20459f731ca36 +MACOS_X64_BOOT_JDK_URL=https://github.com/SAP/SapMachine/releases/download/sapmachine-22.0.1/sapmachine-jdk-22.0.1_macos-x64_bin.tar.gz +MACOS_X64_BOOT_JDK_SHA256=99d49029c3fd6afaf8630ddb6444e879b203fe50c868b2e24c6b738330ed9876 MACOS_AARCH64_BOOT_JDK_EXT=tar.gz -MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk22/830ec9fcccef480bb3e73fb7ecafe059/36/GPL/openjdk-22_macos-aarch64_bin.tar.gz -MACOS_AARCH64_BOOT_JDK_SHA256=d10f82429d01047968c52c7975c326388cb5d212791e14c1de21c987463a4b53 +MACOS_AARCH64_BOOT_JDK_URL=https://github.com/SAP/SapMachine/releases/download/sapmachine-22.0.1/sapmachine-jdk-22.0.1_macos-aarch64_bin.tar.gz +MACOS_AARCH64_BOOT_JDK_SHA256=cce91a02f4a2c6cc7f6be169794df9b18d462f1daaa200b3475df7936929562c WINDOWS_X64_BOOT_JDK_EXT=zip -WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk22/830ec9fcccef480bb3e73fb7ecafe059/36/GPL/openjdk-22_windows-x64_bin.zip -WINDOWS_X64_BOOT_JDK_SHA256=8f5138fecb53c08c20abd4fa6812f9400051f3852582a2142ffda0dff73a5824 +WINDOWS_X64_BOOT_JDK_URL=https://github.com/SAP/SapMachine/releases/download/sapmachine-22.0.1/sapmachine-jdk-22.0.1_windows-x64_bin.zip +WINDOWS_X64_BOOT_JDK_SHA256=e0f11b9697f2cf6c8d04fadab1086c8bb5b3f0499c17b7d172dcbca9891f8dd5 diff --git a/make/modules/java.base/Copy.gmk b/make/modules/java.base/Copy.gmk index 911fff60ea8..b9b0cfe7087 100644 --- a/make/modules/java.base/Copy.gmk +++ b/make/modules/java.base/Copy.gmk @@ -289,3 +289,16 @@ $(eval $(call SetupTextFileProcessing, CREATE_CLASSFILE_CONSTANTS_H, \ TARGETS += $(CREATE_CLASSFILE_CONSTANTS_H) ################################################################################ +# SapMachine 2023-11-28: Copy mallochook.h + +ifeq ($(call isTargetOs, linux macosx), true) + + $(eval $(call SetupCopyFiles, CREATE_MALLOC_HOOKS_H, \ + FILES := $(TOPDIR)/src/java.base/unix/native/libmallochooks/mallochooks.h, \ + DEST := $(SUPPORT_OUTPUTDIR)/modules_include/java.base/, \ + )) + + TARGETS += $(CREATE_MALLOC_HOOKS_H) +endif + +################################################################################ diff --git a/make/modules/java.base/lib/CoreLibraries.gmk b/make/modules/java.base/lib/CoreLibraries.gmk index 7fed18491d1..177965a525e 100644 --- a/make/modules/java.base/lib/CoreLibraries.gmk +++ b/make/modules/java.base/lib/CoreLibraries.gmk @@ -180,3 +180,14 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJLI, \ )) TARGETS += $(BUILD_LIBJLI) + +# SapMachine 2023-11-28: build libmallochooks +ifeq ($(call isTargetOs, linux macosx), true) + $(eval $(call SetupJdkLibrary, BUILD_LIBMALLOCHOOKS, \ + NAME := mallochooks, \ + OPTIMIZATION := LOW, \ + LIBS := -ldl, \ + )) + + TARGETS += $(BUILD_LIBMALLOCHOOKS) +endif diff --git a/make/modules/jdk.jdwp.agent/Lib.gmk b/make/modules/jdk.jdwp.agent/Lib.gmk index d83ff06f786..96e85c56f6a 100644 --- a/make/modules/jdk.jdwp.agent/Lib.gmk +++ b/make/modules/jdk.jdwp.agent/Lib.gmk @@ -47,6 +47,22 @@ TARGETS += $(BUILD_LIBDT_SOCKET) ## Build libjdwp ################################################################################ +# SapMachine 2024-02-16: Add dt_filesocket implementation +$(eval $(call SetupJdkLibrary, BUILD_LIBDT_FILESOCKET, \ + NAME := dt_filesocket, \ + OPTIMIZATION := LOW, \ + DISABLED_WARNINGS_clang_fileSocketTransport.c := format-nonliteral, \ + EXTRA_HEADER_DIRS := \ + include \ + libjdwp/export, \ + LIBS_windows := ws2_32.lib Advapi32.lib, \ +)) + +# Include file socket transport with JDWP agent to allow for safe local debugging +TARGETS += $(BUILD_LIBDT_FILESOCKET) + +################################################################################ + # JDWP_LOGGING causes log messages to be compiled into the library. $(eval $(call SetupJdkLibrary, BUILD_LIBJDWP, \ NAME := jdwp, \ diff --git a/make/test/JtregNativeHotspot.gmk b/make/test/JtregNativeHotspot.gmk index ecc7e34c917..0f59be3bf71 100644 --- a/make/test/JtregNativeHotspot.gmk +++ b/make/test/JtregNativeHotspot.gmk @@ -853,6 +853,9 @@ ifeq ($(call isTargetOs, linux), true) BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exeFPRegs := -ldl BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libAsyncGetCallTraceTest := -ldl BUILD_HOTSPOT_JTREG_LIBRARIES_LDFLAGS_libfast-math := -ffast-math +# SapMachine 2023-10-04: Added link flags for malloc hooks tests + BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exetestmallochooks := -ldl + BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libtestmallochooks := -ldl else BUILD_HOTSPOT_JTREG_EXCLUDE += libtest-rw.c libtest-rwx.c \ exeinvoke.c exestack-gap.c exestack-tls.c libAsyncGetCallTraceTest.cpp diff --git a/src/hotspot/os/aix/vitals_aix.cpp b/src/hotspot/os/aix/vitals_aix.cpp new file mode 100644 index 00000000000..55f4b9470a7 --- /dev/null +++ b/src/hotspot/os/aix/vitals_aix.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019, 2021 SAP SE. All rights reserved. + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "runtime/os.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "vitals/vitals_internals.hpp" + +namespace sapmachine_vitals { + +bool platform_columns_initialize() { + return true; +} + +void sample_platform_values(Sample* record) { +} + +} // namespace sapmachine_vitals diff --git a/src/hotspot/os/bsd/vitals_bsd.cpp b/src/hotspot/os/bsd/vitals_bsd.cpp new file mode 100644 index 00000000000..55f4b9470a7 --- /dev/null +++ b/src/hotspot/os/bsd/vitals_bsd.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019, 2021 SAP SE. All rights reserved. + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "runtime/os.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "vitals/vitals_internals.hpp" + +namespace sapmachine_vitals { + +bool platform_columns_initialize() { + return true; +} + +void sample_platform_values(Sample* record) { +} + +} // namespace sapmachine_vitals diff --git a/src/hotspot/os/linux/globals_linux.hpp b/src/hotspot/os/linux/globals_linux.hpp index 1cb0b553c52..e09b44a06ee 100644 --- a/src/hotspot/os/linux/globals_linux.hpp +++ b/src/hotspot/os/linux/globals_linux.hpp @@ -66,9 +66,47 @@ "resulting in file-backed shared mappings of the process to " \ "be dumped into the corefile.") \ \ + /* SapMachine 2022-05-01: HiMemReport */ \ + product(bool, HiMemReport, false, \ + "VM writes a high memory report and optionally execute " \ + "additional jcmds if its rss+swap reaches 66%, 75% or 90% of " \ + "a maximum. If the VM is containerized, that maximum is the " \ + "container memory limit at VM start. If the VM is not " \ + "containerized, that maximum is half of the total physical " \ + "memory. The maximum can be overridden with " \ + "-XX:HiMemReportMaximum=. Per default, the report is " \ + "printed to stderr. If HiMemReportDir is specified, that " \ + "report is redirected to \"/" \ + "_pid_.log\".") \ + product(size_t, HiMemReportMax, 0, \ + "Overrides the maximum reference size for HiMemReport.") \ + product(ccstr, HiMemReportDir, NULL, \ + "Specifies a directory into which reports are written. Gets " \ + "created (one level only) if it does not exist.") \ + product(ccstr, HiMemReportExec, NULL, \ + "Specifies one or more jcmds to be executed after a high " \ + "memory report has been written. Multiple commands are " \ + "separated by ';'. Command output is written to stderr. If " \ + "HiMemReportDir is specified, command output is redirected to " \ + "\"/_pid_timestamp.(out|err)\"." \ + "If one of the commands is \"GC.heap_dump\" and its " \ + "arguments are omitted, the heap dump is written as " \ + "\"GC.heap_dump_pid_timestamp\" to either report " \ + "directory or current directory if HiMemReportDir is " \ + "omitted.\n" \ + "Example: \"-XX:HiMemReportExec=GC.class_histogram -all;GC.heap_dump\"") \ + \ product(bool, UseCpuAllocPath, false, DIAGNOSTIC, \ "Use CPU_ALLOC code path in os::active_processor_count ") \ \ + /* SapMachine 2021-09-01: malloc-trace */ \ + product(bool, EnableMallocTrace, false, DIAGNOSTIC, \ + "Enable malloc trace at VM initialization") \ + \ + /* SapMachine 2021-09-01: malloc-trace */ \ + product(bool, PrintMallocTraceAtExit, false, DIAGNOSTIC, \ + "Print Malloc Trace upon VM exit") \ + \ product(bool, DumpPerfMapAtExit, false, DIAGNOSTIC, \ "Write map file for Linux perf tool at exit") \ \ diff --git a/src/hotspot/os/linux/malloctrace/assertHandling.cpp b/src/hotspot/os/linux/malloctrace/assertHandling.cpp new file mode 100644 index 00000000000..64fca1f58b8 --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/assertHandling.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "malloctrace/assertHandling.hpp" +#include "malloctrace/locker.hpp" +#include "malloctrace/mallocTrace.hpp" +#include "runtime/atomic.hpp" + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +namespace sap { + +static volatile bool g_asserting = false; + +bool prepare_assert() { + + // Ignore all but the first assert + if (Atomic::cmpxchg(&g_asserting, false, true) != false) { + ::printf("Ignoring secondary assert in malloc trace...\n"); + return false; + } + + // manually disable lock. + Locker::unlock(); + + // disable hooks (if this asserts too, + // the assert is just ignored, see above) + MallocTracer::disable(); + + return true; +} + +} // namespace sap + +#endif // HAVE_GLIBC_MALLOC_HOOKS diff --git a/src/hotspot/os/linux/malloctrace/assertHandling.hpp b/src/hotspot/os/linux/malloctrace/assertHandling.hpp new file mode 100644 index 00000000000..9fe2bfd771c --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/assertHandling.hpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_MALLOCTRACE_ASSERTHANDLING_HPP +#define OS_LINUX_MALLOCTRACE_ASSERTHANDLING_HPP + +#include "malloctrace/mallocTrace.hpp" +#include "utilities/globalDefinitions.hpp" + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +namespace sap { + +// Asserts in the malloctrace code need a bit of extra attention. +// We must prevent the assert handler itself deadlocking. Therefore, +// before executing the assert, we: +// - must prevent recursive assert from the malloc tracer +// - manually disable the lock to prevent recursive locking (since error reporting +// never rolls back the stack this is okay) +// - disable malloc hooks + +#ifdef ASSERT + +bool prepare_assert(); + +#define malloctrace_assert(cond, ...) \ +do { \ + if (!(cond) && prepare_assert()) { \ + report_vm_error(__FILE__, __LINE__, "malloctrace_assert(" #cond ") failed", __VA_ARGS__); \ + } \ +} while (0) +#else +#define malloctrace_assert(cond, ...) +#endif + +} // namespace sap + +#endif // __GLIBC__ + +#endif // OS_LINUX_MALLOCTRACE_ASSERTHANDLING_HPP diff --git a/src/hotspot/os/linux/malloctrace/locker.cpp b/src/hotspot/os/linux/malloctrace/locker.cpp new file mode 100644 index 00000000000..baee30c3db2 --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/locker.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "malloctrace/locker.hpp" +#include "malloctrace/mallocTrace.hpp" +#include + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +namespace sap { + +//static +pthread_mutex_t Locker::_pthread_mutex = PTHREAD_MUTEX_INITIALIZER; + +} // namespace sap + +#endif // HAVE_GLIBC_MALLOC_HOOKS diff --git a/src/hotspot/os/linux/malloctrace/locker.hpp b/src/hotspot/os/linux/malloctrace/locker.hpp new file mode 100644 index 00000000000..bf000520749 --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/locker.hpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_MALLOCTRACE_LOCKER_HPP +#define OS_LINUX_MALLOCTRACE_LOCKER_HPP + +#include "malloctrace/assertHandling.hpp" +#include "malloctrace/mallocTrace.hpp" +#include "utilities/globalDefinitions.hpp" +#include + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +class outputStream; + +namespace sap { + +/////// A simple native lock using pthread mutexes + +class Locker { + static pthread_mutex_t _pthread_mutex; + bool _locked; + + bool lock() { + malloctrace_assert(!_locked, "already locked"); + if (::pthread_mutex_lock(&_pthread_mutex) != 0) { + malloctrace_assert(false, "MALLOCTRACE lock failed"); + return false; + } + return true; + } + +public: + + // Manually unlock is public since we need it in case of asserts + // (see malloctrace_assert) + static void unlock() { + ::pthread_mutex_unlock(&_pthread_mutex); + } + + Locker() : _locked(false) { + _locked = lock(); + } + + ~Locker() { + if (_locked) { + unlock(); + } + } + +}; + +} // namespace sap + +#endif // HAVE_GLIBC_MALLOC_HOOKS + +#endif // OS_LINUX_MALLOCTRACE_LOCKER_HPP diff --git a/src/hotspot/os/linux/malloctrace/mallocTrace.cpp b/src/hotspot/os/linux/malloctrace/mallocTrace.cpp new file mode 100644 index 00000000000..a0d73a08c1e --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/mallocTrace.cpp @@ -0,0 +1,388 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "jvm_io.h" +#include "malloctrace/assertHandling.hpp" +#include "malloctrace/locker.hpp" +#include "malloctrace/mallocTrace.hpp" +#include "malloctrace/mallocTracePosix.hpp" +#include "malloctrace/siteTable.hpp" +#include "memory/allStatic.hpp" +#include "runtime/globals.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +#include + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +// Since JDK-8289633 we forbid calling raw C-heap allocation functions using Kim's FORBID_C_FUNCTION. +// Callers need to explicitly opt in with ALLOW_C_FUNCTION. +// Since this code calls raw C-heap functions as a matter of course, instead of marking each call site +// with ALLOW_C_FUNCTION(..), I just mark them wholesale. +#if (__GNUC__ >= 10) +PRAGMA_DISABLE_GCC_WARNING("-Wattribute-warning") +#endif + +namespace sap { + +// Needed to stop the gcc from complaining about malloc hooks being deprecated. +PRAGMA_DISABLE_GCC_WARNING("-Wdeprecated-declarations") + +typedef void* (*malloc_hook_fun_t) (size_t len, const void* caller); +typedef void* (*realloc_hook_fun_t) (void* old, size_t len, const void* caller); +typedef void* (*memalign_hook_fun_t) (size_t alignment, size_t size, const void* caller); + +static void* my_malloc_hook(size_t size, const void *caller); +static void* my_realloc_hook(void* old, size_t size, const void *caller); +static void* my_memalign_hook(size_t alignment, size_t size, const void *caller); + +// Hook changes, hook ownership: +// +// Hooks are a global resource and everyone can change them concurrently. In practice +// this does not happen often, so using them for our purposes here is generally safe +// and we can generally rely on us being the sole changer of hooks. +// +// Exceptions: +// 1) gdb debugging facilities like mtrace() or MALLOC_CHECK_ use them too +// 2) there is a initialization race: both hooks are initially set to glibc-internal +// initialization functions which will do some stuff, them set them to NULL for the +// rest of the program run. These init functions (malloc_hook_ini() and realloc_hook_ini()), +// see malloc/hooks.c) run *lazily*, the first time malloc or realloc is called. +// So there is a race window here where we could possibly install our hooks while +// some other thread calls realloc, still sees the original function pointer, executed +// the init function and resets our hook. To make matters worse and more surprising, the +// realloc hook function also resets the malloc hook for some reason (I consider this a +// bug since realloc(3) may run way later than malloc(3)). +// +// There is nothing we can do about (1) except, well, not do it. About (2), we can effectively +// prevent that from happening by calling malloc and realloc very early. The earliest we +// can manage is during C++ dyn init of the libjvm: +struct RunAtDynInit { + RunAtDynInit() { + // Call malloc, realloc, free, calloc and posix_memalign. + // This may be overkill, but I want all hooks to have executed once, in case + // they have side effects on the other hooks (like the realloc hook which resets the malloc + // hook) + void* p = ::malloc(10); + p = ::realloc(p, 20); + ::free(p); + if (::posix_memalign(&p, 8, 10) == 0) { + ::free(p); + } + } +}; +static RunAtDynInit g_run_at_dyn_init; + +class HookControl : public AllStatic { + static bool _hooks_are_active; + static malloc_hook_fun_t _old_malloc_hook; + static realloc_hook_fun_t _old_realloc_hook; + static memalign_hook_fun_t _old_memalign_hook; + +public: + +#ifdef ASSERT + static char* print_hooks(char* out, size_t outlen) { + jio_snprintf(out, outlen, "__malloc_hook=" PTR_FORMAT ", __realloc_hook=" PTR_FORMAT ", __memalign_hook=" PTR_FORMAT ", " + "my_malloc_hook=" PTR_FORMAT ", my_realloc_hook=" PTR_FORMAT ", my_memalign_hook=" PTR_FORMAT ".", + (intptr_t)__malloc_hook, (intptr_t)__realloc_hook, (intptr_t)__memalign_hook, + (intptr_t)my_malloc_hook, (intptr_t)my_realloc_hook, (intptr_t)my_memalign_hook); + return out; + } + static void verify() { + char tmp[256]; + if (_hooks_are_active) { + malloctrace_assert(__malloc_hook == my_malloc_hook && __realloc_hook == my_realloc_hook && + __memalign_hook == my_memalign_hook, + "Hook mismatch (expected my hooks to be active). Hook state: %s", + print_hooks(tmp, sizeof(tmp))); + } else { + malloctrace_assert(__malloc_hook != my_malloc_hook && __realloc_hook != my_realloc_hook && + __memalign_hook != my_memalign_hook, + "Hook mismatch (expected default hooks to be active). Hook state: %s", + print_hooks(tmp, sizeof(tmp))); + } + } +#endif + + // Return true if my hooks are active + static bool hooks_are_active() { + DEBUG_ONLY(verify();) + return _hooks_are_active; + } + + static void enable() { + DEBUG_ONLY(verify();) + MallocStatistic::disable(NULL); + malloctrace_assert(!hooks_are_active(), "Sanity"); + _old_malloc_hook = __malloc_hook; + __malloc_hook = my_malloc_hook; + _old_realloc_hook = __realloc_hook; + __realloc_hook = my_realloc_hook; + _old_memalign_hook = __memalign_hook; + __memalign_hook = my_memalign_hook; + _hooks_are_active = true; + } + + static void disable() { + DEBUG_ONLY(verify();) + malloctrace_assert(hooks_are_active(), "Sanity"); + __malloc_hook = _old_malloc_hook; + __realloc_hook = _old_realloc_hook; + __memalign_hook = _old_memalign_hook; + _hooks_are_active = false; + } +}; + +bool HookControl::_hooks_are_active = false; +malloc_hook_fun_t HookControl::_old_malloc_hook = NULL; +realloc_hook_fun_t HookControl::_old_realloc_hook = NULL; +memalign_hook_fun_t HookControl::_old_memalign_hook = NULL; + +// A stack mark for temporarily disabling hooks - if they are active - and +// restoring the old state +class DisableHookMark { + const bool _state; +public: + DisableHookMark() : _state(HookControl::hooks_are_active()) { + if (_state) { + HookControl::disable(); + } + } + ~DisableHookMark() { + if (_state) { + HookControl::enable(); + } + } +}; + +///////////////////////////////////////////////////////////////// + +static SiteTable* g_sites = NULL; + +static bool g_use_backtrace = true; +static uint64_t g_num_captures = 0; +static uint64_t g_num_captures_without_stack = 0; + +#ifdef ASSERT +static int g_times_enabled = 0; +static int g_times_printed = 0; +#endif + +#define CAPTURE_STACK_AND_ADD_TO_SITE_TABLE \ +{ \ + Stack stack; \ + if (Stack::capture_stack(&stack, g_use_backtrace)) { \ + malloctrace_assert(g_sites != NULL, "Site table not allocated"); \ + g_sites->add_site(&stack, alloc_size); \ + } else { \ + g_num_captures_without_stack ++; \ + } \ +} + +static void* my_malloc_or_realloc_hook(void* old, size_t alloc_size) { + Locker lck; + g_num_captures ++; + + // If someone switched off tracing while we waited for the lock, just quietly do + // malloc/realloc and tippytoe out of this function. Don't modify hooks, don't + // collect stacks. + if (HookControl::hooks_are_active() == false) { + return old != NULL ? ::realloc(old, alloc_size) : ::malloc(alloc_size); + } + + // From here on disable hooks. We will collect a stack, then register it with + // the site table, then call the real malloc to satisfy the allocation for the + // caller. All of these things may internally malloc (even the sitemap, which may + // assert). These recursive mallocs should not end up in this hook otherwise we + // deadlock. + // + // Concurrency note: Concurrent threads will not be disturbed by this since: + // - either they already entered this function, in which case they wait at the lock + // - or they call malloc/realloc after we restored the hooks. In that case they + // just will end up doing the original malloc. We loose them for the statistic, + // but we wont disturb them, nor they us. + // (caveat: we assume here that the order in which we restore the hooks - which + // will appear random for outside threads - does not matter. After studying the + // glibc sources, I believe it does not.) + HookControl::disable(); + + CAPTURE_STACK_AND_ADD_TO_SITE_TABLE + + // Now do the actual allocation for the caller + void* p = old != NULL ? ::realloc(old, alloc_size) : ::malloc(alloc_size); + +#ifdef ASSERT + if ((g_num_captures % 10000) == 0) { // expensive, do this only sometimes + g_sites->verify(); + } +#endif + + // Reinstate my hooks + HookControl::enable(); + + return p; +} + +static void* my_malloc_hook(size_t size, const void *caller) { + return my_malloc_or_realloc_hook(NULL, size); +} + +static void* my_realloc_hook(void* old, size_t size, const void *caller) { + // realloc(0): "If size was equal to 0, either NULL or a pointer suitable to be passed to free() is returned." + // The glibc currently does the former (unlike malloc(0), which does the latter and can cause leaks). As long + // as we are sure the glibc returns NULL for realloc(0), we can shortcut here. + if (size == 0) { + return NULL; + } + return my_malloc_or_realloc_hook(old, size); +} + +static void* posix_memalign_wrapper(size_t alignment, size_t size) { + void* p = NULL; + if (::posix_memalign(&p, alignment, size) == 0) { + return p; + } + return NULL; +} + +static void* my_memalign_hook(size_t alignment, size_t alloc_size, const void *caller) { + Locker lck; + g_num_captures ++; + + // For explanations, see my_malloc_or_realloc_hook + + if (HookControl::hooks_are_active() == false) { + return posix_memalign_wrapper(alignment, alloc_size); + } + + HookControl::disable(); + + CAPTURE_STACK_AND_ADD_TO_SITE_TABLE + + // Now do the actual allocation for the caller + void* p = posix_memalign_wrapper(alignment, alloc_size); + +#ifdef ASSERT + if ((g_num_captures % 10000) == 0) { // expensive, do this only sometimes + g_sites->verify(); + } +#endif + + // Reinstate my hooks + HookControl::enable(); + + return p; +} + + +/////////// Externals ///////////////////////// + +bool MallocTracer::enable(bool use_backtrace) { + Locker lck; + if (!HookControl::hooks_are_active()) { + if (g_sites == NULL) { + // First time malloc trace is enabled, allocate the site table. We don't want to preallocate it + // unconditionally since it costs several MB. + g_sites = SiteTable::create(); + if (g_sites == NULL) { + return false; + } + } + HookControl::enable(); // << from this moment on concurrent threads may enter our hooks but will then wait on the lock + g_use_backtrace = use_backtrace; + DEBUG_ONLY(g_times_enabled ++;) + } + return true; +} + +void MallocTracer::disable() { + Locker lck; + if (HookControl::hooks_are_active()) { + HookControl::disable(); + } +} + +void MallocTracer::reset() { + Locker lck; + if (g_sites != NULL) { + g_sites->reset(); + g_num_captures = g_num_captures_without_stack = 0; + } +} + +void MallocTracer::reset_deltas() { + Locker lck; + if (g_sites != NULL) { + g_sites->reset_deltas(); + } +} + +void MallocTracer::print(outputStream* st, bool all) { + Locker lck; + if (g_sites != NULL) { + bool state_now = HookControl::hooks_are_active(); // query hooks before temporarily disabling them + { + DisableHookMark disableHookMark; + g_sites->print_table(st, all); + g_sites->print_stats(st); + st->cr(); + st->print_cr("Malloc trace %s.", state_now ? "on" : "off"); + if (state_now) { + st->print_cr(" (method: %s)", g_use_backtrace ? "backtrace" : "nmt-ish"); + } + st->cr(); + st->print_cr(UINT64_FORMAT " captures (" UINT64_FORMAT " without stack).", g_num_captures, g_num_captures_without_stack); + DEBUG_ONLY(g_times_printed ++;) + DEBUG_ONLY(st->print_cr("%d times enabled, %d times printed", g_times_enabled, g_times_printed)); + DEBUG_ONLY(g_sites->verify();) + // After each print, we reset table deltas + g_sites->reset_deltas(); + } + } else { + // Malloc trace has never been activated. + st->print_cr("Malloc trace off."); + } +} + +void MallocTracer::print_on_error(outputStream* st) { + // Don't lock. Don't change hooks. Just print the table stats. + if (g_sites != NULL) { + g_sites->print_stats(st); + } +} + +/////////////////////// + +// test: enable at libjvm load +// struct AutoOn { AutoOn() { MallocTracer::enable(); } }; +// static AutoOn g_autoon; + +} // namespace sap + +#endif // HAVE_GLIBC_MALLOC_HOOKS diff --git a/src/hotspot/os/linux/malloctrace/mallocTrace.hpp b/src/hotspot/os/linux/malloctrace/mallocTrace.hpp new file mode 100644 index 00000000000..ef76f0bec2d --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/mallocTrace.hpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_MALLOCTRACE_MALLOCTRACE_HPP +#define OS_LINUX_MALLOCTRACE_MALLOCTRACE_HPP + +#include "memory/allStatic.hpp" +#include "utilities/globalDefinitions.hpp" + +// MallocTracer needs glibc malloc hooks. Unfortunately, glibc removed them with 2.32. +// If we built against a newer glibc, there is no point in even trying to resolve +// them dynamically, since the binary will not run with older glibc's anyway. Therefore +// we can just disable them at built time. +#if defined(__GLIBC__) +#if (__GLIBC__ <= 2) && (__GLIBC_MINOR__ <= 31) +#define HAVE_GLIBC_MALLOC_HOOKS +#endif +#endif + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +class outputStream; + +namespace sap { + +class MallocTracer : public AllStatic { +public: + static bool enable(bool use_backtrace = false); + static void disable(); + static void reset(); + static void reset_deltas(); + static void print(outputStream* st, bool all); + static void print_on_error(outputStream* st); +}; + +} + +#endif // HAVE_GLIBC_MALLOC_HOOKS + +#endif // OS_LINUX_MALLOCTRACE_MALLOCTRACE_HPP diff --git a/src/hotspot/os/linux/malloctrace/mallocTraceDCmd.cpp b/src/hotspot/os/linux/malloctrace/mallocTraceDCmd.cpp new file mode 100644 index 00000000000..585a4624735 --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/mallocTraceDCmd.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "malloctrace/mallocTrace.hpp" +#include "malloctrace/mallocTraceDCmd.hpp" +#include "utilities/globalDefinitions.hpp" + +#include + +namespace sap { + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +// By default, let's use nmt-like capturing. I see (very rarely) crashes with backtrace(3) on x86. +// backtrace(3) gives us better callstack but runs a (small) risk of crashing, especially on x86. +const bool use_backtrace_default = false; + +void MallocTraceDCmd::execute(DCmdSource source, TRAPS) { + const char* const subopt = _suboption.value(); + if (::strcmp(_option.value(), "on") == 0) { + bool use_backtrace = use_backtrace_default; + if (subopt != NULL) { + if (::strcmp(subopt, "bt") == 0) { + use_backtrace = true; + } else if (::strcmp(subopt, "nmt") == 0) { + use_backtrace = false; + } else { + _output->print_cr("Invalid sub option"); + return; + } + } + if (MallocTracer::enable(use_backtrace)) { + _output->print_raw("Tracing active"); + } else { + _output->print("Failed to activate"); + } + } else if (::strcmp(_option.value(), "off") == 0) { + MallocTracer::disable(); + _output->print_raw("Tracing inactive"); + } else if (::strcmp(_option.value(), "print") == 0) { + bool all = false; + if (subopt != NULL) { + if (::strcmp(subopt, "all") == 0) { + all = true; + } else { + _output->print_cr("Invalid sub option"); + return; + } + } + MallocTracer::print(_output, all); + } else if (::strcmp(_option.value(), "reset") == 0) { + MallocTracer::reset(); + _output->print_raw("Tracing table reset"); + } else { + _output->print_cr("unknown sub command %s", _option.value()); + } + _output->cr(); +} +#else +void MallocTraceDCmd::execute(DCmdSource source, TRAPS) { +#ifdef __GLIBC__ + _output->print_cr("Glibc too new. Needs glibc version <= 2.31."); +#else + _output->print_cr("Not a glibc system."); +#endif +} +#endif // HAVE_GLIBC_MALLOC_HOOKS + +static const char* const usage_for_option = + "Valid Values:\n" + " - on [bt|nmt]\n" + " Switches trace on. Optional second parameter overrides the stack walk method.\n" + " - nmt (default): uses internal stackwalking.\n" + " - bt: uses glibc stackwalking (may give better results, but can be unstable).\n" + " - off\n" + " Switches trace off.\n" + " - print [all]\n" + " Print the capture table. By default only hot sites are printed; specifying \"all\" will print the full table.\n" + " - reset\n" + " Resets the capture table.\n"; + +MallocTraceDCmd::MallocTraceDCmd(outputStream* output, bool heap) : + DCmdWithParser(output, heap), + _option("option", usage_for_option, "STRING", true), + _suboption("suboption", "see option", "STRING", false) +{ + _dcmdparser.add_dcmd_argument(&_option); + _dcmdparser.add_dcmd_argument(&_suboption); +} + +} // namespace sap diff --git a/src/hotspot/os/linux/malloctrace/mallocTraceDCmd.hpp b/src/hotspot/os/linux/malloctrace/mallocTraceDCmd.hpp new file mode 100644 index 00000000000..5bb35ff34e9 --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/mallocTraceDCmd.hpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_MALLOCTRACE_MALLOCTRACEDCMD_HPP +#define OS_LINUX_MALLOCTRACE_MALLOCTRACEDCMD_HPP + +#include "services/diagnosticCommand.hpp" +#include "utilities/globalDefinitions.hpp" + +class outputStream; + +namespace sap { + +class MallocTraceDCmd : public DCmdWithParser { + DCmdArgument _option; + DCmdArgument _suboption; +public: + static int num_arguments() { return 2; } + MallocTraceDCmd(outputStream* output, bool heap); + static const char* name() { + return "System.malloctrace"; + } + static const char* description() { + return "Trace malloc call sites\n" + "Note: do *not* use in conjunction with MALLOC_CHECK_..!"; + } + static const char* impact() { + return "Low"; + } + static const JavaPermission permission() { + JavaPermission p = { "java.lang.management.ManagementPermission", "control", NULL }; + return p; + } + virtual void execute(DCmdSource source, TRAPS); +}; + +} // namespace sap + +#endif // OS_LINUX_MALLOCTRACE_MALLOCTRACEDCMD_HPP diff --git a/src/hotspot/os/linux/malloctrace/malloc_trace.md b/src/hotspot/os/linux/malloctrace/malloc_trace.md new file mode 100644 index 00000000000..eadc5336afd --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/malloc_trace.md @@ -0,0 +1,140 @@ +(c) SAP 2021 + +# The SapMachine MallocTrace facility + +## Preface + +When analyzing native OOMs resulting from C-heap exhaustion, we have two facilities built into the VM: + +- NMT (available in OpenJDK, SapMachine, and SAP JVM) +- the SAP JVM malloc statistics (as the name indicates, SAP JVM only) + +Both facilities have pros and cons, but they share one significant disadvantage: they require the malloc sites to be instrumented at compile time. Therefore they are unsuited for analyzing situations where outside code within the VM process allocates C-heap. + +In the past, if we were facing a suspected C-heap leak from code outside the VM (outside the coverage of NMT or SAP JVM malloc statistics), we were helpless. In these situations, one would typically use tools like perf or Valgrind, but that is seldom an option for us. + +### mtrace? + +A straightforward possibility to do this would be the Glibc-internal trace. Using `mtrace(3)`, one can force the Glibc to write a trace file for all malloc calls. Unfortunately, it is very costly. In my experiments, VM slowed down by factor 8-10. It also has to be enabled at the start of the VM (when the VM still happens to be single-threaded). That usually rules it out in production scenarios. + +## The SapMachine MallocTrace facility + +The new Linux-only malloc trace facility in the SapMachine uses Glibc malloc hooks to hook into the allocation process. In that way, it resembles the Glibc-internal `mtrace(3)`. But unlike `mtrace(3)`, it accumulates data in memory and provides a condensed report upon request, making it a lot faster. Moreover, it does not have to be started at VM startup (though it can), but one can switch it on when needed. + +### Usage via jcmd + +#### Switch trace on: + +``` +thomas@starfish$ jcmd AllocCHeap System.malloctrace on +268112: +Tracing activated +``` + +#### Switch trace off: +``` +thomas@starfish$ jcmd AllocCHeap System.malloctrace off +268112: +Tracing deactivated +``` + +#### Print a SapMachine MallocTrace report: + +Two options exist: +- a full report which can be lengthy but will show all call sites. +- (default) an abridged report which only shows the ten "hottest" call sites. + +``` +jcmd (VM) System.malloctrace print [all] +``` + +Example: + +``` +thomas@starfish$ jcmd AllocCHeap System.malloctrace print +268112: +---- 10 hottest malloc sites: ---- +---- 0 ---- +Invocs: 2813 (+0) +Alloc Size Range: 8 - 312 +[0x00007fd04159f3d0] sap::my_malloc_hook(unsigned long, void const*)+192 in libjvm.so +[0x00007fd040e0a004] AllocateHeap(unsigned long, MEMFLAGS, AllocFailStrategy::AllocFailEnum)+68 in libjvm.so +[0x00007fd041891eb2] SymbolTable::allocate_symbol(char const*, int, bool)+226 in libjvm.so +[0x00007fd041895c94] SymbolTable::do_add_if_needed(char const*, int, unsigned long, bool)+116 in libjvm.so +[0x00007fd04189669f] SymbolTable::new_symbols(ClassLoaderData*, constantPoolHandle const&, int, char const**, int*, int*, unsigned int*)+95 in libjvm.so +[0x00007fd040fc0042] ClassFileParser::parse_constant_pool_entries(ClassFileStream const*, ConstantPool*, int, JavaThread*)+3026 in libjvm.so +[0x00007fd040fc0282] ClassFileParser::parse_constant_pool(ClassFileStream const*, ConstantPool*, int, JavaThread*)+34 in libjvm.so +[0x00007fd040fc1c9a] ClassFileParser::ClassFileParser(ClassFileStream*, Symbol*, ClassLoaderData*, ClassLoadInfo const*, ClassFileParser::Publicity, JavaThread*)+938 in libjvm.so +[0x00007fd04149dd3e] KlassFactory::create_from_stream(ClassFileStream*, Symbol*, ClassLoaderData*, ClassLoadInfo const&, JavaThread*)+558 in libjvm.so +[0x00007fd0418a3310] SystemDictionary::resolve_class_from_stream(ClassFileStream*, Symbol*, Handle, ClassLoadInfo const&, JavaThread*)+496 in libjvm.so +[0x00007fd041357bce] jvm_define_class_common(char const*, _jobject*, signed char const*, int, _jobject*, char const*, JavaThread*) [clone .constprop.285]+510 in libjvm.so +[0x00007fd041357d06] JVM_DefineClassWithSource+134 in libjvm.so +[0x00007fd0402cf6d2] Java_java_lang_ClassLoader_defineClass1+450 in libjava.so +[0x00007fd0254b453a] 0x00007fd0254b453aBufferBlob (0x00007fd0254afb10) used for Interpreter +---- 1 ---- +Invocs: 2812 (+0) +Alloc Size: 16 +[0x00007fd04159f3d0] sap::my_malloc_hook(unsigned long, void const*)+192 in libjvm.so +[0x00007fd040e0a004] AllocateHeap(unsigned long, MEMFLAGS, AllocFailStrategy::AllocFailEnum)+68 in libjvm.so +[0x00007fd041895cd6] SymbolTable::do_add_if_needed(char const*, int, unsigned long, bool)+182 in libjvm.so +[0x00007fd04189669f] SymbolTable::new_symbols(ClassLoaderData*, constantPoolHandle const&, int, char const**, int*, int*, unsigned int*)+95 in libjvm.so +[0x00007fd040fc0042] ClassFileParser::parse_constant_pool_entries(ClassFileStream const*, ConstantPool*, int, JavaThread*)+3026 in libjvm.so +[0x00007fd040fc0282] ClassFileParser::parse_constant_pool(ClassFileStream const*, ConstantPool*, int, JavaThread*)+34 in libjvm.so +[0x00007fd040fc1c9a] ClassFileParser::ClassFileParser(ClassFileStream*, Symbol*, ClassLoaderData*, ClassLoadInfo const*, ClassFileParser::Publicity, JavaThread*)+938 in libjvm.so +[0x00007fd04149dd3e] KlassFactory::create_from_stream(ClassFileStream*, Symbol*, ClassLoaderData*, ClassLoadInfo const&, JavaThread*)+558 in libjvm.so +[0x00007fd0418a3310] SystemDictionary::resolve_class_from_stream(ClassFileStream*, Symbol*, Handle, ClassLoadInfo const&, JavaThread*)+496 in libjvm.so +[0x00007fd041357bce] jvm_define_class_common(char const*, _jobject*, signed char const*, int, _jobject*, char const*, JavaThread*) [clone .constprop.285]+510 in libjvm.so +[0x00007fd041357d06] JVM_DefineClassWithSource+134 in libjvm.so +[0x00007fd0402cf6d2] Java_java_lang_ClassLoader_defineClass1+450 in libjava.so +[0x00007fd0254b453a] 0x00007fd0254b453aBufferBlob (0x00007fd0254afb10) used for Interpreter +... + +... +Table size: 8171, num_entries: 3351, used slots: 519, longest chain: 5, invocs: 74515, lost: 0, collisions: 5844 +Malloc trace on. + (method: nmt-ish) + +74515 captures (0 without stack). +``` + +#### Reset the call site table: + +It is possible to reset the call site table. + +``` +jcmd (VM) System.malloctrace reset +``` + +This command will clear the table but not affect any running trace (if active) - the table will repopulate. + + +### Usage via command line + +One can switch on tracing at VM startup using the switch `-XX:+EnableMallocTrace`. A final report is printed upon VM shutdown to stdout via `-XX:+PrintMallocTraceAtExit`. + +Both options are diagnostic; one needs to unlock them with `-XX:+UnlockDiagnosticVMOptions` in release builds. + +Note: Starting the trace at VM startup is certainly possible but may not be the best option; the internal call site table will fill with many one-shot call sites that are only relevant during VM startup. A too-full call site table may slow down subsequent tracing. + +### Memory costs + +The internal data structures cost about ~5M. This memory is limited, and it will not grow - if we hit the limit, we won't register new call sites we encounter (but will continue to account for old call sites). + +Note that the call site table holds 32K call sites. That far exceeds the usual number of malloc call sites in the VM, so we should typically never hit this limit. + +### Performance costs + +In measurements, the MallocTrace increased VM startup time by about 6%. The slowdown highly depends on the frequency of malloc calls, though: a thread continuously doing malloc in a tight loop may slow down by factor 2-3. + +### Limitations + +This facility uses Glibc malloc hooks. + +Glibc malloc hooks are decidedly thread-unsafe, and we use them in a multithreaded context. Therefore what we can do with these hooks is very restricted. + +1) This is **not** a complete leak analysis tool! All we see with this facility is the "hotness" of malloc call sites. These may be innocuous; e.g., a malloc call site may be followed by a free call right away - it still would show up as a hot call site in the MallocTrace report. +2) **Not every allocation** will be captured since there are small-time windows where the hooks need to be disabled. + +One should use the MallocTrace tool to analyze suspected C-heap leaks when NMT/SAP JVM malloc statistics show up empty. It shows you which malloc sites are hot; nothing more. + +It works with third-party code, even with code that just happens to run in the VM process, e.g., system libraries. diff --git a/src/hotspot/os/linux/malloctrace/siteTable.cpp b/src/hotspot/os/linux/malloctrace/siteTable.cpp new file mode 100644 index 00000000000..28f600d22f1 --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/siteTable.cpp @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "code/codeBlob.hpp" +#include "code/codeCache.hpp" +#include "malloctrace/assertHandling.hpp" +#include "malloctrace/mallocTrace.hpp" +#include "malloctrace/siteTable.hpp" +#include "runtime/frame.inline.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +#include + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +// Since JDK-8289633 we forbid calling raw C-heap allocation functions using Kim's FORBID_C_FUNCTION. +// Callers need to explicitly opt in with ALLOW_C_FUNCTION. +// Since this code calls raw C-heap functions as a matter of course, instead of marking each call site +// with ALLOW_C_FUNCTION(..), I just mark them wholesale. +#if (__GNUC__ >= 10) +PRAGMA_DISABLE_GCC_WARNING("-Wattribute-warning") +#endif + +namespace sap { + +/////// Wrapper for the glibc backtrace(3) function; +// (we need to load it dynamically since it is not always guaranteed to be there.) + +class BackTraceWrapper { + typedef int (*backtrace_fun_t) (void **buffer, int size); + backtrace_fun_t _fun = NULL; + static backtrace_fun_t load_symbol() { + ::dlerror(); // clear state + void* sym = ::dlsym(RTLD_DEFAULT, "backtrace"); + if (sym != NULL && ::dlerror() == NULL) { + return (backtrace_fun_t)sym; + } + return NULL; + } +public: + BackTraceWrapper() : _fun(load_symbol()) {} + + // Capture a stack using backtrace(3); return true on success. + bool capture(Stack* stack) const { + if (_fun == NULL) { + return false; + } + return _fun((void**)stack->_frames, Stack::num_frames) > 0; + } +}; + +static BackTraceWrapper g_backtrace_wrapper; + +/////// NMT-like callstack function + +static bool capture_stack_nmt_like(Stack* stack) { + int frame_idx = 0; + int num_frames = 0; + frame fr = os::current_frame(); + while (fr.pc() && frame_idx < Stack::num_frames) { + stack->_frames[frame_idx ++] = fr.pc(); + num_frames++; + if (fr.fp() == NULL || fr.cb() != NULL || + fr.sender_pc() == NULL || os::is_first_C_frame(&fr)) break; + if (fr.sender_pc() && !os::is_first_C_frame(&fr)) { + fr = os::get_sender_for_C_frame(&fr); + } else { + break; + } + } + return num_frames > 0; +} + +void Stack::print_on(outputStream* st) const { + char tmp[256]; + for (int i = 0; i < num_frames && _frames[i] != NULL; i++) { + st->print("[" PTR_FORMAT "] ", p2i(_frames[i])); + if (os::print_function_and_library_name(st, _frames[i], tmp, sizeof(tmp), true, true, false)) { + st->cr(); + } else if (CodeCache::contains((void*)_frames[i])) { + CodeBlob* b = CodeCache::find_blob((void*)(void*)_frames[i]); + if (b != NULL) { + //b->dump_for_addr(_frames[i], st, false); + b->print_value_on(st); + } + } else { + st->cr(); + } + } +} + +// Capture stack; try both methods and use the result from the +// one getting the better results. +bool Stack::capture_stack(Stack* stack, bool use_backtrace) { + stack->reset(); + return use_backtrace ? g_backtrace_wrapper.capture(stack) : capture_stack_nmt_like(stack); +} + +#ifdef ASSERT +void SiteTable::verify() const { + unsigned num_sites_found = 0; + uint64_t num_invocations_found = 0; + for (unsigned slot = 0; slot < table_size; slot ++) { + for (Node* n = _table[slot]; n != NULL; n = n->next) { + num_sites_found ++; + num_invocations_found += n->site.invocations; + malloctrace_assert(slot_for_stack(&n->site.stack) == slot, "hash mismatch"); + malloctrace_assert(n->site.invocations > 0, "sanity"); + malloctrace_assert(n->site.invocations >= n->site.invocations_delta, "sanity"); + } + } + malloctrace_assert(num_sites_found <= _max_entries && num_sites_found == _size, + "mismatch (found: %u, max: %u, size: %u)", num_sites_found, _max_entries, _size); + malloctrace_assert(num_invocations_found + _lost == _invocations, + "mismatch (" UINT64_FORMAT " vs " UINT64_FORMAT, num_invocations_found, _invocations); + malloctrace_assert(num_sites_found <= max_entries(), "sanity"); +} +#endif // ASSERT + +SiteTable::SiteTable() { + reset(); +} + +void SiteTable::reset_deltas() { + for (unsigned slot = 0; slot < table_size; slot ++) { + for (Node* n = _table[slot]; n != NULL; n = n->next) { + n->site.invocations_delta = 0; + } + } +} + +void SiteTable::reset() { + _size = 0; + _invocations = _lost = _collisions = 0; + ::memset(_table, 0, sizeof(_table)); + _nodeheap.reset(); +}; + +SiteTable* SiteTable::create() { + void* p = ::malloc(sizeof(SiteTable)); + return new(p) SiteTable; +} + +void SiteTable::print_stats(outputStream* st) const { + unsigned longest_chain = 0; + unsigned used_slots = 0; + for (unsigned slot = 0; slot < table_size; slot ++) { + unsigned len = 0; + for (Node* n = _table[slot]; n != NULL; n = n->next) { + len ++; + } + longest_chain = MAX2(len, longest_chain); + if (len > 1) { + used_slots ++; + } + } + // Note: if you change this format, check gtest test_site_table parser. + st->print("Table size: %u, num_entries: %u, used slots: %u, longest chain: %u, invocs: " + UINT64_FORMAT ", lost: " UINT64_FORMAT ", collisions: " UINT64_FORMAT, + table_size, _size, used_slots, longest_chain, + _invocations, _lost, _collisions); +} + +// Sorting stuff for printing the table + +static int qsort_helper(const void* s1, const void* s2) { + return ((const Site*)s2)->invocations > ((const Site*)s1)->invocations ? 1 : -1; +} + +void SiteTable::print_table(outputStream* st, bool all) const { + + if (_size == 0) { + st->print_cr("Table is empty."); + } + + // We build up an index array of the filtered entries, then sort it by invocation counter. + unsigned num_entries = 0; + Site* const sorted_items = NEW_C_HEAP_ARRAY(Site, _size, mtInternal); + + for (unsigned slot = 0; slot < table_size; slot ++) { + for (Node* n = _table[slot]; n != NULL; n = n->next) { + if (n->site.invocations > 0) { + sorted_items[num_entries] = n->site; + num_entries ++; + } + } + } + malloctrace_assert(num_entries <= _size, "sanity"); + malloctrace_assert(num_entries <= max_entries(), "sanity"); + ::qsort(sorted_items, num_entries, sizeof(Site), qsort_helper); + + int rank = 0; + const unsigned max_show = all ? _size : MIN2(_size, (unsigned)10); + if (max_show < _size) { + st->print_cr("---- %d hottest malloc sites: ----", max_show); + } + for (unsigned i = 0; i < max_show; i ++) { + // For each call site, print out ranking, number of invocation, + // alloc size or alloc size range if non-uniform sizes, and stack. + st->print_cr("---- %d ----", i); + st->print_cr("Invocs: " UINT64_FORMAT " (+" UINT64_FORMAT ")", + sorted_items[i].invocations, sorted_items[i].invocations_delta); + if (sorted_items[i].max_alloc_size == sorted_items[i].min_alloc_size) { + st->print_cr("Alloc Size: " UINT32_FORMAT, sorted_items[i].max_alloc_size); + } else { + st->print_cr("Alloc Size Range: " UINT32_FORMAT " - " UINT32_FORMAT, + sorted_items[i].min_alloc_size, sorted_items[i].max_alloc_size); + } + sorted_items[i].stack.print_on(st); + } + if (max_show < _size) { + st->print_cr("---- %d entries omitted - use \"all\" to print full table.", + _size - max_show); + } + st->cr(); + FREE_C_HEAP_ARRAY(Site, sorted_items); +} + + +} // namespace sap + +#endif // HAVE_GLIBC_MALLOC_HOOKS diff --git a/src/hotspot/os/linux/malloctrace/siteTable.hpp b/src/hotspot/os/linux/malloctrace/siteTable.hpp new file mode 100644 index 00000000000..e6846d4ae6d --- /dev/null +++ b/src/hotspot/os/linux/malloctrace/siteTable.hpp @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_MALLOCTRACE_SITETABLE_HPP +#define OS_LINUX_MALLOCTRACE_SITETABLE_HPP + +#include "malloctrace/assertHandling.hpp" +#include "malloctrace/mallocTrace.hpp" +#include "utilities/globalDefinitions.hpp" + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +class outputStream; + +namespace sap { + +//////////////////////////////////////////////////// +// We currently support two ways to get a stack trace: +// - using backtrace(3) +// - using an NMT-like callstack walker +// I am not sure yet which is better. Have to experiment. + +enum class capture_method_t { + nmt_like = 0, using_backtrace = 1 +}; + +///// Stack //////////////////// +// simple structure holding a fixed-sized native stack + +struct Stack { + static const int num_frames = 16; + address _frames[num_frames]; + + unsigned calculate_hash() const { + uintptr_t hash = 0; + for (int i = 0; i < num_frames; i++) { + hash += (uintptr_t)_frames[i]; + } + return hash; + } + + void reset() { + ::memset(_frames, 0, sizeof(_frames)); + } + + void copy_to(Stack* other) const { + ::memcpy(other->_frames, _frames, sizeof(_frames)); + } + + bool equals(const Stack* other) const { + return ::memcmp(_frames, other->_frames, sizeof(_frames)) == 0; + } + + void print_on(outputStream* st) const; + + static bool capture_stack(Stack* stack, bool use_backtrace); + +}; + +///// Site //////////////////// +// Stack + invocation counters + +struct Site { + Stack stack; + uint64_t invocations; + uint64_t invocations_delta; // delta since last printing + uint32_t min_alloc_size; // min and max allocation size + uint32_t max_alloc_size; // from that call site + // (note: can be zero: we also trace zero-sized allocs since malloc(0) + // could also be a leak) +}; + +///// SiteTable //////////////////// +// A hashmap containing all captured malloc call sites. +// This map is kept very simple. We never remove entries, just +// reset the table as a whole. Space for the nodes is pre-allocated when +// the table is created to prevent malloc calls disturbing the statistics +// run. +class SiteTable { + + static const int _max_entries = 32 * K; + + struct Node { + Node* next; + Site site; + }; + + // We preallocate all nodes in this table to avoid + // swamping the VM with internal malloc calls while the + // trace is running. + class NodeHeap { + Node _nodes[SiteTable::_max_entries]; + int _used; + public: + NodeHeap() : _used(0) { + ::memset(_nodes, 0, sizeof(_nodes)); + } + Node* get_node() { + Node* n = NULL; + if (_used < SiteTable::_max_entries) { + n = _nodes + _used; + _used ++; + } + return n; + } + void reset() { + ::memset(_nodes, 0, sizeof(_nodes)); + _used = 0; + } + }; + + NodeHeap _nodeheap; + const static int table_size = 8171; //prime + Node* _table[table_size]; + + unsigned _size; // Number of entries + uint64_t _invocations; // invocations (including lost) + uint64_t _lost; // lost adds due to table full + uint64_t _collisions; // hash collisions + + static unsigned slot_for_stack(const Stack* stack) { + unsigned hash = stack->calculate_hash(); + malloctrace_assert(hash != 0, "sanity"); + return hash % table_size; + } + +public: + + SiteTable(); + + void add_site(const Stack* stack, uint32_t alloc_size) { + _invocations ++; + + const unsigned slot = slot_for_stack(stack); + + // Find entry + for (Node* p = _table[slot]; p != NULL; p = p->next) { + if (p->site.stack.equals(stack)) { + // Call site already presented in table + p->site.invocations ++; + p->site.invocations_delta ++; + p->site.max_alloc_size = MAX2(p->site.max_alloc_size, alloc_size); + p->site.min_alloc_size = MIN2(p->site.min_alloc_size, alloc_size); + return; + } else { + _collisions ++; + } + } + + Node* n = _nodeheap.get_node(); + if (n == NULL) { // hashtable too full, reject. + assert(_size == max_entries(), "sanity"); + _lost ++; + return; + } + n->site.invocations = n->site.invocations_delta = 1; + n->site.max_alloc_size = n->site.min_alloc_size = alloc_size; + stack->copy_to(&(n->site.stack)); + n->next = _table[slot]; + _table[slot] = n; + _size ++; + } + + void print_table(outputStream* st, bool raw) const; + void print_stats(outputStream* st) const; + void reset_deltas(); + void reset(); + DEBUG_ONLY(void verify() const;) + + // create a table from c-heap + static SiteTable* create(); + + // Maximum number of entries the table can hold. + static unsigned max_entries() { return _max_entries; } + + // Number of entries currently in the table. + unsigned size() const { return _size; } + + // Number of invocations. + uint64_t invocations() const { return _invocations; } + + // Number of invocations lost because table was full. + uint64_t lost() const { return _lost; } + +}; + +} // namespace sap + +#endif // HAVE_GLIBC_MALLOC_HOOKS + +#endif // OS_LINUX_MALLOCTRACE_SITETABLE_HPP diff --git a/src/hotspot/os/linux/osContainer_linux.cpp b/src/hotspot/os/linux/osContainer_linux.cpp index fdb138c864f..d74e0b247db 100644 --- a/src/hotspot/os/linux/osContainer_linux.cpp +++ b/src/hotspot/os/linux/osContainer_linux.cpp @@ -37,6 +37,24 @@ bool OSContainer::_is_initialized = false; bool OSContainer::_is_containerized = false; CgroupSubsystem* cgroup_subsystem; +// SapMachine 2022-05-01: Vitals +// This is an ugly hack aimed at having as little merge surface as possible across JDK versions. +// We use the OsContainer class just for its ability to figure out the controller path; we expose +// that and read the data ourselves (since OsContainer omits certain data and does things I don't +// want to happen in the Vitals sampler thread). +extern const char* sapmachine_get_memory_controller_path() { + if (cgroup_subsystem != NULL) { + CachingCgroupController* cc = cgroup_subsystem->memory_controller(); + if (cc != NULL) { + CgroupController* c = cc->controller(); + if (c != NULL) { + return c->subsystem_path(); + } + } + } + return NULL; +} + /* init * * Initialize the container support and determine if diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp index 52866a44b26..0f0d763ddc6 100644 --- a/src/hotspot/os/linux/os_linux.cpp +++ b/src/hotspot/os/linux/os_linux.cpp @@ -84,6 +84,9 @@ #include "jfr/support/jfrNativeLibraryLoadEvent.hpp" #endif +// SapMachine 2021-09-01: malloc-trace +#include "malloctrace/mallocTrace.hpp" + // put OS-includes here # include # include @@ -2149,6 +2152,9 @@ void os::print_os_info(outputStream* st) { VM_Version::print_platform_virtualization_info(st); + // SapMachine 2019-07-02: 8225345: Provide Cloud IAAS related info on Linux in the hs_err file + os::Linux::print_cloud_info(st); + os::Linux::print_steal_info(st); } @@ -2509,6 +2515,87 @@ bool os::Linux::print_container_info(outputStream* st) { return true; } +// SapMachine 2019-07-02: 8225345: Provide Cloud IAAS related info on Linux in the hs_err file +static int check_matching_lines_from_file(const char* filename, const char* keywords_to_match[]) { + char line[500]; + FILE* fp = fopen(filename, "r"); + if (fp == NULL) { + return -1; + } + + while (fgets(line, sizeof(line), fp) != NULL) { + int i = 0; + while (keywords_to_match[i] != NULL) { + if (strstr(line, keywords_to_match[i]) != NULL) { + fclose(fp); + return i; + } + i++; + } + } + fclose(fp); + return -1; +} + +// SapMachine 2019-07-02: 8225345: Provide Cloud IAAS related info on Linux in the hs_err file +// Add Cloud information where possible, a basic detection can be done by using dmi info +// Google GCP: /sys/class/dmi/id/product_name contains 'Google Compute Engine' (or just 'Google') +// Alibaba : /sys/class/dmi/id/product_name contains 'Alibaba Cloud ECS' +// OpenStack : /sys/class/dmi/id/product_name contains 'OpenStack' e.g. 'OpenStack Nova' +// Azure : /sys/class/dmi/id/chassis_asset_tag contains '7783-7084-3265-9085-8269-3286-77' (means ASCII-encoded: 'MS AZURE VM') +// AWS KVM/Baremetal : /sys/class/dmi/id/chassis_asset_tag contains 'Amazon EC2' +// AWS Xen : /sys/class/dmi/id/bios_version and /sys/class/dmi/id/product_version contain amazon (plus some more info) +// /sys/class/dmi/id/bios_vendor and /sys/class/dmi/id/chassis_vendor contain 'Xen' +void os::Linux::print_cloud_info(outputStream* st) { + // dmidir is /sys/class/dmi/id + const char* filename = "/sys/class/dmi/id/product_name"; + const char* kwcld[] = { "Google", "Google Compute Engine", "Alibaba Cloud", "OpenStack", NULL }; + int res = check_matching_lines_from_file(filename, kwcld); + if (res != -1) { // a matching Cloud identifier has been found + st->print("Cloud infrastructure detected:"); + if (res == 0 || res == 1) { + st->print_cr("Google cloud"); + } + if (res == 2) { + st->print_cr("Alibaba cloud"); + } + if (res == 3) { + st->print_cr("OpenStack based cloud"); + // output version info too, e.g. "16.1.6-16.1.6~dev5" + _print_ascii_file("/sys/class/dmi/id/product_version", st); + } + return; + } + // AWS KVM/Baremetal + const char* filename2 = "/sys/class/dmi/id/chassis_asset_tag"; + const char* kwaws[] = { "Amazon EC2", "7783-7084-3265-9085-8269-3286-77", NULL }; + res = check_matching_lines_from_file(filename2, kwaws); + if (res != -1) { + st->print("Cloud infrastructure detected:"); + if (res == 0) { + st->print_cr("Amazon EC2 cloud"); + } + if (res == 1) { + st->print_cr("Microsoft Azure"); + } + return; + } + // AWS Xen is a bit tricky, it might not contain a "nice" product name + const char* chassis_vendor_file = "/sys/class/dmi/id/chassis_vendor"; + const char* bios_vendor_file = "/sys/class/dmi/id/bios_vendor"; + const char* kwxen[] = { "Xen", NULL }; + int res1 = check_matching_lines_from_file(chassis_vendor_file, kwxen); + int res2 = check_matching_lines_from_file(bios_vendor_file, kwxen); + if (res1 != -1 || res2 != -1) { + const char* pvfile = "/sys/class/dmi/id/product_version"; + const char* kwam[] = { "amazon", NULL }; + res = check_matching_lines_from_file(pvfile, kwam); + if (res != -1) { + st->print_cr("Cloud infrastructure detected: Amazon Xen-based cloud"); + } + } +} + void os::Linux::print_steal_info(outputStream* st) { if (has_initial_tick_info) { CPUPerfTicks pticks; @@ -4807,6 +4894,21 @@ jint os::init_2(void) { } } +#ifdef HAVE_GLIBC_MALLOC_HOOKS + // SapMachine 2021-09-01: malloc-trace + if (EnableMallocTrace) { + sap::MallocTracer::enable(); + } +#else + if (!FLAG_IS_DEFAULT(EnableMallocTrace)) { +#ifdef __GLIBC__ + warning("EnableMallocTrace ignored (Glibc too new. Needs glibc version <= 2.31.)"); +#else + warning("Not a glibc system. EnableMallocTrace ignored."); +#endif + } +#endif // __GLIBC__ + return JNI_OK; } diff --git a/src/hotspot/os/linux/os_linux.hpp b/src/hotspot/os/linux/os_linux.hpp index 6b902e82802..9770b371b47 100644 --- a/src/hotspot/os/linux/os_linux.hpp +++ b/src/hotspot/os/linux/os_linux.hpp @@ -76,6 +76,8 @@ class os::Linux { static void print_process_memory_info(outputStream* st); static void print_system_memory_info(outputStream* st); + // SapMachine 2019-07-02: 8225345: Provide Cloud IAAS related info on Linux in the hs_err file + static void print_cloud_info(outputStream* st); static bool print_container_info(outputStream* st); static void print_steal_info(outputStream* st); static void print_distro_info(outputStream* st); diff --git a/src/hotspot/os/linux/vitals_linux.cpp b/src/hotspot/os/linux/vitals_linux.cpp new file mode 100644 index 00000000000..f616978cf62 --- /dev/null +++ b/src/hotspot/os/linux/vitals_linux.cpp @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2019, 2022 SAP SE. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "jvm_io.h" +#include "osContainer_linux.hpp" +#include "vitals_linux_oswrapper.hpp" +#include "logging/log.hpp" +#include "runtime/globals.hpp" +#include "runtime/os.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" +#include "vitals/vitals_internals.hpp" + +namespace sapmachine_vitals { + +/////// Columns //////// + +// A special class to display cpu time +class CPUTimeColumn: public Column { + + long _clk_tck; + int _num_cores; + + int do_print0(outputStream* st, value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const { + // CPU values may overflow, so the delta may be negative. + if (last_value > value) { + return 0; + } + int l = 0; + if (last_value != INVALID_VALUE) { + + // If the last sample is less than one second old, we omit calculating the cpu + // usage. + if (last_value_age > 0) { + + // Values are in ticks. Convert to ms. + const uint64_t value_ms = (value * 1000) / _clk_tck; + const uint64_t last_value_ms = (last_value * 1000) / _clk_tck; + const uint64_t delta_ms = value_ms - last_value_ms; + + // Calculate the number of wallclock milliseconds for the delta interval... + const uint64_t age_ms = last_value_age * 1000; + + // times number of available cores. + const uint64_t total_cpu_time_ms = age_ms * _num_cores; + + // Put the spent cpu time in reference to the total available cpu time. + const double percentage = (100.0f * delta_ms) / total_cpu_time_ms; + + char buf[32]; + l = jio_snprintf(buf, sizeof(buf), "%.0f", percentage); + if (st != NULL) { + st->print_raw(buf); + } + } + } + return l; + } + +public: + CPUTimeColumn(const char* category, const char* header, const char* name, const char* description, Extremum extremum) + : Column(category, header, name, description, extremum) + { + _clk_tck = ::sysconf(_SC_CLK_TCK); + _num_cores = os::active_processor_count(); + } + +}; + +static bool g_show_system_memavail = false; +static Column* g_col_system_memavail = NULL; +static Column* g_col_system_memcommitted = NULL; +static Column* g_col_system_memcommitted_ratio = NULL; +static Column* g_col_system_swap = NULL; + +static Column* g_col_system_pages_swapped_in = NULL; +static Column* g_col_system_pages_swapped_out = NULL; + +static Column* g_col_system_num_procs = NULL; +static Column* g_col_system_num_threads = NULL; + +static Column* g_col_system_num_procs_running = NULL; +static Column* g_col_system_num_procs_blocked = NULL; + +static bool g_show_cgroup_info = false; +static Column* g_col_system_cgrp_limit_in_bytes = NULL; +static Column* g_col_system_cgrp_soft_limit_in_bytes = NULL; +static Column* g_col_system_cgrp_usage_in_bytes = NULL; +static Column* g_col_system_cgrp_kmem_usage_in_bytes = NULL; + +static Column* g_col_system_cpu_user = NULL; +static Column* g_col_system_cpu_system = NULL; +static Column* g_col_system_cpu_idle = NULL; +static Column* g_col_system_cpu_steal = NULL; +static Column* g_col_system_cpu_guest = NULL; + +static Column* g_col_process_virt = NULL; + +static bool g_show_rss_detail_info = false; +static Column* g_col_process_rss = NULL; +static Column* g_col_process_rssanon = NULL; +static Column* g_col_process_rssfile = NULL; +static Column* g_col_process_rssshmem = NULL; + +static Column* g_col_process_swapped_out = NULL; + +static Column* g_col_process_chp_used = NULL; +static Column* g_col_process_chp_free = NULL; + +static Column* g_col_process_cpu_user = NULL; +static Column* g_col_process_cpu_system = NULL; + +static Column* g_col_process_num_of = NULL; +static Column* g_col_process_io_bytes_read = NULL; +static Column* g_col_process_io_bytes_written = NULL; + +static Column* g_col_process_num_threads = NULL; + +bool platform_columns_initialize() { + + const char* const system_cat = "system"; + const char* const process_cat = "process"; + + Legend::the_legend()->add_footnote(" [host]: values are host-global (not containerized)."); + Legend::the_legend()->add_footnote(" [cgrp]: if containerized or running in systemd slice"); + Legend::the_legend()->add_footnote(" [krn]: depends on kernel version"); + Legend::the_legend()->add_footnote(" [glibc]: only shown for glibc-based distros"); + + // Update values once, to get up-to-date readings. Some of those we need to decide whether to show or hide certain columns + OSWrapper::initialize(); + OSWrapper::update_if_needed(); + + // syst-avail depends on kernel version. + g_show_system_memavail = OSWrapper::syst_avail() != INVALID_VALUE; + g_col_system_memavail = + define_column(system_cat, NULL, "avail", "Memory available without swapping [host] [krn]", g_show_system_memavail, MIN); + g_col_system_memcommitted = + define_column(system_cat, NULL, "comm", "Committed memory [host]", true); + g_col_system_memcommitted_ratio = + define_column(system_cat, NULL, "crt", "Committed-to-Commit-Limit ratio (percent) [host]", true); + g_col_system_swap = + define_column(system_cat, NULL, "swap", "Swap space used [host]", true); + + g_col_system_pages_swapped_in = + define_column(system_cat, NULL, "si", "Number of pages swapped in [host] [delta]", true); + g_col_system_pages_swapped_out = + define_column(system_cat, NULL, "so", "Number of pages pages swapped out [host] [delta]", true); + + g_col_system_num_procs = + define_column(system_cat, NULL, "p", "Number of processes", true); + g_col_system_num_threads = + define_column(system_cat, NULL, "t", "Number of threads", true); + + g_col_system_num_procs_running = + define_column(system_cat, NULL, "tr", "Number of threads running", true); + g_col_system_num_procs_blocked = + define_column(system_cat, NULL, "tb", "Number of threads blocked on disk IO", true); + + g_col_system_cpu_user = + define_column(system_cat, "cpu", "us", "CPU user time [host]", true); + g_col_system_cpu_system = + define_column(system_cat, "cpu", "sy", "CPU system time [host]", true); + g_col_system_cpu_idle = + define_column(system_cat, "cpu", "id", "CPU idle time [host]", true); + g_col_system_cpu_steal = + define_column(system_cat, "cpu", "st", "CPU time stolen [host]", true); + g_col_system_cpu_guest = + define_column(system_cat, "cpu", "gu", "CPU time spent on guest [host]", true); + + // I show cgroup information if the container layer thinks we are containerized OR we have limits established + // (which should come out as the same, but you never know + g_show_cgroup_info = OSContainer::is_containerized() || (OSWrapper::syst_cgro_lim() != INVALID_VALUE || OSWrapper::syst_cgro_limsw() != INVALID_VALUE); + g_col_system_cgrp_limit_in_bytes = + define_column(system_cat, "cgroup", "lim", "cgroup memory limit [cgrp]", g_show_cgroup_info, MIN); + g_col_system_cgrp_soft_limit_in_bytes = + define_column(system_cat, "cgroup", "slim", "cgroup memory soft limit [cgrp]", g_show_cgroup_info, MIN); + g_col_system_cgrp_usage_in_bytes = + define_column(system_cat, "cgroup", "usg", "cgroup memory usage [cgrp]", g_show_cgroup_info); + g_col_system_cgrp_kmem_usage_in_bytes = + define_column(system_cat, "cgroup", "kusg", "cgroup kernel memory usage (cgroup v1 only) [cgrp]", g_show_cgroup_info); + + // Process + + g_col_process_virt = + define_column(process_cat, NULL, "virt", "Virtual size", true); + + // RSS detail needs kernel >= 4.5 + g_show_rss_detail_info = OSWrapper::proc_rss_anon() != INVALID_VALUE; + g_col_process_rss = + define_column(process_cat, "rss", "all", "Resident set size, total", true); + g_col_process_rssanon = + define_column(process_cat, "rss", "anon", "Resident set size, anonymous memory [krn]", g_show_rss_detail_info); + g_col_process_rssfile = + define_column(process_cat, "rss", "file", "Resident set size, file mappings [krn]", g_show_rss_detail_info); + g_col_process_rssshmem = + define_column(process_cat, "rss", "shm", "Resident set size, shared memory [krn]", g_show_rss_detail_info); + + g_col_process_swapped_out = + define_column(process_cat, NULL, "swdo", "Memory swapped out", true); + + // glibc heap info depends on, obviously, glibc. +#ifdef __GLIBC__ + const bool show_glibc_heap_info = true; +#else + const bool show_glibc_heap_info = false; +#endif + g_col_process_chp_used = + define_column(process_cat, "cheap", "usd", "C-Heap, in-use allocations (may be unavailable if RSS > 4G) [glibc]", show_glibc_heap_info); + g_col_process_chp_free = + define_column(process_cat, "cheap", "free", "C-Heap, bytes in free blocks (may be unavailable if RSS > 4G) [glibc]", show_glibc_heap_info); + + g_col_process_cpu_user = + define_column(process_cat, "cpu", "us", "Process cpu user time", true); + + g_col_process_cpu_system = + define_column(process_cat, "cpu", "sy", "Process cpu system time", true); + + g_col_process_num_of = + define_column(process_cat, "io", "of", "Number of open files", true); + + g_col_process_io_bytes_read = + define_column(process_cat, "io", "rd", "IO bytes read from storage or cache", true); + + g_col_process_io_bytes_written = + define_column(process_cat, "io", "wr", "IO bytes written", true); + + g_col_process_num_threads = + define_column(process_cat, NULL, "thr", "Number of native threads", true); + + return true; +} + +static void set_value_in_sample(Column* col, Sample* sample, value_t val) { + if (col != NULL) { + int index = col->index(); + sample->set_value(index, val); + } +} + +void sample_platform_values(Sample* sample) { + + int idx = 0; + + OSWrapper::update_if_needed(); + + if (g_show_system_memavail) { + set_value_in_sample(g_col_system_memavail, sample, OSWrapper::syst_avail()); + } + set_value_in_sample(g_col_system_swap, sample, OSWrapper::syst_swap()); + + set_value_in_sample(g_col_system_memcommitted, sample, OSWrapper::syst_comm()); + set_value_in_sample(g_col_system_memcommitted_ratio, sample, OSWrapper::syst_crt()); + + set_value_in_sample(g_col_system_pages_swapped_in, sample, OSWrapper::syst_si()); + set_value_in_sample(g_col_system_pages_swapped_out, sample, OSWrapper::syst_so()); + + set_value_in_sample(g_col_system_cpu_user, sample, OSWrapper::syst_cpu_us()); + set_value_in_sample(g_col_system_cpu_system, sample, OSWrapper::syst_cpu_sy()); + set_value_in_sample(g_col_system_cpu_idle, sample, OSWrapper::syst_cpu_id()); + set_value_in_sample(g_col_system_cpu_steal, sample, OSWrapper::syst_cpu_st()); + set_value_in_sample(g_col_system_cpu_guest, sample, OSWrapper::syst_cpu_gu()); + + set_value_in_sample(g_col_system_num_procs_running, sample, OSWrapper::syst_tr()); + set_value_in_sample(g_col_system_num_procs_blocked, sample, OSWrapper::syst_tb()); + + // cgroups business + if (g_show_cgroup_info) { + set_value_in_sample(g_col_system_cgrp_usage_in_bytes, sample, OSWrapper::syst_cgro_usg()); + // set_value_in_sample(g_col_system_cgrp_memsw_usage_in_bytes, sample, OSWrapper::syst_cgro_usgsw()); + set_value_in_sample(g_col_system_cgrp_kmem_usage_in_bytes, sample, OSWrapper::syst_cgro_kusg()); + set_value_in_sample(g_col_system_cgrp_limit_in_bytes, sample, OSWrapper::syst_cgro_lim()); + set_value_in_sample(g_col_system_cgrp_soft_limit_in_bytes, sample, OSWrapper::syst_cgro_slim()); + // set_value_in_sample(g_col_system_cgrp_memsw_limit_in_bytes, sample, OSWrapper::syst_cgro_limsw()); + } + + set_value_in_sample(g_col_system_num_procs, sample, OSWrapper::syst_p()); + set_value_in_sample(g_col_system_num_threads, sample, OSWrapper::syst_t()); + + set_value_in_sample(g_col_process_virt, sample, OSWrapper::proc_virt()); + set_value_in_sample(g_col_process_swapped_out, sample, OSWrapper::proc_swdo()); + set_value_in_sample(g_col_process_rss, sample, OSWrapper::proc_rss_all()); + + if (g_show_rss_detail_info) { + set_value_in_sample(g_col_process_rssanon, sample, OSWrapper::proc_rss_anon()); + set_value_in_sample(g_col_process_rssfile, sample, OSWrapper::proc_rss_file()); + set_value_in_sample(g_col_process_rssshmem, sample, OSWrapper::proc_rss_shm()); + } + + set_value_in_sample(g_col_process_num_threads, sample, OSWrapper::proc_thr()); + set_value_in_sample(g_col_process_num_of, sample, OSWrapper::proc_io_of()); + + set_value_in_sample(g_col_process_io_bytes_read, sample, OSWrapper::proc_io_rd()); + set_value_in_sample(g_col_process_io_bytes_written, sample, OSWrapper::proc_io_wr()); + + set_value_in_sample(g_col_process_cpu_user, sample, OSWrapper::proc_cpu_us()); + set_value_in_sample(g_col_process_cpu_system, sample, OSWrapper::proc_cpu_sy()); + +#ifdef __GLIBC__ + set_value_in_sample(g_col_process_chp_used, sample, OSWrapper::proc_chea_usd()); + set_value_in_sample(g_col_process_chp_free, sample, OSWrapper::proc_chea_free()); +#endif // __GLIBC__ + +} // end: sample_platform_values + +} // namespace sapmachine_vitals diff --git a/src/hotspot/os/linux/vitals_linux_himemreport.cpp b/src/hotspot/os/linux/vitals_linux_himemreport.cpp new file mode 100644 index 00000000000..cd8ff25b085 --- /dev/null +++ b/src/hotspot/os/linux/vitals_linux_himemreport.cpp @@ -0,0 +1,941 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "jvm_io.h" +#include "vitals_linux_himemreport.hpp" +#include "vitals_linux_oswrapper.hpp" +#include "logging/log.hpp" +#include "memory/allStatic.hpp" +#include "nmt/memBaseline.hpp" +#include "nmt/memReporter.hpp" +#include "nmt/memTracker.hpp" +#include "runtime/arguments.hpp" +#include "runtime/globals.hpp" +#include "runtime/globals_extension.hpp" +#include "runtime/os.hpp" +#include "runtime/vmOperations.hpp" +#include "runtime/vmThread.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" +#include "utilities/ostream.hpp" +#include "vitals/vitals_internals.hpp" + +#include +#include +#include +#include + +// Newer JDKS: NMT is always on and this macro does not exist +// Older JDKs: NMT can be off at compile time; in that case INCLUDE_NMT +// will be defined=0 via CFLAGS; or on, in that case it will be defined=1 in macros.hpp. +#ifndef INCLUDE_NMT +#define INCLUDE_NMT 1 +#endif + +// Logging and output: +// We log during initialization phase to UL using the "vitals" tag. +// In the high memory detection thread itself, when triggering the report, we write strictly to +// stderr, directly. We don't use tty since we want to bypass ttylock. Sub command output also +// gets written to stderr. + +// We print to the stderr stream directly in this code (since we want to bypass ttylock) +static fdStream stderr_stream(2); + +namespace sapmachine_vitals { + +static const int HiMemReportDecaySeconds = 60 * 5; + +//////////// pretty printing stuff ////////////////////////////////// + +#define STRFTIME_FROM_TIME_T(st, fmt, t) \ + char buf[32]; \ + struct tm timeinfo; \ + if (::localtime_r(&t, &timeinfo) != NULL && \ + ::strftime(buf, sizeof(buf), fmt, &timeinfo) != 0) { \ + st->print_raw(buf); \ + } else { \ + st->print_raw("unknown_date"); \ + } + + +static void print_date_and_time(outputStream *st, time_t t) { + STRFTIME_FROM_TIME_T(st, "%F %T", t); +} + +// For use in file names +static void print_date_and_time_underscored(outputStream *st, time_t t) { + STRFTIME_FROM_TIME_T(st, "%Y_%m_%d_%H_%M_%S", t); +} + +static void print_current_date_and_time(outputStream *st) { + time_t t; + time(&t); + print_date_and_time(st, t); +} + +//////////// Alert state //////////////////////////////////////////// + +class AlertState : public CHeapObj { + + // this is 100% + const size_t _maximum; + + // Alert percentages per level + static const int _alvl_perc[5]; + + // alert level: 0: all is well, 1..6: we are at x percent + int _alvl; + + // time when alert level was increased last (for decay) + time_t _last_avlv_increase; + + // We count spikes. A spike is a single increase to at least the lowest + // alert level, followed by a reset because we recovered. + int _spike_no; + + int calc_percentage(size_t size) const { + return (int)((100.0f * (double)size)/(double)_maximum); + } + + int calc_alvl(int percentage) const { + int i = 0; + while ((_alvl_perc[i + 1] != -1) && (_alvl_perc[i + 1] <= percentage)) { + i ++; + } + return i; + } + +public: + + AlertState(size_t maximum) : + _maximum(maximum), _alvl(0), _last_avlv_increase(0), _spike_no(0) { + assert(_maximum > 0, "sanity"); + } + + size_t maximum() const { + return _maximum; + } + + int current_spike_no() const { + return _spike_no; + } + + int current_alert_level() const { + return _alvl; + } + + static int alert_level_percentage(int alvl) { + assert(alvl >= 0 && alvl < (int)(sizeof(_alvl_perc) / sizeof(int)), "oob"); + return _alvl_perc[alvl]; + } + + int current_alert_level_percentage() const { + return alert_level_percentage(_alvl); + } + + // Update the state. + // If we changed the alert level (either increased it or reset it after decay), + // return true. + bool update(size_t current_size) { + const int percentage = calc_percentage(current_size); + const int new_alvl = calc_alvl(percentage); + // If we reached a new alert level, hold information and inform caller. + if (new_alvl > _alvl) { + // If we increased from zero, it means we entered a new spike, so + // increase spike number + if (_alvl == 0) { + _spike_no ++; + } + _alvl = new_alvl; + ::time(&_last_avlv_increase); + return true; + } + // If all is well now, but we had an alert situation before, and enough + // time has passed, reset alert level + if (new_alvl == 0 && _alvl > 0) { + time_t t; + time(&t); + if ((t - _last_avlv_increase) >= HiMemReportDecaySeconds) { + _alvl = 0; + _last_avlv_increase = 0; + return true; + } + } + return false; + } + +}; + +const int AlertState::_alvl_perc[5] = { 0, 66, 75, 90, -1 }; + +static AlertState* g_alert_state = NULL; + +// What do we test? +enum class compare_type { + compare_rss_vs_phys = 0, // We compare rss+swap vs total physical memory + compare_rss_vs_cgroup_limit = 1, // We compare rss+swap vs the cgroup limit + compare_rss_vs_manual_limit = 2, // HiMemReportMaximum is set, we compare rss+swap with that limit + compare_none +}; + +static compare_type g_compare_what = compare_type::compare_none; + +static const char* describe_maximum_by_compare_type(compare_type t) { + const char* s = ""; + switch (g_compare_what) { + case compare_type::compare_rss_vs_cgroup_limit: s = "cgroup memory limit"; break; + case compare_type::compare_rss_vs_phys: s = "the half of total physical memory"; break; + case compare_type::compare_rss_vs_manual_limit: s = "HiMemReportMaximum"; break; + default: ShouldNotReachHere(); + } + return s; +} + +//////////// NMT stuff ////////////////////////////////////////////// + +// NMT is nice, but the interface is unnecessary convoluted. For now, to keep merge surface small, +// we work with what we have + +#if INCLUDE_NMT +class NMTStuff : public AllStatic { + + static MemBaseline _baseline; + static time_t _baseline_time; + + // Fill a given baseline + static void fill_baseline(MemBaseline& baseline) { + const NMT_TrackingLevel lvl = MemTracker::tracking_level(); + if (lvl >= NMT_summary) { + const bool summary_only = (lvl == NMT_summary); + baseline.baseline(summary_only); + } + } + +public: + + static bool is_enabled() { + const NMT_TrackingLevel lvl = MemTracker::tracking_level(); + // Note: I avoid assumptions about numerical values of NMT_TrackingLevel + // (e.g. "lvl >= NMT_summary") since their order changed over time and we + // want to be JDK-version-agnostic here. + return lvl == NMT_summary || lvl == NMT_detail; + } + + // Capture a baseline right now + static void capture_baseline() { + fill_baseline(_baseline); + time(&_baseline_time); + } + + // Do the best possible report with the given NMT tracking level. + // If we are at summary level, do a summary level report + // If we are at detail level, do a detail level report + // If we have a baseline captured, do a diff level report + static void report_as_best_as_possible(outputStream* st) { + + if (NMTStuff::is_enabled()) { + + // Get the state now + MemBaseline baseline_now; + fill_baseline(baseline_now); + + // prepare and print suitable report + if (_baseline.baseline_type() == baseline_now.baseline_type()) { + // We already captured a baseline, and its type fits us (nobody changed NMT levels inbetween calls) + time_t t; + time(&t); + st->print("(diff against baseline taken at "); + print_date_and_time(st, _baseline_time); + st->print_cr(", %d seconds ago)", (int)(t - _baseline_time)); + st->cr(); + const bool summary_only = (baseline_now.baseline_type() == MemBaseline::Summary_baselined); + if (summary_only) { + MemSummaryDiffReporter rpt(_baseline, baseline_now, st, K); + rpt.report_diff(); + } else { + MemDetailDiffReporter rpt(_baseline, baseline_now, st, K); + rpt.report_diff(); + } + } else { + // We don't have a baseline yet. Just report the raw numbers + const bool summary_only = (baseline_now.baseline_type() == MemBaseline::Summary_baselined); + if (summary_only) { + MemSummaryReporter rpt(baseline_now, st, K); + rpt.report(); + } else { + MemDetailReporter rpt(baseline_now, st, K); + rpt.report(); + } + } + } else { + st->print_cr("NMT is disabled, nothing to print"); + } + + } + + // If the situation calmed down, reset (clear the base line) + static void reset() { + _baseline_time = 0; + _baseline.reset(); + } + +}; + +MemBaseline NMTStuff::_baseline; +time_t NMTStuff::_baseline_time = 0; +#endif // INCLUDE_NMT + +//////////// Reporting ////////////////////////////////////////////// + +class ReportDir : public CHeapObj { + // absolute, always ends with slash + stringStream _dir; + +public: + + const char* path() const { return _dir.base(); } + + bool initialize(const char* d) { + + assert(d != NULL && strlen(d) > 0, "sanity"); + assert(_dir.size() == 0, "Only initialize once"); + + // Set _dir from d. Resolve relative path if d is relative, and ensure it always + // ends with "/" + if (d[0] != '/') { + char path[PATH_MAX]; + const char* cwd = os::get_current_directory(path, sizeof(path)); + if (cwd == NULL) { + log_warning(vitals)("HiMemReportDir: Failed to resolve current directory (%d)", errno); + return false; + } + _dir.print("%s/", cwd); + } + _dir.print_raw(d); + const size_t l = ::strlen(d); + if (d[l - 1] != '/') { + _dir.put('/'); + } + + // Create the report directory (just the leaf dir, I don't bother creating the whole hierarchy) + struct stat s; + if (::stat(path(), &s) == -1) { + if (::mkdir(path(), S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH) != -1) { + log_info(vitals)("HiMemReportDir: Created report directory \"%s\"", path()); + } else { + log_warning(vitals)("HiMemReportDir: Failed to create report directory \"%s\" (%d)", path(), errno); + return false; + } + } else { + if (S_ISDIR(s.st_mode)) { + log_info(vitals)("HiMemReportDir: Found existing report directory at \"%s\"", path()); + } else { + log_warning(vitals)("HiMemReportDir: \"%s\" exists, but its not a directory.", path()); + return false; + } + } + // Test access by touching a file in this dir. For convenience, we leave the touched file in it + // and write the VM start time and some other info into it. + stringStream testfile; + testfile.print("%sVM_start.pid%d.log", path(), os::current_process_id()); + fileStream fs(testfile.base()); + if (!fs.is_open()) { + log_warning(os)("HiMemReportDir: Cannot write to \"%s\" (%d)", testfile.base(), errno); + return false; + } + print_current_date_and_time(&fs); + return true; + } +}; + +static ReportDir* g_report_dir = NULL; + +static void print_high_memory_report_header(outputStream* st, const char* message, int pid, time_t t) { + char tmp[255]; + st->print_cr("############"); + st->print_cr("#"); + st->print_cr("# High Memory Report:"); + st->print_cr("# pid: %d thread id: " INTX_FORMAT, pid, os::current_thread_id()); + st->print_cr("# %s", message); + st->print_raw("# "); print_date_and_time(st, t); st->cr(); + st->print_cr("# Spike number: %d", g_alert_state->current_spike_no()); + st->print_cr("#"); + st->flush(); +} + +static void print_high_memory_report(outputStream* st) { + + // Note that this report may be interrupted by VM death, e.g. OOM killed. + // Therefore we frequently flush, and print the most important things first. + + char buf[O_BUFLEN]; + + st->print_cr("vm_info: %s", VM_Version::internal_vm_info_string()); + + st->cr(); + st->cr(); + st->flush(); + + Arguments::print_summary_on(st); + st->cr(); + st->cr(); + st->flush(); + + st->print_cr("--- Vitals ---"); + sapmachine_vitals::print_info_t info; + sapmachine_vitals::default_settings(&info); + info.sample_now = true; + info.no_legend = true; + sapmachine_vitals::print_report(st, &info); + st->print_cr("--- /Vitals ---"); + + st->cr(); + st->cr(); + st->flush(); + +#if INCLUDE_NMT + st->cr(); + st->print_cr("--- NMT report ---"); + NMTStuff::report_as_best_as_possible(st); + st->print_cr("--- /NMT report ---"); +#endif + + st->cr(); + st->cr(); + st->flush(); + + st->print_cr("#"); + st->print_cr("# END: High Memory Report"); + st->print_cr("#"); + + st->flush(); +} + +// Create a file name into the report directory: /.__. +// (leave dir NULL to just get a file name) +static void print_file_name(stringStream* ss, const char* name, int pid, time_t timestamp, const char* suffix) { + assert(g_report_dir != NULL, "must be"); + const char* dir = g_report_dir->path(); + // Should already have been made absolute, and should end with / (see ReportDir::initialize()). + assert(dir[0] == '/' && dir[::strlen(dir) - 1] == '/', + "bad value for report dir? %s", dir); + ss->print("%s", dir); + ss->print("%s_pid%d_", name, pid); + print_date_and_time_underscored(ss, timestamp); + ss->print("%s", suffix); +} + +///////////////////// JCmd support ////////////////////////////////////////// + +class ParsedCommand { + + stringStream _name; // command name without args + stringStream _args; // arguments + +public: + + ParsedCommand(const char* command) { + // trim front + const char* p = command; + while (isspace(*p)) { + p ++; + } + if ((*p) != '\0') { + // read name + while (!isspace(*p) && (*p) != '\0') { + _name.put(*p); + p++; + } + // find start of args; read args + while (isspace(*p)) { + p ++; + } + _args.print_raw(p); + } + } + + bool is_empty() const { return _name.size() == 0; } + const char* name() const { return _name.base(); } + + bool has_arguments() const { return _args.size() > 0; } + const char* args() const { return _args.base(); } + + // Unfortunately, the DCmd framework lacks the ability to check DCmd without + // executing them. Here, we do some simple basic checks. Failing them will + // exit the VM right away, but passing them does still not mean the command + // is well formed since we don't check the arguments. + bool is_valid() const { + static const char* valid_prefixes[] = { "Compiler", "GC", "JFR", "JVMTI", + "Management", "System", "Thread", + "VM", "help", NULL }; + if (_name.size() > 0) { + for (const char** p = valid_prefixes; (*p) != NULL; p ++) { + if (::strncasecmp(_name.base(), *p, ::strlen(*p)) == 0) { + return true; + } + } + } + return false; + } +}; + +// Helper structures for posix_spawn_file_actions_t and posix_spawnattr_t where +// cleanup depends on successful initialization. +// Helper structures for posix_spawn_file_actions_t and posix_spawnattr_t where +// cleanup depends on successful initialization. +struct PosixSpawnFileActions { + posix_spawn_file_actions_t v; + const bool ok; + PosixSpawnFileActions() : ok(::posix_spawn_file_actions_init(&v) == 0) {} + ~PosixSpawnFileActions() { ok && ::posix_spawn_file_actions_destroy(&v); } +}; + +struct PosixSpawnAttr { + posix_spawnattr_t v; + const bool ok; + PosixSpawnAttr() : ok(::posix_spawnattr_init(&v) == 0) {} + ~PosixSpawnAttr() { ok && ::posix_spawnattr_destroy(&v); } +}; + +// Call jcmd. If outFile and errFile are not Null, redirect stdout and stderr, otherwise +// print both stdout and stderr to VMs stderr. +// Returns true if command was executed successfully and exitcode was 0, false otherwise. +// If command failed, err_msg will contain an error string. +static bool spawn_command(const char** argv, const char* outFile, const char* errFile, stringStream* err_msg) { + + // I want vfork, but use posix_spawn, since vfork() is becoming obsolete and compilers + // will warn. Its also safer, and with modern glibcs it is as cheap as vfork. + PosixSpawnFileActions fa; + PosixSpawnAttr atr; + + bool rc = fa.ok && atr.ok; + + if (outFile != NULL) { // Redirect stdout, stderr to files + assert(errFile != NULL, "Require both"); + rc = rc && (::posix_spawn_file_actions_addopen(&fa.v, 1, outFile, O_WRONLY | O_CREAT | O_TRUNC, 0664) == 0) && + (::posix_spawn_file_actions_addopen(&fa.v, 2, errFile, O_WRONLY | O_CREAT | O_TRUNC, 0664) == 0); + } else { // Dup stdout to stderr + rc = rc && (::posix_spawn_file_actions_adddup2 (&fa.v, 2, 1) == 0); + } + pid_t child_pid = -1; + + // Hint toward vfork. Note that newer glibcs (2.24+) will ignore this, but they use clone(), + // so its alright. + rc = rc && (posix_spawnattr_setflags(&atr.v, POSIX_SPAWN_USEVFORK) == 0); + + if (rc == false) { + err_msg->print("Error during posix_spawn setup"); + return false; + } + + // Note about inheriting file descriptors: in theory, posix_spawn should close all stray descriptors: + // "If file_actions is a null pointer, then file descriptors open in the calling process shall remain open + // in the child process, except for those whose close-on- exec flag FD_CLOEXEC is set (see fcntl)." + // (https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_spawnp.html) + // - which I assume means they get closed if we specify a file actions object, which we do. + rc = rc && (::posix_spawn(&child_pid, argv[0], &fa.v, &atr.v, (char**)argv, os::get_environ()) == 0); + + if (rc) { + int status; + rc = (::waitpid(child_pid, &status, 0) != -1) && + (WIFEXITED(status) && WEXITSTATUS(status) == 0); + if (rc == false) { + err_msg->print("Command failed or crashed"); + } + } else { + err_msg->print("posix_spawn failed (%s)", os::strerror(errno)); + } + + return rc; +} + +// Calls a single jcmd via posix_spawn. Output is written to /--- +// if HiMemReportDir is given; to stdout if not. +static void call_single_jcmd(const ParsedCommand* cmd, int pid, time_t t) { + + // if report dir is given, calc .out and .err file names + const char* out_file = NULL; + const char* err_file = NULL; + stringStream out_file_ss, err_file_ss; + if (g_report_dir != NULL) { + // output files are named _pid_.(out|err), e.g. "VM.info_4711_2022_08_01_07_52_22.out". + print_file_name(&out_file_ss, cmd->name(), pid, t, ".out"); + out_file = out_file_ss.base(); + print_file_name(&err_file_ss, cmd->name(), pid, t, ".err"); + err_file = err_file_ss.base(); + } + + stringStream jcmd_executable; + jcmd_executable.print("%s/bin/jcmd", Arguments::get_java_home()); + + stringStream target_pid; + target_pid.print("%d", pid); + + stringStream jcmd_command; + jcmd_command.print_raw(cmd->name()); + if (cmd->has_arguments()) { + jcmd_command.put(' '); + jcmd_command.print_raw(cmd->args()); + } + + // Special consideration for GC.heap_dump: if the command was given without arguments, we append + // a file name for the heap dump ("/heapdump_pid_.dump") + if (!cmd->has_arguments() && ::strcmp(cmd->name(), "GC.heap_dump") == 0) { + jcmd_command.put(' '); + print_file_name(&jcmd_command, "GC.heap_dump", pid, t, ".dump"); + } + + const char* argv[4]; + argv[0] = jcmd_executable.base(); + argv[1] = target_pid.base(); + argv[2] = jcmd_command.base(); + argv[3] = NULL; + + stringStream err_msg; + const jlong t1 = os::javaTimeNanos(); + if (spawn_command(argv, out_file, err_file, &err_msg)) { + const jlong t2 = os::javaTimeNanos(); + const int command_time_ms = (t2 - t1) / (1000 * 1000); + stderr_stream.print("HiMemReport: Successfully executed \"%s\" (%d ms)", jcmd_command.base(), command_time_ms); + if (out_file != NULL) { + stderr_stream.print(", output redirected to report dir"); + } + stderr_stream.cr(); + } else { + stderr_stream.print("HiMemReport: Failed to execute \"%s\" (%s)", jcmd_command.base(), err_msg.base()); + } +} + +// Helper, trims string +static char* trim_string(char* s) { + char* p = s; + while (::isspace(*p)) p++; + char* p2 = p + ::strlen(p) - 1; + while (p2 > p && ::isspace(*p2)) { + *p2 = '\0'; + p2--; + } + return p; +} + +struct JcmdClosure { + virtual bool do_it(const char* cmd) = 0; +}; + +static bool iterate_exec_string(const char* exec_string, JcmdClosure* closure) { + char* exec_copy = os::strdup(exec_string); + char* save = NULL; + for (char* tok = strtok_r(exec_copy, ";", &save); + tok != NULL; tok = ::strtok_r(NULL, ";", &save)) { + const char* p = trim_string(tok); + if (::strlen(p) > 0 && !closure->do_it(p)) { + os::free(exec_copy); + return false; + } + } + os::free(exec_copy); + return true; +} + +class CallJCmdClosure : public JcmdClosure { + const pid_t _pid; + const time_t _time; +public: + CallJCmdClosure(int pid, time_t time) : _pid(pid), _time(time) {} + bool do_it(const char* command_string) override { + ParsedCommand cmd(command_string); + assert(cmd.is_valid(), "Invalid command"); + call_single_jcmd(&cmd, _pid, _time); + return true; + } +}; + +struct VerifyJCmdClosure : public JcmdClosure { + bool do_it(const char* command_string) override { + log_info(vitals)("HiMemReportExec: storing command \"%s\".", command_string); + if (!ParsedCommand(command_string).is_valid()) { + // We print a warning here, fingerpointing the specific command that failed, then exit the VM later. + log_warning(vitals)("HiMemReportExec: Command \"%s\" invalid.", command_string); + return false; + } + return true; + } +}; + +//////////////////// alert handling and reporting /////////////////////////////////////////////////////////////// + +static int g_num_alerts = 0; + +// We don't want to flood the report directory if the footprint of the VM wobbles strongly. We will give up +// after a reasonable amount of reports have been printed. +static const int max_spikes = 32; + +static void trigger_high_memory_report(int alvl, int spikeno, int percentage, size_t triggering_size) { + + if (spikeno >= max_spikes) { + if (spikeno == max_spikes) { + stderr_stream.print_cr("# HiMemReport: Too many spikes encountered. Further reports will be omitted."); + } + return; + } + + g_num_alerts ++; + + stringStream reason; + reason.print("rss+swap (" SIZE_FORMAT " K) larger than %d%% of %s (" SIZE_FORMAT " K).", + triggering_size / K, percentage, describe_maximum_by_compare_type(g_compare_what), + g_alert_state->maximum() / K); + const char* message = reason.base(); + + const int pid = os::current_process_id(); + time_t t; + time(&t); + + bool printed = false; + + print_high_memory_report_header(&stderr_stream, message, pid, t); + + if (g_report_dir != NULL) { + // Dump to file in report dir + stringStream ss; + print_file_name(&ss, "sapmachine_himemalert", pid, t, ".log"); + fileStream fs(ss.base()); + if (fs.is_open()) { + stderr_stream.print_cr("# Printing to %s", ss.base()); + print_high_memory_report_header(&fs, message, pid, t); + print_high_memory_report(&fs); + printed = true; + } else { + stderr_stream.print_cr("# Failed to open %s. Printing to stderr instead.", ss.base()); + stderr_stream.cr(); + } + stderr_stream.flush(); + } + + if (!printed) { + print_high_memory_report(&stderr_stream); + } + + stderr_stream.print_cr("# Done."); + stderr_stream.print_raw("#"); + stderr_stream.cr(); + stderr_stream.flush(); + + if (HiMemReportExec != NULL) { + CallJCmdClosure closure(pid, t); + iterate_exec_string(HiMemReportExec, &closure); + } + +} + +///////////////// Monitor thread ///////////////////////////////////////// + +void pulse_himem_report() { + assert(HiMemReport, "only call for +HiMemReport"); + assert(g_compare_what != compare_type::compare_none && g_alert_state != NULL, "Not initialized"); + + OSWrapper::update_if_needed(); + + const value_t rss = OSWrapper::proc_rss_all(); + const value_t swap = OSWrapper::proc_swdo(); + if (rss != INVALID_VALUE && swap != INVALID_VALUE) { + const size_t rss_swap = (size_t)rss + (size_t)swap; + const int old_alvl = g_alert_state->current_alert_level(); + g_alert_state->update(rss_swap); + const int new_alvl = g_alert_state->current_alert_level(); + const int spikeno = g_alert_state->current_spike_no(); + + if (new_alvl > old_alvl) { + const int new_percentage = g_alert_state->current_alert_level_percentage(); + stderr_stream.print_cr("HiMemoryReport: rss+swap=" SIZE_FORMAT " K - alert level increased to %d (>=%d%%).", + rss_swap / K, new_alvl, new_percentage); + int skipped = 0; + for (int i = old_alvl + 1; i < new_alvl; i ++) { + skipped ++; + // We may have missed some intermediary steps because the pulse interval was too large. + stderr_stream.print_cr("HiMemoryReport: ... seems we passed alert level %d (%d%%) without noticing.", + i, AlertState::alert_level_percentage(i)); + } + // If the alert level increased to a new value, trigger a new report + trigger_high_memory_report(new_alvl, spikeno, new_percentage, rss_swap); +#if INCLUDE_NMT + // Upon first alert, do a NMT baseline + if (old_alvl == 0 && new_alvl > 0) { + if (NMTStuff::is_enabled()) { + NMTStuff::capture_baseline(); + stderr_stream.print_cr("HiMemoryReport: ... captured NMT baseline"); + } + } +#endif // INCLUDE_NMT + } else if (old_alvl > 0 && new_alvl == 0){ + // Memory usage recovered, and we hit the decay time, and now all is well again. + stderr_stream.print_cr("HiMemoryReport: rss+swap=" SIZE_FORMAT " K - seems we recovered. Resetting alert level.", + rss_swap / K); +#if INCLUDE_NMT + NMTStuff::reset(); +#endif + } + } +} + +class HiMemReportThread: public NamedThread { + + static const int interval_seconds = 2; + +public: + + HiMemReportThread() : NamedThread() { + this->set_name("himem reporter"); + } + + virtual void run() { + record_stack_base_and_size(); + for (;;) { + pulse_himem_report(); + os::naked_sleep(interval_seconds * 1000); + } + } + +}; + +static HiMemReportThread* g_reporter_thread = NULL; + +static bool initialize_reporter_thread() { + g_reporter_thread = new HiMemReportThread(); + if (g_reporter_thread != NULL) { + if (os::create_thread(g_reporter_thread, os::os_thread)) { + os::start_thread(g_reporter_thread); + } + return true; + } + return false; +} + +///////////////// Externals ////////////////////////////////////////////// + +extern void initialize_himem_report_facility() { + + static bool initialized = false; + assert(initialized == false, "HiMemReport already initialized"); + initialized = true; + + // Note: + // unrecoverable errors: + // - errors the user can easily correct (bad arguments) cause exit right away + // - errors which are subject to environment and cannot be dealt with/are unpredictable + // cause facility to be disabled (with UL warning) + + assert(HiMemReport, "only call for +HiMemReport"); + + assert(g_compare_what == compare_type::compare_none && g_alert_state == NULL, "Only initialize once"); + + // Verify the exec string + VerifyJCmdClosure closure; + if (HiMemReportExec != NULL && iterate_exec_string(HiMemReportExec, &closure) == false) { + vm_exit_during_initialization("Vitals HiMemReportExec: One or more Exec commands were invalid"); + } + + // We need to decide what we will compare with what. To do that, we get the current system values. + // - If user manually specified a maximum, we will compare rss+swap with that maximum + // - If we live inside a cgroup with a memory limit, we will compare process rss+swap vs this limit + // (snapshotted at VM start; maybe later we can react to dynamic limit changes, but for the moment I don't care) + // - If we do not live in a cgroup, or in a cgroup with no limit, compare process rss+swap vs the + // physical memory of the machine. + size_t limit = 0; + if (HiMemReportMax != 0) { + g_compare_what = compare_type::compare_rss_vs_manual_limit; + limit = HiMemReportMax; + log_info(vitals)("Vitals HiMemReport: Setting limit to HiMemReportMax (" SIZE_FORMAT " K).", limit / K); + } else { + OSWrapper::update_if_needed(); + if (OSWrapper::syst_cgro_lim() != INVALID_VALUE) { + // limit against cgroup limit + g_compare_what = compare_type::compare_rss_vs_cgroup_limit; + limit = (size_t)OSWrapper::syst_cgro_lim(); + log_info(vitals)("Vitals HiMemReport: Setting limit to cgroup memory limit (" SIZE_FORMAT " K).", limit / K); + } else if (OSWrapper::syst_phys() != INVALID_VALUE) { + // limit against total physical memory + g_compare_what = compare_type::compare_rss_vs_phys; + limit = (size_t)OSWrapper::syst_phys() / 2; + log_info(vitals)("Vitals HiMemReport: Setting limit to half of total physical memory (" SIZE_FORMAT " K).", limit / K); + } + } + + if (limit == 0) { + log_warning(vitals)("Vitals HiMemReport: limit could not be established; will disable high memory reports " + "(specify -XX:HiMemReportMax= to establish a manual limit)."); + FLAG_SET_ERGO(HiMemReport, false); + return; + } + + // HiMemReportDir: + // We fix up the report directory when VM starts, so if its relative, it refers to the initial current directory. + // If it cannot be established, we treat it as predictable argument error and exit the VM. + if (HiMemReportDir != NULL && ::strlen(HiMemReportDir) > 0) { + g_report_dir = new ReportDir(); + if (!g_report_dir->initialize(HiMemReportDir)) { + log_warning(vitals)("Vitals: Cannot access HiMemReportDir %s.", g_report_dir->path()); + vm_exit_during_initialization("Vitals HiMemReport: Failed to create or access HiMemReportDir \"%s\".", g_report_dir->path()); + return; + } + } + + g_alert_state = new AlertState(limit); + + if (!initialize_reporter_thread()) { + log_warning(vitals)("Vitals HiMemReport: Failed to start monitor thread. Will disable."); + FLAG_SET_ERGO(HiMemReport, false); + return; + } + + log_info(vitals)("Vitals: HiMemReport subsystem initialized."); + +} + +extern void print_himemreport_state(outputStream* st) { + if (g_alert_state != NULL) { + st->print("HiMemReport: monitoring rss+swap vs %s (" SIZE_FORMAT " K)", + describe_maximum_by_compare_type(g_compare_what), + g_alert_state->maximum() / K); + if (g_alert_state->current_alert_level() == 0) { + st->print(", all is well"); + } else { + st->print(", current level: %d (%d%%)", g_alert_state->current_alert_level(), + g_alert_state->current_alert_level_percentage()); + } + st->print(", spikes: %d, alerts: %d", g_alert_state->current_spike_no(), g_num_alerts); + } else { + st->print("HiMemReport: not monitoring."); + } +} + +// For printing in thread lists only. +extern const Thread* himem_reporter_thread() { return g_reporter_thread; } + +} // namespace sapmachine_vitals + diff --git a/src/hotspot/os/linux/vitals_linux_himemreport.hpp b/src/hotspot/os/linux/vitals_linux_himemreport.hpp new file mode 100644 index 00000000000..cc173b1b853 --- /dev/null +++ b/src/hotspot/os/linux/vitals_linux_himemreport.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_VITALS_LINUX_HELPERS_HPP +#define OS_LINUX_VITALS_LINUX_HELPERS_HPP + +#include "vitals/vitals_internals.hpp" + +namespace sapmachine_vitals { + +void initialize_himem_report_facility(); + +void print_himemreport_state(outputStream* st); + +// For printing in thread lists only. +extern const Thread* himem_reporter_thread(); + +} // namespace sapmachine_vitals + +#endif // OS_LINUX_VITALS_LINUX_HELPERS_HPP + diff --git a/src/hotspot/os/linux/vitals_linux_oswrapper.cpp b/src/hotspot/os/linux/vitals_linux_oswrapper.cpp new file mode 100644 index 00000000000..edff0ec368f --- /dev/null +++ b/src/hotspot/os/linux/vitals_linux_oswrapper.cpp @@ -0,0 +1,570 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "jvm_io.h" +#include "osContainer_linux.hpp" +#include "vitals_linux_oswrapper.hpp" +#include "logging/log.hpp" +#include "runtime/os.hpp" +#include "utilities/globalDefinitions.hpp" +#include "vitals/vitals_internals.hpp" + +#include +#include +#include + + + +#define LOG_HERE_F(msg, ...) { printf("[%d] ", (int)::getpid()); ::printf(msg, __VA_ARGS__); printf("\n"); fflush(stdout); } +#define LOG_HERE(msg) { printf("[%d] ", (int)::getpid()); ::printf("%s", msg); printf("\n"); fflush(stdout); } + +extern const char* sapmachine_get_memory_controller_path(); + +namespace sapmachine_vitals { + +#define DEFINE_VARIABLE(name) \ + value_t OSWrapper::_##name = INVALID_VALUE; + +ALL_VALUES_DO(DEFINE_VARIABLE) + +#undef DEFINE_VARIABLE + +time_t OSWrapper::_last_update = 0; + +static const int num_seconds_until_update = 1; + +///////////// procfs stuff ////////////////////////////////////////////////// + +class ProcFile { + char* _buf; + + // To keep the code simple, I just use a fixed sized buffer. + enum { bufsize = 64*K }; + +public: + + ProcFile() : _buf(NULL) { + _buf = (char*)os::malloc(bufsize, mtInternal); + } + + ~ProcFile () { + os::free(_buf); + } + + bool read(const char* filename) { + + FILE* f = ::fopen(filename, "r"); + if (f == NULL) { + log_debug(vitals)("Failed to fopen %s (%d)", filename, errno); + return false; + } + + size_t bytes_read = ::fread(_buf, 1, bufsize - 1, f); + _buf[bytes_read] = '\0'; + + ::fclose(f); + + return bytes_read > 0 && bytes_read < bufsize; + } + + const char* text() const { return _buf; } + + // Utility function; parse a number string as value_t + static value_t as_value(const char* text, size_t scale = 1) { + value_t value; + errno = 0; + char* endptr = NULL; + value = (value_t)::strtoll(text, &endptr, 10); + if (endptr == text || errno != 0) { + value = INVALID_VALUE; + } else { + value *= scale; + } + return value; + } + + // Return the start of the file, as number. Useful for proc files which + // contain a single number. Returns INVALID_VALUE if value did not parse + value_t as_value(size_t scale = 1) const { + return as_value(_buf, scale); + } + + const char* get_prefixed_line(const char* prefix) const { + return ::strstr(_buf, prefix); + } + + value_t parsed_prefixed_value(const char* prefix, size_t scale = 1) const { + value_t value = INVALID_VALUE; + const char* const s = get_prefixed_line(prefix); + if (s != NULL) { + errno = 0; + const char* p = s + ::strlen(prefix); + return as_value(p, scale); + } + return value; + } + +}; + +struct cpu_values_t { + value_t user; + value_t nice; + value_t system; + value_t idle; + value_t iowait; + value_t steal; + value_t guest; + value_t guest_nice; +}; + +static void parse_proc_stat_cpu_line(const char* line, cpu_values_t* out) { + // Note: existence of some of these values depends on kernel version + out->user = out->nice = out->system = out->idle = out->iowait = out->steal = out->guest = out->guest_nice = + INVALID_VALUE; + uint64_t user, nice, system, idle, iowait, irq, softirq, steal, guest, guest_nice; + int num = ::sscanf(line, + "cpu " + UINT64_FORMAT " " UINT64_FORMAT " " UINT64_FORMAT " " UINT64_FORMAT " " UINT64_FORMAT " " + UINT64_FORMAT " " UINT64_FORMAT " " UINT64_FORMAT " " UINT64_FORMAT " " UINT64_FORMAT " ", + &user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal, &guest, &guest_nice); + if (num >= 4) { + out->user = user; + out->nice = nice; + out->system = system; + out->idle = idle; + if (num >= 5) { // iowait (5) (since Linux 2.5.41) + out->iowait = iowait; + if (num >= 8) { // steal (8) (since Linux 2.6.11) + out->steal = steal; + if (num >= 9) { // guest (9) (since Linux 2.6.24) + out->guest = guest; + if (num >= 10) { // guest (9) (since Linux 2.6.33) + out->guest_nice = guest_nice; + } + } + } + } + } +} + + +#ifdef __GLIBC__ +// We use either mallinfo (which may be obsolete or removed in newer glibc versions) or mallinfo2 +// (which does not exist prior to glibc 2.34). + +#define MALLINFO_MEMBER_DO(f) \ + f(arena) \ + f(ordblks) \ + f(smblks) \ + f(hblks) \ + f(hblkhd) \ + f(usmblks) \ + f(fsmblks) \ + f(uordblks) \ + f(fordblks) \ + f(keepcost) + +struct glibc_mallinfo { +#define DEF_MALLINFO_MEMBER(f) int f; + MALLINFO_MEMBER_DO(DEF_MALLINFO_MEMBER) +#undef DEF_MALLINFO_MEMBER +}; + +struct glibc_mallinfo2 { +#define DEF_MALLINFO2_MEMBER(f) size_t f; + MALLINFO_MEMBER_DO(DEF_MALLINFO2_MEMBER) +#undef DEF_MALLINFO2_MEMBER +}; + +typedef struct glibc_mallinfo (*mallinfo_func_t)(void); +typedef struct glibc_mallinfo2 (*mallinfo2_func_t)(void); + +static mallinfo_func_t g_mallinfo = NULL; +static mallinfo2_func_t g_mallinfo2 = NULL; + +static void mallinfo_init() { + g_mallinfo = CAST_TO_FN_PTR(mallinfo_func_t, dlsym(RTLD_DEFAULT, "mallinfo")); + g_mallinfo2 = CAST_TO_FN_PTR(mallinfo2_func_t, dlsym(RTLD_DEFAULT, "mallinfo2")); +} + +#undef MALLINFO_MEMBER_DO + +#endif // __GLIBC__ + +// Helper function, returns true if string is a numerical id +static bool is_numerical_id(const char* s) { + const char* p = s; + while(*p >= '0' && *p <= '9') { + p ++; + } + return *p == '\0' ? true : false; +} + +/////////////// cgroup stuff +// We use part of the hotspot cgroup wrapper, but not all of it. +// The reason: +// - wrapper uses UL heavily, which I don't want to happen in a sampler thread (I only log in initialization, which is ok) +// - wrapper does not expose all metrics I need (eg kmem) +// What the wrapper does very nicely is the parse stuff, which I don't want to re-invent, therefore +// I use the wrapper to get the controller path. + +class CGroups : public AllStatic { + + static bool _containerized; + static const char* _file_usg; + static const char* _file_usgsw; + static const char* _file_lim; + static const char* _file_limsw; + static const char* _file_slim; + static const char* _file_kusg; + +public: + + static bool initialize() { + + // For the heck of it, I go through with initialization even if we are not + // containerized, since I like to know controller paths even for those cases. + + _containerized = OSContainer::is_containerized(); + log_debug(vitals)("Vitals cgroup initialization: containerized = %d", _containerized); + + const char* controller_path = sapmachine_get_memory_controller_path(); + if (controller_path == NULL) { + log_debug(vitals)("Vitals cgroup initialization: controller path NULL"); + return false; + } + size_t pathlen = ::strlen(controller_path); + if (pathlen == 0) { + log_debug(vitals)("Vitals cgroup initialization: controller path empty?"); + return false; + } + stringStream path; + if (controller_path[pathlen - 1] == '/') { + path.print("%s", controller_path); + } else { + path.print("%s/", controller_path); + } + + log_debug(vitals)("Vitals cgroup initialization: controller path: %s", path.base()); + + // V1 or V2? + stringStream ss; + ss.print("%smemory.usage_in_bytes", path.base()); + struct stat s; + const bool isv1 = os::file_exists(ss.base()); + if (isv1) { + log_debug(vitals)("Vitals cgroup initialization: v1"); + } else { + ss.reset(); + ss.print("%smemory.current", path.base()); + if (os::file_exists(ss.base())) { + // okay, its v2 + log_debug(vitals)("Vitals cgroup initialization: v2"); + } else { + log_debug(vitals)("Vitals cgroup initialization: no clue. Giving up."); + return false; + } + } + + _file_usg = os::strdup(ss.base()); // so, we have that. + +#define STORE_PATH(variable, filename) \ + ss.reset(); ss.print("%s%s", path.base(), filename); variable = os::strdup(ss.base()); + + if (isv1) { + STORE_PATH(_file_usgsw, "memory.memsw.usage_in_bytes"); + STORE_PATH(_file_kusg, "memory.kmem.usage_in_bytes"); + STORE_PATH(_file_lim, "memory.limit_in_bytes"); + STORE_PATH(_file_limsw, "memory.memsw.limit_in_bytes"); + STORE_PATH(_file_slim, "memory.soft_limit_in_bytes"); + } else { + STORE_PATH(_file_usgsw, "memory.swap.current"); + STORE_PATH(_file_kusg, "memory.kmem.usage_in_bytes"); + STORE_PATH(_file_lim, "memory.max"); + STORE_PATH(_file_limsw, "memory.swap.max"); + STORE_PATH(_file_slim, "memory.low"); + } +#undef STORE_PATH + +#define LOG_PATH(variable) \ + log_debug(vitals)("Vitals: %s=%s", #variable, variable == NULL ? "" : variable); + LOG_PATH(_file_usg) + LOG_PATH(_file_usgsw) + LOG_PATH(_file_kusg) + LOG_PATH(_file_lim) + LOG_PATH(_file_limsw) + LOG_PATH(_file_slim) +#undef LOG_PATH + + // Initialization went through. We show columns if we are containerized. + return _containerized; + } + + struct cgroup_values_t { + value_t lim; + value_t limsw; + value_t slim; + value_t usg; + value_t usgsw; + value_t kusg; + }; + + static bool get_stats(cgroup_values_t* v) { + v->lim = v->limsw = v->slim = v->usg = v->usgsw = v->kusg = INVALID_VALUE; + ProcFile pf; +#define GET_VALUE(var) \ + { \ + const char* what = _file_ ## var; \ + if (what != NULL && pf.read(what)) { \ + v-> var = pf.as_value(1); \ + } \ + } + GET_VALUE(usg); + GET_VALUE(usgsw); + GET_VALUE(kusg); + GET_VALUE(lim); + GET_VALUE(limsw); + GET_VALUE(slim); +#undef GET_VALUE + // Cgroup limits defaults to PAGE_COUNTER_MAX in the kernel; so a very large number means "no limit" + // Note that on 64-bit, the default is LONG_MAX aligned down to pagesize; but I am not sure this is + // always true, so I just assume a very high value. + const size_t practically_infinite = LP64_ONLY(128 * K * G) NOT_LP64(4 * G); + if (v->lim > practically_infinite) v->lim = INVALID_VALUE; + if (v->slim > practically_infinite) v->slim = INVALID_VALUE; + if (v->limsw > practically_infinite) v->limsw = INVALID_VALUE; + return true; + + } // end: CGroups::get_stats() + +}; // end: CGroups + +bool CGroups::_containerized = false; +const char* CGroups::_file_usg = NULL; +const char* CGroups::_file_usgsw = NULL; +const char* CGroups::_file_lim = NULL; +const char* CGroups::_file_limsw = NULL; +const char* CGroups::_file_slim = NULL; +const char* CGroups::_file_kusg = NULL; + +void OSWrapper::update_if_needed() { + + time_t t; + time(&t); + if (t != (time_t)-1 && + t < (_last_update + num_seconds_until_update)) { + return; // still good + } + _last_update = t; + + static bool first_call = true; + + // Update Values from ProcFS (and elsewhere) +#define RESETVAL(name) _ ## name = INVALID_VALUE; +ALL_VALUES_DO(RESETVAL) +#undef RESETVAL + + ProcFile bf; + if (bf.read("/proc/meminfo")) { + + if (first_call) { + log_trace(vitals)("Read /proc/meminfo: \n%s", bf.text()); + } + + // All values in /proc/meminfo are in KB + const size_t scale = K; + + _syst_phys = bf.parsed_prefixed_value("MemTotal:", scale); + _syst_avail = bf.parsed_prefixed_value("MemAvailable:", scale); + + const value_t swap_total = bf.parsed_prefixed_value("SwapTotal:", scale); + const value_t swap_free = bf.parsed_prefixed_value("SwapFree:", scale); + if (swap_total != INVALID_VALUE && swap_free != INVALID_VALUE) { + _syst_swap = swap_total - swap_free; + } + + // Calc committed ratio. Values > 100% indicate overcommitment. + const value_t commitlimit = bf.parsed_prefixed_value("CommitLimit:", scale); + const value_t committed = bf.parsed_prefixed_value("Committed_AS:", scale); + if (commitlimit != INVALID_VALUE && commitlimit != 0 && committed != INVALID_VALUE) { + _syst_comm = committed; + const value_t ratio = (committed * 100) / commitlimit; + _syst_crt = ratio; + } + + } + + if (bf.read("/proc/vmstat")) { + _syst_si = bf.parsed_prefixed_value("pswpin"); + _syst_so = bf.parsed_prefixed_value("pswpout"); + } + + if (bf.read("/proc/stat")) { + // Read and parse global cpu values + cpu_values_t values; + const char* line = bf.get_prefixed_line("cpu"); + parse_proc_stat_cpu_line(line, &values); + + _syst_cpu_us = values.user + values.nice; + _syst_cpu_sy = values.system; + _syst_cpu_id = values.idle; + _syst_cpu_st = values.steal; + _syst_cpu_gu = values.guest + values.guest_nice; + + // procs_running: this is actually number of threads running + // procs_blocked: number of threads blocked on real disk IO + // See https://utcc.utoronto.ca/~cks/space/blog/linux/ProcessStatesAndProcStat + // and https://lore.kernel.org/lkml/12601530441257@xenotime.net/#t + // and the canonical man page description at https://www.kernel.org/doc/Documentation/filesystems/proc.txt + _syst_tr = bf.parsed_prefixed_value("procs_running"); + _syst_tb = bf.parsed_prefixed_value("procs_blocked"); + } + + // cgroups business + CGroups::cgroup_values_t v; + if (CGroups::get_stats(&v)) { + _syst_cgro_usg = v.usg; + _syst_cgro_usgsw = v.usgsw; + _syst_cgro_kusg = v.kusg; + _syst_cgro_lim = v.lim; + _syst_cgro_limsw = v.limsw; + _syst_cgro_slim = v.slim; + } + + if (bf.read("/proc/self/status")) { + + _proc_virt = bf.parsed_prefixed_value("VmSize:", K); + _proc_swdo = bf.parsed_prefixed_value("VmSwap:", K); + _proc_rss_all = bf.parsed_prefixed_value("VmRSS:", K); + _proc_rss_anon = bf.parsed_prefixed_value("RssAnon:", K); + _proc_rss_file = bf.parsed_prefixed_value("RssFile:", K); + _proc_rss_shm = bf.parsed_prefixed_value("RssShmem:", K); + + _proc_thr = bf.parsed_prefixed_value("Threads:"); + + } + + // Number of open files: iterate over /proc/self/fd and count. + { + DIR* d = ::opendir("/proc/self/fd"); + if (d != NULL) { + value_t v = 0; + struct dirent* en = NULL; + do { + en = ::readdir(d); + if (en != NULL) { + v ++; + } + } while(en != NULL); + ::closedir(d); + assert(v >= 2, "should have read at least '.' and '..'"); + v -= 2; // We discount . and .. + _proc_io_of = v; + } + } + + // Number of processes: iterate over /proc/ and count. + // Number of threads: read "num_threads" from /proc//stat + { + DIR* d = ::opendir("/proc"); + if (d != NULL) { + value_t v_p = 0; + value_t v_t = 0; + struct dirent* en = NULL; + do { + en = ::readdir(d); + if (en != NULL) { + if (is_numerical_id(en->d_name)) { + v_p ++; + char tmp[128]; + jio_snprintf(tmp, sizeof(tmp), "/proc/%s/stat", en->d_name); + if (bf.read(tmp)) { + const char* text = bf.text(); + // See man proc(5) + // (20) num_threads %ld + long num_threads = 0; + ::sscanf(text, "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %*u %*u %*d %*d %*d %*d %ld", &num_threads); + v_t += num_threads; + } + } + } + } while(en != NULL); + ::closedir(d); + _syst_p = v_p; + _syst_t = v_t; + } + } + + if (bf.read("/proc/self/io")) { + _proc_io_rd = bf.parsed_prefixed_value("rchar:"); + _proc_io_wr = bf.parsed_prefixed_value("wchar:"); + } + + if (bf.read("/proc/self/stat")) { + const char* text = bf.text(); + // See man proc(5) + // (14) utime %lu + // (15) stime %lu + long unsigned cpu_utime = 0; + long unsigned cpu_stime = 0; + ::sscanf(text, "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %lu %lu", &cpu_utime, &cpu_stime); + _proc_cpu_us = cpu_utime; + _proc_cpu_sy = cpu_stime; + } + +#ifdef __GLIBC__ + // Note "glibc heap used", from experiments and glibc source code reading, would be aprox. the sum + // of mmaped data area size (contains large allocations) and the small block sizes. + if (g_mallinfo2 != NULL) { + glibc_mallinfo2 mi = g_mallinfo2(); + _proc_chea_usd = mi.uordblks + mi.hblkhd; + _proc_chea_free = mi.fordblks; + } else if (g_mallinfo != NULL) { + // disregard output from old style mallinfo if rss > 4g, since we cannot + // know whether we wrapped. For rss < 4g, we know values in mallinfo cannot + // have wrapped. + if (LP64_ONLY(_proc_rss_all < (4 * G)) NOT_LP64(true)) { + glibc_mallinfo mi = g_mallinfo(); + _proc_chea_usd = (value_t)(unsigned)mi.uordblks + (value_t)(unsigned)mi.hblkhd; + _proc_chea_free = (value_t)(unsigned)mi.fordblks; + } + } +#endif // __GLIBC__ + + first_call = false; + +} + +bool OSWrapper::initialize() { +#ifdef __GLIBC__ + mallinfo_init(); +#endif + return CGroups::initialize(); +} + +} // namespace sapmachine_vitals diff --git a/src/hotspot/os/linux/vitals_linux_oswrapper.hpp b/src/hotspot/os/linux/vitals_linux_oswrapper.hpp new file mode 100644 index 00000000000..f58813dda0e --- /dev/null +++ b/src/hotspot/os/linux/vitals_linux_oswrapper.hpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_VITALS_LINUX_PROCFS_HPP +#define OS_LINUX_VITALS_LINUX_PROCFS_HPP + +#include "utilities/globalDefinitions.hpp" +#include "vitals/vitals_internals.hpp" + +namespace sapmachine_vitals { + +class OSWrapper { + + static time_t _last_update; + +#define ALL_VALUES_DO(f) \ + f(syst_phys) \ + f(syst_avail) \ + f(syst_comm) \ + f(syst_crt) \ + f(syst_swap) \ + f(syst_si) \ + f(syst_so) \ + f(syst_p) \ + f(syst_t) \ + f(syst_tr) \ + f(syst_tb) \ + f(syst_cpu_us) \ + f(syst_cpu_sy) \ + f(syst_cpu_id) \ + f(syst_cpu_st) \ + f(syst_cpu_gu) \ + f(syst_cgro_lim) \ + f(syst_cgro_limsw) \ + f(syst_cgro_slim) \ + f(syst_cgro_usg) \ + f(syst_cgro_usgsw) \ + f(syst_cgro_kusg) \ + f(proc_virt) \ + f(proc_rss_all) \ + f(proc_rss_anon) \ + f(proc_rss_file) \ + f(proc_rss_shm) \ + f(proc_swdo) \ + f(proc_chea_usd) \ + f(proc_chea_free) \ + f(proc_cpu_us) \ + f(proc_cpu_sy) \ + f(proc_io_of) \ + f(proc_io_rd) \ + f(proc_io_wr) \ + f(proc_thr) \ + +#define DECLARE_VARIABLE(name) \ + static value_t _##name; + +ALL_VALUES_DO(DECLARE_VARIABLE) + +#undef DECLARE_VARIABLE + +public: + +#define DEFINE_GETTER(name) \ + static value_t name() { return _ ## name; } + +ALL_VALUES_DO(DEFINE_GETTER) + +#undef DEFINE_GETTER + + static void update_if_needed(); + + static bool initialize(); + +}; + +} // namespace sapmachine_vitals + +#endif // OS_LINUX_VITALS_LINUX_HELPERS_HPP diff --git a/src/hotspot/os/posix/malloctrace/mallocTracePosix.cpp b/src/hotspot/os/posix/malloctrace/mallocTracePosix.cpp new file mode 100644 index 00000000000..c3167c3944b --- /dev/null +++ b/src/hotspot/os/posix/malloctrace/mallocTracePosix.cpp @@ -0,0 +1,2649 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#if defined(LINUX) || defined(__APPLE__) + +#include "precompiled.hpp" + +#include "jvm_io.h" +#include "mallochooks.h" +#include "malloctrace/mallocTracePosix.hpp" + +#include "code/codeBlob.hpp" +#include "code/codeCache.hpp" +#include "runtime/arguments.hpp" +#include "runtime/atomic.hpp" +#include "runtime/interfaceSupport.inline.hpp" +#include "runtime/timer.hpp" +#include "utilities/ostream.hpp" +#include "utilities/powerOfTwo.hpp" +#include "utilities/ticks.hpp" + +#include +#include + +#if !defined(__APPLE__) +#include +#endif + +// To test in jtreg tests use +// JTREG="JAVA_OPTIONS=-XX:+UseMallocHooks -XX:+MallocTraceAtStartup -XX:MallocTraceDumpCount=10 -XX:MallocTraceDumpInterval=10s -XX:MallocTraceDumpDelay=10s -XX:MallocTraceDumpOutput=`pwd`/mtrace_@pid.txt -XX:ErrorFile=`pwd`/hs_err%p.log" + +// A simple smoke test +// jconsole -J-XX:+UseMallocHooks -J-XX:+MallocTraceAtStartup -J-XX:MallocTraceDumpCount=10 -J-XX:MallocTraceStackDepth=12 -J-XX:MallocTraceDumpInterval=10s -J-XX:MallocTraceDumpDelay=10s + + +// Some compile time constants for the maps. + +constexpr double MAX_STACK_MAP_LOAD = 0.5; +constexpr int STACK_MAP_INIT_SIZE = 1024; +static_assert(is_power_of_2(STACK_MAP_INIT_SIZE), "stack map size must be power of 2"); + +constexpr double MAX_ALLOC_MAP_LOAD = 0.5; +constexpr int ALLOC_MAP_INIT_SIZE = 1024; +static_assert(is_power_of_2(ALLOC_MAP_INIT_SIZE), "alloc map size must be power of 2"); + +constexpr int MAX_FRAMES = 31; +static_assert(is_power_of_2(MAX_FRAMES + 1), "max frames must be power of 2 minus 1"); + +// The number of top frames to skip. +constexpr int FRAMES_TO_SKIP = 0; + +constexpr int NR_OF_STACK_MAPS = 16; +static_assert(is_power_of_2(NR_OF_STACK_MAPS), "nr of stack maps must be power of 2"); + +constexpr int NR_OF_ALLOC_MAPS = 32; +static_assert(is_power_of_2(NR_OF_ALLOC_MAPS), "nr of alloc maps must be power of 2"); + +namespace sap { + +// The real allocation funcstions to use. This is be initialized later. +static real_malloc_funcs_t* real_malloc_funcs = nullptr; + +static bool is_non_empty_string(char const* str) { + return (str != nullptr) && (str[0] != '\0'); +} + +static uint64_t parse_timespan_part(char const* start, char const* end, char const** error) { + char buf[32]; + + // Strip trailing spaces. + while ((end > start) && (end[-1] == ' ')) { + end--; + } + + if (start == end) { + *error = "empty time"; + return 0; + } + + size_t size = (size_t) (end - start); + + if (size >= sizeof(buf)) { + *error = "time too long"; + return 0; + } + + memcpy(buf, start, size); + buf[size] = '\0'; + + char* found_end; + int64_t result = (int64_t) strtoll(buf, &found_end, 10); + + if ((found_end != end) && (*found_end != '\0')) { + *error = "Could not parse integer"; + } else if (result < 0) { + *error = "negative time"; + } + + return (uint64_t) result; +} + +static uint64_t parse_timespan(char const* spec, char const** error = nullptr) { + uint64_t result = 0; + char const* start = spec; + char const* pos = start; + char const* backup_error; + uint64_t limit_in_days = 365; + + if (error == nullptr) { + error = &backup_error; + } + + *error = nullptr; + + while (*pos != '\0') { + switch (*pos) { + case ' ': + if (pos == start) { + start++; + } + break; + + case 's': + result += parse_timespan_part(start, pos, error); + start = pos + 1; + break; + + case 'm': + result += 60 * parse_timespan_part(start, pos, error); + start = pos + 1; + break; + + case 'h': + result += 60 * 60 * parse_timespan_part(start, pos, error); + start = pos + 1; + break; + + case 'd': + result += 24 * 60 * 60 * parse_timespan_part(start, pos, error); + start = pos + 1; + break; + + default: + if ((*pos < '0') || (*pos > '9')) { + *error = "Unexpected character"; + return 0; + } + } + + pos++; + } + + if (pos != start) { + *error = "time without unit"; + } + + if (result / (24 * 60 * 60) > limit_in_days) { + *error = "time too large"; + } + + return result; +} + +// Keep sap namespace free from implementation classes. +namespace mallocStatImpl { + +// Allocates memory of the same size. It's pretty fast, but doesn't return +// free memory to the OS. +class Allocator { +private: + // We need padding, since we have arrays of this class used in parallel. + char _pre_pad[DEFAULT_CACHE_LINE_SIZE]; + size_t _allocation_size; + int _entries_per_chunk; + void** _chunks; + int _nr_of_chunks; + void** _free_list; + size_t _free_entries; + char _post_pad[DEFAULT_CACHE_LINE_SIZE]; + +public: + Allocator(size_t allocation_size, int entries_per_chunk); + ~Allocator(); + + void* allocate(); + void free(void* ptr); + size_t allocated(); + size_t unused(); +}; + +Allocator::Allocator(size_t allocation_size, int entries_per_chunk) : + _allocation_size(align_up(allocation_size, 8)), // We need no stricter alignment + _entries_per_chunk(entries_per_chunk), + _chunks(nullptr), + _nr_of_chunks(0), + _free_list(nullptr), + _free_entries(0) { +} + +Allocator::~Allocator() { + for (int i = 0; i < _nr_of_chunks; ++i) { + real_malloc_funcs->free(_chunks[i]); + } +} + +void* Allocator::allocate() { + if (_free_list != nullptr) { + void** result = _free_list; + _free_list = (void**) result[0]; + assert(_free_entries > 0, "free entries count invalid."); + _free_entries -= 1; + + return result; + } + + // We need a new chunk. + char* new_chunk = (char*) real_malloc_funcs->malloc(_entries_per_chunk * _allocation_size); + + if (new_chunk == nullptr) { + return nullptr; + } + + void** new_chunks = (void**) real_malloc_funcs->realloc(_chunks, sizeof(void**) * (_nr_of_chunks + 1)); + + if (new_chunks == nullptr) { + return nullptr; + } + + new_chunks[_nr_of_chunks] = new_chunk; + _nr_of_chunks += 1; + _chunks = new_chunks; + + for (int i = 0; i < _entries_per_chunk; ++i) { + free(new_chunk + i * _allocation_size); + } + + return allocate(); +} + +void Allocator::free(void* ptr) { + if (ptr != nullptr) { + void** as_array = (void**) ptr; + as_array[0] = (void*) _free_list; + _free_list = as_array; + _free_entries += 1; + } +} + +size_t Allocator::allocated() { + return _allocation_size * _entries_per_chunk * _nr_of_chunks; +} + +size_t Allocator::unused() { +#if defined(ASSERT) + size_t real_free_entries = 0; + void** entry = _free_list; + + while (entry != nullptr) { + real_free_entries += 1; + entry = (void**) entry[0]; + } + + assert(_free_entries == real_free_entries, "free entries inconsistent"); +#endif + + return _allocation_size * _free_entries; +} + +class AddressHashSet { +private: + int _mask; + int _count; + address* _set; + + int get_slot(address to_check); + +public: + AddressHashSet(bool enabled); + ~AddressHashSet(); + + bool contains(address to_check); + bool add(address to_add); + size_t allocated(); + // The average chain length. + double load(); +}; + +AddressHashSet::AddressHashSet(bool enabled) : + _mask(enabled ? 0 : 1), + _count(0), + _set(nullptr) { +} + +AddressHashSet::~AddressHashSet() { + real_malloc_funcs->free(_set); +} + +int AddressHashSet::get_slot(address to_check) { + assert(to_check != nullptr, "Invalid value"); + + if (_set == nullptr) { + // Initialize lazily. + if (_mask == 0) { + _mask = 8191; + _set = (address*) real_malloc_funcs->calloc(_mask + 1, sizeof(address)); + } + + // When we overflow, return treat each address as not be contained. This is + // the safe behaviour for our use case. + return -1; + } + + int slot = (int) (((uintptr_t) to_check) & _mask); + + while (_set[slot] != nullptr) { + if (_set[slot] == to_check) { + return slot; + } + + slot = (slot + 1) & _mask; + } + + return slot; +} + +bool AddressHashSet::contains(address to_check) { + int slot = get_slot(to_check); + + return (slot >= 0) && (_set[slot] != nullptr); +} + +bool AddressHashSet::add(address to_add) { + int slot = get_slot(to_add); + + if ((slot < 0) || (_set[slot] != nullptr)) { + // Already present. + return false; + } + + // Check if we should resize. + if (_count * 2 > _mask) { + address* old_set = _set; + int old_mask = _mask; + + _mask = _mask * 2 + 1; + _count = 0; + + _set = (address*) real_malloc_funcs->calloc(_mask + 1, sizeof(address)); + + // If full, we fall back to always return false. + if (_set == nullptr) { + real_malloc_funcs->free(old_set); + + return false; + } + + for (int i = 0; i <= old_mask; ++i) { + if (old_set[i] != nullptr) { + add(old_set[i]); + } + } + + real_malloc_funcs->free(old_set); + add(to_add); + } else { + _set[slot] = to_add; + _count += 1; + } + + return true; +} + +size_t AddressHashSet::allocated() { + if (_set == nullptr) { + return 0; + } + + return (_mask + 1) * sizeof(address); +} + +double AddressHashSet::load() { + if (_set == nullptr) { + return 0.0; + } + + return 1.0 * _count / (_mask + 1); +} + +class Locker : public StackObj { +private: + pthread_mutex_t* _mutex; + +public: + Locker(pthread_mutex_t* mutex, bool disabled = false); + ~Locker(); +}; + +Locker::Locker(pthread_mutex_t* lock, bool disabled) : + _mutex(disabled ? nullptr : lock) { + if ((_mutex != nullptr) && (pthread_mutex_lock(_mutex) != 0)) { + fatal("Could not lock mutex"); + } +} + +Locker::~Locker() { + if ((_mutex != nullptr) && (pthread_mutex_unlock(_mutex) != 0)) { + fatal("Could not unlock mutex"); + } +} + +// Entry for the hash map containing statistics about allocation stack traces. +class StatEntry { +private: + StatEntry* _next; + uint64_t _hash_and_nr_of_frames; + uint64_t _size; + uint64_t _count; + address _frames[]; + +public: + StatEntry(uint64_t hash, size_t size, int nr_of_frames, address* frames) : + _next(nullptr), + _hash_and_nr_of_frames((hash * (MAX_FRAMES + 1)) + nr_of_frames), + _size(size), + _count(1) { + assert(nr_of_frames >= 0, "Must not be negative"); + assert(nr_of_frames <= MAX_FRAMES, "too many frames"); + memcpy(_frames, frames, sizeof(address) * nr_of_frames); + assert(hash == this->hash(), "Must be the same: " UINT64_FORMAT " " UINT64_FORMAT, hash, this->hash()); + assert(nr_of_frames == this->nr_of_frames(), "Must be equal"); + + } + + uint64_t hash() { + return _hash_and_nr_of_frames / (MAX_FRAMES + 1); + } + + static int scaled_hash(uint64_t hash) { + return hash / NR_OF_STACK_MAPS; + } + + static size_t size(int frames) { + return sizeof(StatEntry) + sizeof(address) * frames; + } + + StatEntry* next() { + return _next; + } + + void set_next(StatEntry* next) { + _next = next; + } + + void add_allocation(size_t size) { + _size += size; + _count += 1; + } + + void remove_allocation(size_t size) { + assert(_size >= size, "Size cannot get negative (" UINT64_FORMAT " removed from " \ + UINT64_FORMAT ", count " UINT64_FORMAT ")", (uint64_t) size, _size, _count); + assert(_count >= 1, "Count cannot get negative"); + _size -= size; + _count -= 1; + } + + uint64_t size() { + return _size; + } + + uint64_t count() { + return _count; + } + + int nr_of_frames() { + return _hash_and_nr_of_frames & MAX_FRAMES; + } + + address* frames() { + return _frames; + } +}; + + +struct StatEntryCopy { + StatEntry* _entry; + uint64_t _size; + uint64_t _count; +}; + +// The entry for a single allocation. Note that we don't store the pointer itself +// but use the hash code instead. Our hash function is resersible, so this is OK. +class AllocEntry { +private: + uint64_t _hash; + StatEntry* _entry; + AllocEntry* _next; + DEBUG_ONLY(void* _ptr); // Is not really needed, but helps debugging. + +public: + + AllocEntry(uint64_t hash, StatEntry* entry, AllocEntry* next DEBUG_ONLY(COMMA void* ptr)) : + _hash(hash), + _entry(entry), + _next(next) + DEBUG_ONLY(COMMA _ptr(ptr)) { + } + + uint64_t hash() { + return _hash; + } + + static int scaled_hash(uint64_t hash) { + return hash / NR_OF_ALLOC_MAPS; + } + + StatEntry* entry() { + return _entry; + } + + AllocEntry* next() { + return _next; + } + + void set_next(AllocEntry* next) { + _next = next; + } + + AllocEntry** next_ptr() { + return &_next; + } + +#if defined(ASSERT) + void* ptr() { + return _ptr; + } +#endif +}; + + + +static register_hooks_t* register_hooks; +static get_real_malloc_funcs_t* get_real_malloc_funcs; + +#if defined(__APPLE__) +#define LD_PRELOAD "DYLD_INSERT_LIBRARIES" +#define LIB_MALLOC_HOOKS "libmallochooks.dylib" +#else +#define LD_PRELOAD "LD_PRELOAD" +#define LIB_MALLOC_HOOKS "libmallochooks.so" +#endif + +static void print_needed_preload_env(outputStream* st) { + st->print_cr("%s=%s/%s", LD_PRELOAD, Arguments::get_dll_dir(), LIB_MALLOC_HOOKS); + st->print_cr("Its current value is %s", getenv(LD_PRELOAD)); +} + +static void remove_malloc_hooks_from_env() { + char const* env = ::getenv(LD_PRELOAD); + + if ((env == nullptr) || (env[0] == '\0')) { + return; + } + + // Create a env with ':' prepended and appended. This makes the + // code easier. + stringStream guarded_env; + guarded_env.print(":%s:", env); + + stringStream new_env; + size_t len = strlen(LIB_MALLOC_HOOKS); + char const* base = guarded_env.base(); + char const* pos = base; + + while ((pos = strstr(pos, LIB_MALLOC_HOOKS)) != nullptr) { + if (pos[len] != ':') { + pos += 1; + + continue; + } + + if (pos[-1] == ':') { + new_env.print("%.*s%s", (int) (pos - base) - 1, base, pos + len); + } else if (pos[-1] == '/') { + char const* c = pos - 1; + + while (c[0] != ':') { + --c; + } + + new_env.print("%.*s%s", (int) (c - base + 1), base, pos + len + 1); + } else { + pos += 1; + + continue; + } + + if (new_env.size() <= 2) { + ::unsetenv(LD_PRELOAD); + } else { + stringStream ss; + ss.print("%.*s", MAX(0, (int) (new_env.size() - 2)), new_env.base() + 1); + ::setenv(LD_PRELOAD, ss.base(), 1); + } + + return; + } +} + +typedef int backtrace_func_t(void** stacks, int max_depth); + +template struct HashMapData { + char _front_padding[DEFAULT_CACHE_LINE_SIZE]; + Entry** _entries; + pthread_mutex_t _lock; + int _mask; + int _size; + int _limit; + Allocator* _alloc; + char _back_padding[DEFAULT_CACHE_LINE_SIZE]; + + HashMapData() : + _entries(nullptr), + _mask(0), + _size(0), + _limit(0), + _alloc(nullptr) { + } + + void resize(int new_mask, double max_load) { + assert(is_power_of_2(new_mask + 1), "Must be a power of 2 minus 1"); + + Entry** new_entries = (Entry**) real_malloc_funcs->calloc(new_mask + 1, sizeof(Entry*)); + Entry** old_entries = _entries; + + // Fail silently if we don't get the memory. + if (new_entries != nullptr) { + for (int i = 0; i <= _mask; ++i) { + Entry* entry = old_entries[i]; + + while (entry != nullptr) { + Entry* next_entry = entry->next(); + int slot = Entry::scaled_hash(entry->hash()) & new_mask; + entry->set_next(new_entries[slot]); + new_entries[slot] = entry; + entry = next_entry; + } + } + + _entries = new_entries; + _mask = new_mask; + _limit = (int) ((_mask + 1) * max_load); + real_malloc_funcs->free(old_entries); + } + } + + void cleanup() { + Locker locker(&_lock); + + if (_alloc != nullptr) { + _alloc->~Allocator(); + real_malloc_funcs->free(_alloc); + _alloc = nullptr; + } + + if (_entries != nullptr) { + real_malloc_funcs->free(_entries); + _entries = nullptr; + } + } +}; + +typedef HashMapData StackMapData; +typedef HashMapData AllocMapData; + +class MallocStatisticImpl : public AllStatic { +private: + + static backtrace_func_t* _backtrace; + static char const* _backtrace_name; + static bool _use_backtrace; + static volatile bool _initialized; + static bool _enabled; + static bool _shutdown; + static bool _track_free; + static bool _detailed_stats; + static bool _tried_to_load_backtrace; + static int _max_frames; + static registered_hooks_t _malloc_stat_hooks; + static pthread_mutex_t _malloc_stat_lock; + static bool _check_malloc_suspended; + static pthread_key_t _malloc_suspended; + static volatile int _enable_count; + + static StackMapData _stack_maps_data[NR_OF_STACK_MAPS]; + static AllocMapData _alloc_maps_data[NR_OF_ALLOC_MAPS]; + + static uint64_t _to_track_mask; + static uint64_t _to_track_limit; + + static volatile uint64_t _stack_walk_time; + static volatile uint64_t _stack_walk_count; + static volatile uint64_t _tracked_ptrs; + static volatile uint64_t _not_tracked_ptrs; + static volatile uint64_t _failed_frees; + + static void* _rainy_day_fund; + static registered_hooks_t _rainy_day_hooks; + static pthread_mutex_t _rainy_day_fund_lock; + static volatile bool _rainy_day_fund_used; + + static void set_malloc_suspended(bool suspended); + static bool malloc_suspended(); + + // The hooks. + static void* malloc_hook(size_t size, void* caller_address); + static void* calloc_hook(size_t elems, size_t size, void* caller_address); + static void* realloc_hook(void* ptr, size_t size, void* caller_address); + static void free_hook(void* ptr, void* caller_address); + static int posix_memalign_hook(void** ptr, size_t align, size_t size, void* caller_address); + static void* memalign_hook(size_t align, size_t size, void* caller_address); + static void* aligned_alloc_hook(size_t align, size_t size, void* caller_address); + static void* valloc_hook(size_t size, void* caller_address); + static void* pvalloc_hook(size_t size, void* caller_address); + + // The hooks used after we use the rainy day fund + static void* malloc_hook_rd(size_t size, void* caller_address); + static void* calloc_hook_rd(size_t elems, size_t size, void* caller_address); + static void* realloc_hook_rd(void* ptr, size_t size, void* caller_addresse); + static void free_hook_rd(void* ptr, void* caller_address); + static int posix_memalign_hook_rd(void** ptr, size_t align, size_t size, void* caller_address); + static void* memalign_hook_rd(size_t align, size_t size, void* caller_address); + static void* aligned_alloc_hook_rd(size_t align, size_t size, void* caller_address); + static void* valloc_hook_rd(size_t size, void* caller_address); + static void* pvalloc_hook_rd(size_t size, void* caller_address); + static void wait_for_rainy_day_fund(); + + static StatEntry* record_allocation_size(size_t to_add, int nr_of_frames, address* frames, + int* enable_count = nullptr); + static void record_allocation(void* ptr, uint64_t hash, int nr_of_frames, address* frames); + static StatEntry* record_free(void* ptr, uint64_t hash, size_t size); + + static uint64_t ptr_hash_impl(void* ptr); + static uint64_t ptr_hash(void* ptr); + static bool should_track(uint64_t hash); + static int capture_stack(address* frames, address real_func, address caller); + + static bool setup_hooks(registered_hooks_t* hooks, outputStream* st); + static void cleanup(); + + static bool dump_entry(outputStream* st, StatEntryCopy* entry, int index, + uint64_t total_size, uint64_t total_count, int total_entries, + char const* filter, AddressHashSet* filter_cache); + +public: + static void initialize(); + static bool rainy_day_fund_used(); + static bool enable(outputStream* st, TraceSpec const& spec); + static bool disable(outputStream* st); + static bool dump(outputStream* msg_stream, outputStream* dump_stream, DumpSpec const& spec); + static void shutdown(); +}; + +registered_hooks_t MallocStatisticImpl::_malloc_stat_hooks = { + MallocStatisticImpl::malloc_hook, + MallocStatisticImpl::calloc_hook, + MallocStatisticImpl::realloc_hook, + MallocStatisticImpl::free_hook, + MallocStatisticImpl::posix_memalign_hook, + MallocStatisticImpl::memalign_hook, + MallocStatisticImpl::aligned_alloc_hook, + MallocStatisticImpl::valloc_hook, + MallocStatisticImpl::pvalloc_hook +}; + +registered_hooks_t MallocStatisticImpl::_rainy_day_hooks = { + MallocStatisticImpl::malloc_hook_rd, + MallocStatisticImpl::calloc_hook_rd, + MallocStatisticImpl::realloc_hook_rd, + MallocStatisticImpl::free_hook_rd, + MallocStatisticImpl::posix_memalign_hook_rd, + MallocStatisticImpl::memalign_hook_rd, + MallocStatisticImpl::aligned_alloc_hook_rd, + MallocStatisticImpl::valloc_hook_rd, + MallocStatisticImpl::pvalloc_hook_rd +}; + +backtrace_func_t* MallocStatisticImpl::_backtrace; +char const* MallocStatisticImpl::_backtrace_name; +bool MallocStatisticImpl::_use_backtrace; +volatile bool MallocStatisticImpl::_initialized; +bool MallocStatisticImpl::_enabled; +bool MallocStatisticImpl::_shutdown; +bool MallocStatisticImpl::_track_free; +bool MallocStatisticImpl::_detailed_stats; +bool MallocStatisticImpl::_tried_to_load_backtrace; +int MallocStatisticImpl::_max_frames; +pthread_mutex_t MallocStatisticImpl::_malloc_stat_lock; +volatile int MallocStatisticImpl::_enable_count; +bool MallocStatisticImpl::_check_malloc_suspended; +pthread_key_t MallocStatisticImpl::_malloc_suspended; +StackMapData MallocStatisticImpl::_stack_maps_data[NR_OF_STACK_MAPS]; +AllocMapData MallocStatisticImpl::_alloc_maps_data[NR_OF_ALLOC_MAPS]; +uint64_t MallocStatisticImpl::_to_track_mask; +uint64_t MallocStatisticImpl::_to_track_limit; +volatile uint64_t MallocStatisticImpl::_stack_walk_time; +volatile uint64_t MallocStatisticImpl::_stack_walk_count; +volatile uint64_t MallocStatisticImpl::_tracked_ptrs; +volatile uint64_t MallocStatisticImpl::_not_tracked_ptrs; +volatile uint64_t MallocStatisticImpl::_failed_frees; +void* MallocStatisticImpl::_rainy_day_fund; +pthread_mutex_t MallocStatisticImpl::_rainy_day_fund_lock; +volatile bool MallocStatisticImpl::_rainy_day_fund_used; + +ALWAYSINLINE int MallocStatisticImpl::capture_stack(address* frames, address real_func, address caller) { + uint64_t ticks = _detailed_stats ? Ticks::now().nanoseconds() : 0; + int nr_of_frames = 0; + + if (_max_frames <= 2) { + // Skip, since we will fill it in later anyway. + } else if (_use_backtrace) { + nr_of_frames = _backtrace((void**) frames, _max_frames + FRAMES_TO_SKIP); + } else { + // We have to unblock SIGSEGV signal handling, since os::is_first_C_frame() + // calls SafeFetch, which needs the proper handling of SIGSEGV. + sigset_t curr, old; + sigemptyset(&curr); + sigaddset(&curr, SIGSEGV); + pthread_sigmask(SIG_UNBLOCK, &curr, &old); + frame fr = os::current_frame(); + + while (fr.pc() && nr_of_frames < _max_frames + FRAMES_TO_SKIP) { + frames[nr_of_frames] = fr.pc(); + nr_of_frames += 1; + + if (nr_of_frames >= _max_frames + FRAMES_TO_SKIP) { + break; + } + + if (fr.fp() == nullptr || fr.cb() != nullptr || fr.sender_pc() == nullptr || os::is_first_C_frame(&fr)) { + break; + } + + fr = os::get_sender_for_C_frame(&fr); + } + + pthread_sigmask(SIG_SETMASK, &old, nullptr); + } + + // We know at least the function and the caller. + if (nr_of_frames < 2) { + frames[0] = real_func; + frames[1] = caller; + nr_of_frames = MAX2(2, _max_frames); + } + + if (_detailed_stats) { + Atomic::add(&_stack_walk_time, Ticks::now().nanoseconds() - ticks); + Atomic::add(&_stack_walk_count, (uint64_t) 1); + } + + return nr_of_frames; +} + +static void after_child_fork() { + if (register_hooks != nullptr) { + register_hooks(nullptr); + } +} + + +bool MallocStatisticImpl::setup_hooks(registered_hooks_t* hooks, outputStream* st) { + if (register_hooks == nullptr) { + register_hooks = (register_hooks_t*) dlsym((void*) RTLD_DEFAULT, REGISTER_HOOKS_NAME); + get_real_malloc_funcs = (get_real_malloc_funcs_t*) dlsym((void*) RTLD_DEFAULT, + GET_REAL_MALLOC_FUNCS_NAME); + + if ((register_hooks == nullptr) || (get_real_malloc_funcs == nullptr)) { + if (UseMallocHooks) { + st->print_raw_cr("Could not find preloaded libmallochooks while -XX:+UseMallocHooks is set. " \ + "This usually happens if the VM is not loaded via the JDK launcher (e.g. " \ + "java.exe). In this case you must preload the library by setting the " \ + "following environment variable: "); + print_needed_preload_env(st); + } else { + st->print_cr("Could not find preloaded libmallochooks. Try using -XX:+UseMallocHooks " \ + "VM option to automatically preload it using the JDK launcher. Or you can set " \ + "the following environment variable: "); + print_needed_preload_env(st); + } + + st->print_raw_cr("VM arguments:"); + Arguments::print_summary_on(st); + + return false; + } + } + + real_malloc_funcs = get_real_malloc_funcs(); + register_hooks(hooks); + + return true; +} + +// Note that this function must be resersible. We +// rely on it having unique values for a pointer. +// See https://github.com/skeeto/hash-prospector?tab=readme-ov-file#reversible-operation-selection +// for a list of operations which are resersible. +uint64_t MallocStatisticImpl::ptr_hash_impl(void* ptr) { + uint64_t hash = (uint64_t) ptr; + hash = (~hash) + (hash << 21); + hash = hash ^ (hash >> 24); + hash = (hash + (hash << 3)) + (hash << 8); + hash = hash ^ (hash >> 14); + hash = (hash + (hash << 2)) + (hash << 4); + hash = hash ^ (hash >> 28); + hash = hash + (hash << 31); + + return hash; +} + +uint64_t MallocStatisticImpl::ptr_hash(void* ptr) { + if (!_track_free && (_to_track_mask == 0)) { + return 0; + } + + return ptr_hash_impl(ptr); +} + +bool MallocStatisticImpl::should_track(uint64_t hash) { + if (_detailed_stats) { + if ((hash & _to_track_mask) < _to_track_limit) { + Atomic::add(&_tracked_ptrs, (uint64_t) 1); + } else { + Atomic::add(&_not_tracked_ptrs, (uint64_t) 1); + } + } + + return (hash & _to_track_mask) < _to_track_limit; +} + +void MallocStatisticImpl::set_malloc_suspended(bool suspended) { + _check_malloc_suspended = suspended; + pthread_setspecific(_malloc_suspended, suspended ? (void*) 1 : nullptr); +} + +bool MallocStatisticImpl::malloc_suspended() { + return _check_malloc_suspended && (pthread_getspecific(_malloc_suspended) != nullptr); +} + +void* MallocStatisticImpl::malloc_hook(size_t size, void* caller_address) { + void* result = real_malloc_funcs->malloc(size); + uint64_t hash = ptr_hash(result); + + if ((result != nullptr) && should_track(hash) && !malloc_suspended()) { + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, (address) malloc, (address) caller_address); + + if (_track_free) { + record_allocation(result, hash, nr_of_frames, frames); + } else { + record_allocation_size(size, nr_of_frames, frames); + } + } + + return result; +} + +void* MallocStatisticImpl::calloc_hook(size_t elems, size_t size, void* caller_address) { + void* result = real_malloc_funcs->calloc(elems, size); + uint64_t hash = ptr_hash(result); + + if ((result != nullptr) && should_track(hash) && !malloc_suspended()) { + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, (address) calloc, (address) caller_address); + + if (_track_free) { + record_allocation(result, hash, nr_of_frames, frames); + } else { + record_allocation_size(elems * size, nr_of_frames, frames); + } + } + + return result; +} + +void* MallocStatisticImpl::realloc_hook(void* ptr, size_t size, void* caller_address) { + size_t old_size = ptr != nullptr ? real_malloc_funcs->malloc_size(ptr) : 0; + uint64_t old_hash = ptr_hash(ptr); + + // We have to speculate the realloc does not fail, since realloc itself frees + // the ptr potentially and another thread might get it from malloc and tries + // to add to the alloc hash map before we could remove it here. + StatEntry* freed_entry = nullptr; + + if (_track_free && (ptr != nullptr) && should_track(old_hash)) { + freed_entry = record_free(ptr, old_hash, old_size); + } + + void* result = real_malloc_funcs->realloc(ptr, size); + + if ((result == nullptr) && (freed_entry != nullptr) && (size > 0)) { + // We failed, but we already removed the freed memory, so we have to re-add it. + record_allocation(ptr, old_hash, freed_entry->nr_of_frames(), freed_entry->frames()); + + return nullptr; + } + + uint64_t hash = ptr_hash(result); + + if ((result != nullptr) && should_track(hash) && !malloc_suspended()) { + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, (address) realloc, (address) caller_address); + + if (_track_free) { + record_allocation(result, hash, nr_of_frames, frames); + } else if (old_size < size) { + // Track the additional allocate bytes. This is somewhat wrong, since + // we don't know the requested size of the original allocation and + // old_size might be greater. + record_allocation_size(size - old_size, nr_of_frames, frames); + } + } + + return result; +} + +void MallocStatisticImpl::free_hook(void* ptr, void* caller_address) { + if ((ptr != nullptr) && _track_free) { + uint64_t hash = ptr_hash(ptr); + + if (should_track(hash)) { + record_free(ptr, hash, real_malloc_funcs->malloc_size(ptr)); + } + } + + real_malloc_funcs->free(ptr); +} + +int MallocStatisticImpl::posix_memalign_hook(void** ptr, size_t align, size_t size, void* caller_address) { + int result = real_malloc_funcs->posix_memalign(ptr, align, size); + uint64_t hash = ptr_hash(*ptr); + + if ((result == 0) && should_track(hash) && !malloc_suspended()) { + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, (address) posix_memalign, (address) caller_address); + + if (_track_free) { + record_allocation(*ptr, hash, nr_of_frames, frames); + } else { + // Here we track the really allocated size, since it might be very different + // from the requested one. + record_allocation_size(real_malloc_funcs->malloc_size(*ptr), nr_of_frames, frames); + } + } + + return result; +} + +void* MallocStatisticImpl::memalign_hook(size_t align, size_t size, void* caller_address) { + void* result = real_malloc_funcs->memalign(align, size); + uint64_t hash = ptr_hash(result); +#if !defined(__APPLE__) + address real_func = (address) memalign; +#else + address real_func = (address) memalign_hook; +#endif + + if ((result != nullptr) && should_track(hash) && !malloc_suspended()) { + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, real_func, (address) caller_address); + + if (_track_free) { + record_allocation(result, hash, nr_of_frames, frames); + } else { + // Here we track the really allocated size, since it might be very different + // from the requested one. + record_allocation_size(real_malloc_funcs->malloc_size(result), nr_of_frames, frames); + } + } + + return result; +} + +void* MallocStatisticImpl::aligned_alloc_hook(size_t align, size_t size, void* caller_address) { + void* result = real_malloc_funcs->aligned_alloc(align, size); + uint64_t hash = ptr_hash(result); +#if !defined(__APPLE__) + address real_func = (address) aligned_alloc; +#else + address real_func = (address) aligned_alloc_hook; +#endif + + if ((result != nullptr) && should_track(hash) && !malloc_suspended()) { + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, real_func, (address) caller_address); + + if (_track_free) { + record_allocation(result, hash, nr_of_frames, frames); + } else { + // Here we track the really allocated size, since it might be very different + // from the requested one. + record_allocation_size(real_malloc_funcs->malloc_size(result), nr_of_frames, frames); + } + } + + return result; +} + +void* MallocStatisticImpl::valloc_hook(size_t size, void* caller_address) { + void* result = real_malloc_funcs->valloc(size); + uint64_t hash = ptr_hash(result); +#if defined(__GLIBC__) || defined(__APPLE__) + address real_func = (address) valloc; +#else + address real_func = (address) valloc_hook; +#endif + + if ((result != nullptr) && should_track(hash) && !malloc_suspended()) { + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, real_func, (address) caller_address); + + if (_track_free) { + record_allocation(result, hash, nr_of_frames, frames); + } else { + // Here we track the really allocated size, since it might be very different + // from the requested one. + record_allocation_size(real_malloc_funcs->malloc_size(result), nr_of_frames, frames); + } + } + + return result; +} + +void* MallocStatisticImpl::pvalloc_hook(size_t size, void* caller_address) { + void* result = real_malloc_funcs->pvalloc(size); + uint64_t hash = ptr_hash(result); +#if defined(__GLIBC__) + address real_func = (address) pvalloc; +#else + address real_func = (address) pvalloc_hook; +#endif + + if ((result != nullptr) && should_track(hash) && !malloc_suspended()) { + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, real_func, (address) caller_address); + + if (_track_free) { + record_allocation(result, hash, nr_of_frames, frames); + } else { + // Here we track the really allocated size, since it might be very different + // from the requested one. + record_allocation_size(real_malloc_funcs->malloc_size(result), nr_of_frames, frames); + } + } + + return result; +} + + +void* MallocStatisticImpl::malloc_hook_rd(size_t size, void* caller_address) { + wait_for_rainy_day_fund(); + + return real_malloc_funcs->malloc(size); +} + +void* MallocStatisticImpl::calloc_hook_rd(size_t elems, size_t size, void* caller_address) { + wait_for_rainy_day_fund(); + + return real_malloc_funcs->calloc(elems, size); +} + +void* MallocStatisticImpl::realloc_hook_rd(void* ptr, size_t size, void* caller_address) { + wait_for_rainy_day_fund(); + + return real_malloc_funcs->realloc(ptr, size); +} + +void MallocStatisticImpl::free_hook_rd(void* ptr, void* caller_address) { + wait_for_rainy_day_fund(); + + real_malloc_funcs->free(ptr); +} + +int MallocStatisticImpl::posix_memalign_hook_rd(void** ptr, size_t align, size_t size, void* caller_address) { + wait_for_rainy_day_fund(); + + return real_malloc_funcs->posix_memalign(ptr, align, size); +} + +void* MallocStatisticImpl::memalign_hook_rd(size_t align, size_t size, void* caller_address) { + wait_for_rainy_day_fund(); + + return real_malloc_funcs->memalign(align, size); +} + +void* MallocStatisticImpl::aligned_alloc_hook_rd(size_t align, size_t size, void* caller_address) { + wait_for_rainy_day_fund(); + + return real_malloc_funcs->aligned_alloc(align, size); +} + +void* MallocStatisticImpl::valloc_hook_rd(size_t size, void* caller_address) { + wait_for_rainy_day_fund(); + + return real_malloc_funcs->valloc(size); +} + +void* MallocStatisticImpl::pvalloc_hook_rd(size_t size, void* caller_address) { + wait_for_rainy_day_fund(); + + return real_malloc_funcs->pvalloc(size); +} + +void MallocStatisticImpl::wait_for_rainy_day_fund() { + Locker locker(&_rainy_day_fund_lock); +} + +static bool is_same_stack(StatEntry* to_check, int nr_of_frames, address* frames) { + for (int i = 0; i < nr_of_frames; ++i) { + if (to_check->frames()[i] != frames[i]) { + return false; + } + } + + return true; +} + +static uint64_t hash_for_frames(int nr_of_frames, address* frames) { + uint64_t result = 0; + + for (int i = 0; i < nr_of_frames; ++i) { + uint64_t frame_addr = (uint64_t) (intptr_t) frames[i]; + result = result * 31 + ((frame_addr & 0xfffffff0) >> 4) LP64_ONLY(+ 127 * (frame_addr >> 36)); + } + + // Avoid more bits than we can store in the entry. + return result & (((uint64_t) UINT64_MAX) / (MAX_FRAMES + 1)); +} + +StatEntry* MallocStatisticImpl::record_allocation_size(size_t to_add, int nr_of_frames, address* frames, + int* enable_count) { + // Skip the top frame since it is always from the hooks. + nr_of_frames = MAX2(nr_of_frames - FRAMES_TO_SKIP, 0); + frames += FRAMES_TO_SKIP; + + assert(nr_of_frames <= _max_frames, "Overflow"); + + uint64_t hash = hash_for_frames(nr_of_frames, frames); + int idx = hash & (NR_OF_STACK_MAPS - 1); + assert((idx >= 0) && (idx < NR_OF_STACK_MAPS), "invalid map index"); + + StackMapData& map = _stack_maps_data[idx]; + Locker locker(&map._lock); + + if (enable_count != nullptr) { + *enable_count = _enable_count; + } + + if (!_enabled) { + return nullptr; + } + + int slot = StatEntry::scaled_hash(hash) & map._mask; + assert((slot >= 0) || (slot <= map._mask), "Invalid slot"); + StatEntry* to_check = map._entries[slot]; + + // Check if we already know this stack. + while (to_check != nullptr) { + if ((to_check->hash() == hash) && (to_check->nr_of_frames() == nr_of_frames)) { + if (is_same_stack(to_check, nr_of_frames, frames)) { + to_check->add_allocation(to_add); + + return to_check; + } + } + + to_check = to_check->next(); + } + + // Need a new entry. Fail silently if we don't get the memory. + void* mem = map._alloc->allocate(); + + if (mem != nullptr) { + StatEntry* entry = new (mem) StatEntry(hash, to_add, nr_of_frames, frames); + entry->set_next(map._entries[slot]); + map._entries[slot] = entry; + map._size += 1; + + if (map._size > map._limit) { + _stack_maps_data[idx].resize(map._mask * 2 + 1, MAX_STACK_MAP_LOAD); + } + + return entry; + } + + return nullptr; +} + +void MallocStatisticImpl::record_allocation(void* ptr, uint64_t hash, int nr_of_frames, address* frames) { + // Use the size that the malloc implementation used, since we don't store + // the size and have to account for it later in realloc/free. + size_t size = real_malloc_funcs->malloc_size(ptr); + int enable_count; + + StatEntry* stat_entry = record_allocation_size(size, nr_of_frames, frames, &enable_count); + + if (stat_entry == nullptr) { + return; + } + + // hash could be 0 since ptr_hash checked for _track_free without + // lock protection. Recalculate it again. + if (hash == 0) { + hash = ptr_hash_impl(ptr); + } + + int idx = (int) (hash & (NR_OF_ALLOC_MAPS - 1)); + AllocMapData& map = _alloc_maps_data[idx]; + Locker locker(&map._lock); + + // _track_free could have changed concurrently. + if (!(_track_free && _enabled)) { + return; + } + + // We might have enable the trace again after we created the stat + // entry, so if that happened, we bail out. + if (enable_count != _enable_count) { + return; + } + + int slot = AllocEntry::scaled_hash(hash) & map._mask; + + // Should not already be in the table, since this is the pointer to a newly allocated + // piece of memory, so we remove the check in the optimized version. +#ifdef ASSERT + AllocEntry* entry = map._entries[slot]; + + while (entry != nullptr) { + if (entry->hash() == hash) { + char tmp[1024]; + set_malloc_suspended(true); + shutdown(); + + address frames[MAX_FRAMES + FRAMES_TO_SKIP]; + int nr_of_frames = capture_stack(frames, (address) nullptr, (address) nullptr); + + fdStream ss(1); + ss.print_cr("Same hash " UINT64_FORMAT " for %p and %p", (uint64_t) hash, ptr, entry->ptr()); + ss.print_raw_cr("Current stack:"); + + for (int i = 0; i < nr_of_frames; ++i) { + ss.print(" [" PTR_FORMAT "] ", p2i(frames[i])); + os::print_function_and_library_name(&ss, frames[i], tmp, sizeof(tmp), true, true, false); + ss.cr(); + } + + ss.print_raw_cr("Original stack:"); + StatEntry* stat_entry = entry->entry(); + + for (int i = 0; i < stat_entry->nr_of_frames(); ++i) { + address frame = stat_entry->frames()[i]; + ss.print(" [" PTR_FORMAT "] ", p2i(frame)); + + if (os::print_function_and_library_name(&ss, frame, tmp, sizeof(tmp), true, true, false)) { + ss.cr(); + } else { + CodeBlob* blob = CodeCache::find_blob((void*) frame); + + if (blob != nullptr) { + ss.print_raw(" "); + blob->print_value_on(&ss); + ss.cr(); + } else { + ss.print_raw_cr(" "); + } + } + } + } + + assert((entry->hash() != hash) || (ptr == entry->ptr()), "Same hash for different pointer"); + assert(entry->hash() != hash, "Must not be already present"); + entry = entry->next(); + } +#endif + + void* mem = map._alloc->allocate(); + + if (mem != nullptr) { +#if defined(ASSERT) + AllocEntry* entry = new (mem) AllocEntry(hash, stat_entry, map._entries[slot], ptr); +#else + AllocEntry* entry = new (mem) AllocEntry(hash, stat_entry, map._entries[slot]); +#endif + map._entries[slot] = entry; + map._size += 1; + + if (map._size > map._limit) { + _alloc_maps_data[idx].resize(map._mask * 2 + 1, MAX_ALLOC_MAP_LOAD); + } + } +} + +StatEntry* MallocStatisticImpl::record_free(void* ptr, uint64_t hash, size_t size) { + // hash could be 0 since ptr_hash checked for _track_free without + // lock protection. Recalculate it again. + if (hash == 0) { + hash = ptr_hash_impl(ptr); + } + + int idx = (int) (hash & (NR_OF_ALLOC_MAPS - 1)); + AllocMapData& map = _alloc_maps_data[idx]; + Locker locker(&map._lock); + + // _track_free could have changed concurrently. + if (!(_track_free && _enabled)) { + return nullptr; + } + + assert(map._entries != nullptr, "Must be initialized"); + + int slot = (hash / NR_OF_ALLOC_MAPS) & map._mask; + int enable_count = _enable_count; + AllocEntry** entry = &map._entries[slot]; + + while (*entry != nullptr) { + if ((*entry)->hash() == hash) { + StatEntry* stat_entry = (*entry)->entry(); + assert((*entry)->ptr() == ptr, "Same hash must be same pointer"); + AllocEntry* next = (*entry)->next(); + map._alloc->free(*entry); + map._size -= 1; + *entry = next; + + // Should not be in the table anymore. +#ifdef ASSERT + AllocEntry* to_check = map._entries[slot]; + + while (to_check != nullptr) { + assert(to_check->hash() != hash, "Must not be already present"); + to_check = to_check->next(); + } +#endif + + // We need to lock the stat table containing the entry to avoid + // races when changing the size and count fields. + int idx2 = (int) (stat_entry->hash() & (NR_OF_STACK_MAPS - 1)); + Locker locker2(&_stack_maps_data[idx2]._lock); + stat_entry->remove_allocation(size); + + return stat_entry; + } + + entry = (*entry)->next_ptr(); + } + + // We missed an allocation. This is fine, since we might have enabled the + // trace after the allocation itself (or it might be a bug in the progam, + // but we can't be sure). + if (_detailed_stats) { + Atomic::add(&_failed_frees, (uint64_t) 1); + } + + return nullptr; +} + +void MallocStatisticImpl::cleanup() { + _enable_count += 1; + + // Cleanup alloc map first, to avoid having dangling pointers + // to stat entries. + for (int i = 0; i < NR_OF_ALLOC_MAPS; ++i) { + _alloc_maps_data[i].cleanup(); + } + + for (int i = 0; i < NR_OF_STACK_MAPS; ++i) { + _stack_maps_data[i].cleanup(); + } + + _enable_count += 1; + + if (real_malloc_funcs != nullptr) { + real_malloc_funcs->free(_rainy_day_fund); + _rainy_day_fund = nullptr; + } +} + +void MallocStatisticImpl::initialize() { + if (_initialized) { + return; + } + + _initialized = true; + + if (pthread_mutex_init(&_malloc_stat_lock, nullptr) != 0) { + fatal("Could not initialize malloc stat lock"); + } + + pthread_mutexattr_t attr; + + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + + if (pthread_mutex_init(&_rainy_day_fund_lock, &attr) != 0) { + fatal("Could not initialize rainy day fund lock"); + } + + if (pthread_key_create(&_malloc_suspended, nullptr) != 0) { + fatal("Could not initialize malloc suspend key"); + } + + for (int i = 0; i < NR_OF_STACK_MAPS; ++i) { + if (pthread_mutex_init(&_stack_maps_data[i]._lock, nullptr) != 0) { + fatal("Could not initialize stack maps lock"); + } + } + + for (int i = 0; i < NR_OF_ALLOC_MAPS; ++i) { + if (pthread_mutex_init(&_alloc_maps_data[i]._lock, nullptr) != 0) { + fatal("Could not initialize alloc maps lock"); + } + } +} + +bool MallocStatisticImpl::rainy_day_fund_used() { + return _rainy_day_fund_used; +} + +bool MallocStatisticImpl::enable(outputStream* st, TraceSpec const& spec) { + Locker lock(&_malloc_stat_lock); + + if (_enabled) { + if (spec._force) { + _enabled = false; + setup_hooks(nullptr, st); + cleanup(); + + st->print_raw_cr("Disabled already running trace first."); + } else { + st->print_raw_cr("Malloc statistic is already enabled!"); + + return false; + } + } + + if (_shutdown) { + st->print_raw_cr("Malloc statistic is already shut down!"); + + return false; + } + + if (spec._stack_depth < 2 || spec._stack_depth > MAX_FRAMES) { + st->print_cr("The given stack depth %d is outside of the valid range [%d, %d]", + spec._stack_depth, 2, MAX_FRAMES); + + return false; + } + + // Get the backtrace function if needed. + if (spec._use_backtrace && !_tried_to_load_backtrace) { + _tried_to_load_backtrace = true; + +#if defined(__APPLE__) + // Try libunwind first on mac. + _backtrace = (backtrace_func_t*) dlsym(RTLD_DEFAULT, "unw_backtrace"); + _backtrace_name = "backtrace (libunwind)"; + + if (_backtrace == nullptr) { + _backtrace = (backtrace_func_t*) dlsym(RTLD_DEFAULT, "backtrace"); + _backtrace_name = "backtrace"; + } +#else + _backtrace = (backtrace_func_t*) dlsym(RTLD_DEFAULT, "backtrace"); + _backtrace_name = "backtrace"; + + if (_backtrace == nullptr) { + // Try if we have libunwind installed. + char ebuf[512]; + void* libunwind = os::dll_load(MallocTraceUnwindLibName, ebuf, sizeof ebuf); + + if (libunwind != nullptr) { + _backtrace = (backtrace_func_t*) dlsym(libunwind, "unw_backtrace"); + _backtrace_name = "backtrace (libunwind)"; + } + } +#endif + + // Clear dlerror. + dlerror(); + + if (_backtrace != nullptr) { + // Trigger initialization needed. + void* tmp[1]; + _backtrace(tmp, 1); + } + } + + _track_free = spec._track_free; + _detailed_stats = spec._detailed_stats; + + if (_track_free) { + st->print_raw_cr("Tracking live memory."); + } else { + st->print_raw_cr("Tracking all allocated memory."); + } + + if (_detailed_stats) { + st->print_raw_cr("Collecting detailed statistics."); + } + + int only_nth = MIN2(1000, MAX2(1, spec._only_nth)); + + if (only_nth > 1) { + uint64_t pow = ((uint64_t) 1) << 42; + _to_track_limit = pow / only_nth; + _to_track_mask = pow - 1; + + st->print_cr("Tracking about every %d allocations (%d / %d).", only_nth, (int) _to_track_mask, (int) _to_track_limit); + } else { + _to_track_mask = 0; + _to_track_limit = 1; + } + + _use_backtrace = spec._use_backtrace && (_backtrace != nullptr); + + // Reset statistic counters. + _stack_walk_time = 0; + _stack_walk_count = 0; + _tracked_ptrs = 0; + _not_tracked_ptrs = 0; + _failed_frees = 0; + + if (_use_backtrace && spec._use_backtrace) { + st->print_raw_cr("Using backtrace() to sample stacks."); + } else if (spec._use_backtrace) { + st->print_raw_cr("Using fallback mechanism to sample stacks, since backtrace() was not available."); + } else { + st->print_raw_cr("Using fallback mechanism to sample stacks."); + } + + _max_frames = spec._stack_depth; + + if (!setup_hooks(&_malloc_stat_hooks, st)) { + return false; + } + + // Never set _funcs to nullptr, even if we fail. It's just safer that way. + size_t entry_size = StatEntry::size(_max_frames); + + if (spec._rainy_day_fund > 0) { + _rainy_day_fund = real_malloc_funcs->malloc(spec._rainy_day_fund); + + if (_rainy_day_fund == nullptr) { + st->print_cr("Could not allocate rainy day fund of %d bytes", spec._rainy_day_fund); + cleanup(); + + return false; + } + } + + for (int i = 0; i < NR_OF_STACK_MAPS; ++i) { + void* mem = real_malloc_funcs->malloc(sizeof(Allocator)); + + if (mem == nullptr) { + st->print_raw_cr("Could not allocate the allocator!"); + cleanup(); + + return false; + } + + StackMapData& map = _stack_maps_data[i]; + Locker locker(&map._lock); + map._alloc = new (mem) Allocator(entry_size, 256); + map._mask = STACK_MAP_INIT_SIZE - 1; + map._size = 0; + map._limit = (int) ((map._mask + 1) * MAX_STACK_MAP_LOAD); + map._entries = (StatEntry**) real_malloc_funcs->calloc(map._mask + 1, sizeof(StatEntry*)); + + if (map._entries == nullptr) { + st->print_raw_cr("Could not allocate the stack map!"); + cleanup(); + + return false; + } + } + + for (int i = 0; i < NR_OF_ALLOC_MAPS; ++i) { + void* mem = real_malloc_funcs->malloc(sizeof(Allocator)); + + if (mem == nullptr) { + st->print_raw_cr("Could not allocate the allocator!"); + cleanup(); + + return false; + } + + AllocMapData& map = _alloc_maps_data[i]; + Locker locker(&map._lock); + map._alloc = new (mem) Allocator(sizeof(AllocEntry), 2048); + map._mask = ALLOC_MAP_INIT_SIZE - 1; + map._size = 0; + map._limit = (int) ((map._mask + 1) * MAX_ALLOC_MAP_LOAD); + map._entries = (AllocEntry**) real_malloc_funcs->calloc(map._mask + 1, sizeof(AllocEntry*)); + + if (map._entries == nullptr) { + st->print_raw_cr("Could not allocate the alloc map!"); + cleanup(); + + return false; + } + } + + _enabled = true; + return true; +} + +bool MallocStatisticImpl::disable(outputStream* st) { + Locker lock(&_malloc_stat_lock); + + if (!_enabled) { + if (st != nullptr) { + st->print_raw_cr("Malloc statistic is already disabled!"); + } + + return false; + } + + _enabled = false; + setup_hooks(nullptr, st); + cleanup(); + + return true; +} + + +static char const* mem_prefix[] = {"k", "M", "G", "T", nullptr}; + +static void print_percentage(outputStream* st, double f) { + if (f <= 0) { + st->print("0.00 %%"); + } else if (f < 0.01) { + st->print("< 0.01 %%"); + } else if (f < 10) { + st->print("%.2f %%", f); + } else { + st->print("%.1f %%", f); + } +} + +static void print_mem(outputStream* st, uint64_t mem, uint64_t total = 0) { + uint64_t k = 1024; + double perc = 0.0; + if (total > 0) { + perc = 100.0 * mem / total; + } + + if ((int64_t) mem < 0) { + mem = -((int64_t) mem); + st->print("*neg* "); + } + + if (mem < 1000) { + if (total > 0) { + st->print("%'" PRId64 " (", (uint64_t) mem); + print_percentage(st, perc); + st->print_raw(")"); + } else { + st->print("%'" PRId64, (uint64_t) mem); + } + } else { + int idx =0; + uint64_t curr = mem; + double f = 1.0 / k; + + while (mem_prefix[idx] != nullptr) { + if (curr < 1000 * k) { + if (curr < 100 * k) { + if (total > 0) { + st->print("%'" PRId64 " (%.1f %s, ", mem, f * curr, mem_prefix[idx]); + print_percentage(st, perc); + st->print_raw(")"); + } else { + st->print("%'" PRId64 " (%.1f %s)", mem, f * curr, mem_prefix[idx]); + } + } else { + if (total > 0) { + st->print("%'" PRId64 " (%d %s, ", mem, (int) (curr / k), mem_prefix[idx]); + print_percentage(st, perc); + st->print_raw(")"); + } else { + st->print("%'" PRId64 " (%d %s)", mem, (int) (curr / k), mem_prefix[idx]); + } + } + + return; + } + + curr /= k; + idx += 1; + } + + st->print("%'" PRId64 " (%'" PRId64 "%s)", mem, curr, mem_prefix[idx - 1]); + } +} + +static void print_count(outputStream* st, uint64_t count, uint64_t total = 0) { + st->print("%'" PRId64, (int64_t) count); + + if (total > 0) { + double perc = 100.0 * count / total; + + st->print_raw(" ("); + print_percentage(st, perc); + st->print_raw(")"); + } +} + +static void print_frame(outputStream* st, address frame) { + char tmp[256]; + + if (os::print_function_and_library_name(st, frame, tmp, sizeof(tmp), true, true, false)) { + st->cr(); + } else { + CodeBlob* blob = CodeCache::find_blob((void*) frame); + + if (blob != nullptr) { + st->print_raw(" "); + blob->print_value_on(st); + } else { + st->print_raw_cr(" "); + } + } +} + +bool MallocStatisticImpl::dump_entry(outputStream* st, StatEntryCopy* entry, int index, + uint64_t total_size, uint64_t total_count, int total_entries, + char const* filter, AddressHashSet* filter_cache) { + // Use a temp buffer since the output stream might use unbuffered I/O. + char ss_tmp[4096]; + stringStream ss(ss_tmp, sizeof(ss_tmp)); + + // Check if we should print this stack. + if (is_non_empty_string(filter)) { + bool found = false; + + for (int i = 0; i < entry->_entry->nr_of_frames(); ++i) { + address frame = entry->_entry->frames()[i]; + + if (filter_cache->contains(frame)) { + continue; + } + + print_frame(&ss, frame); + + if (strstr(ss.base(), filter) != nullptr) { + found = true; + ss.reset(); + + break; + } else { + filter_cache->add(frame); + } + + ss.reset(); + } + + if (!found) { + return false; + } + } + + // We use int64_t here to easy see if values got negative (instead of seeing + // an insanely large number). + ss.print("Stack %d of %d: ", index, total_entries); + print_mem(&ss, entry->_size, total_size); + ss.print_raw(" bytes, "); + print_count(&ss, entry->_count, total_count); + ss.print_cr(" allocations"); + + for (int i = 0; i < entry->_entry->nr_of_frames(); ++i) { + address frame = entry->_entry->frames()[i]; + ss.print(" [" PTR_FORMAT "] ", p2i(frame)); + print_frame(&ss, frame); + + // Flush the temp buffer if we are near the end. + if (sizeof(ss_tmp) - ss.size() < 512) { + st->write(ss_tmp, ss.size()); + ss.reset(); + } + } + + if (entry->_entry->nr_of_frames() == 0) { + ss.print_raw_cr(" "); + } + + st->write(ss_tmp, ss.size()); + + return true; +} + +static void print_allocation_stats(outputStream* st, HashMapData* data, + int nr_of_maps, char const* type) { + uint64_t allocated = 0; + uint64_t unused = 0; + uint64_t total_entries = 0; + uint64_t total_slots = 0; + + for (int i = 0; i < nr_of_maps; ++i) { + Locker lock(&data[i]._lock); + allocated += (data[i]._mask + 1) * sizeof(void*); + total_entries += data[i]._size; + total_slots += data[i]._mask + 1; + allocated += data[i]._alloc->allocated(); + unused += data[i]._alloc->unused(); + } + + st->cr(); + st->print_cr("Statistic for %s:", type); + st->print_raw("Allocated memory: "); + print_mem(st, allocated); + st->cr(); + st->print_raw("Unused memory : "); + print_mem(st, unused); + st->cr(); + st->print_cr("Average load : %.2f", total_entries / (double) total_slots); + st->print_cr("Nr. of entries : %'" PRId64, total_entries); +} + +static int sort_by_size(const void* p1, const void* p2) { + StatEntryCopy* e1 = (StatEntryCopy*) p1; + StatEntryCopy* e2 = (StatEntryCopy*) p2; + + if (e1->_size > e2->_size) { + return -1; + } + + if (e1->_size < e2->_size) { + return 1; + } + + // For consistent sorting. + if (e1->_entry < e2->_entry) { + return -1; + } + + return 1; +} + +static int sort_by_count(const void* p1, const void* p2) { + StatEntryCopy* e1 = (StatEntryCopy*) p1; + StatEntryCopy* e2 = (StatEntryCopy*) p2; + + if (e1->_count > e2->_count) { + return -1; + } + + if (e1->_count < e2->_count) { + return 1; + } + + // For consistent sorting. + if (e1->_entry < e2->_entry) { + return -1; + } + + return 1; +} + +bool MallocStatisticImpl::dump(outputStream* msg_stream, outputStream* dump_stream, DumpSpec const& spec) { + bool used_rainy_day_fund = false; + + if (spec._on_error) { + if (_initialized) { + // Make sure all other threads don't allocate memory anymore + if (Atomic::cmpxchg(&_rainy_day_fund_used, false, true) == true) { + // Only can be done once. + return false; + } + + used_rainy_day_fund = true; + } else { + return false; + } + } + + Locker locker(&_rainy_day_fund_lock, !used_rainy_day_fund); + + if (used_rainy_day_fund) { + setup_hooks(&_rainy_day_hooks, nullptr); + + // Free rainy day fund so we have some memory to use. + real_malloc_funcs->free(_rainy_day_fund); + _rainy_day_fund = nullptr; + msg_stream->print_raw_cr("Emergency dump of malloc trace statistic ..."); + } + + // We need to avoid having the trace disabled concurrently. + Locker lock(&_malloc_stat_lock, spec._on_error); + + if (!_enabled) { + msg_stream->print_raw_cr("Malloc statistic not enabled!"); + + return false; + } + + // Hide allocations done by this thread during dumping if requested. + // Note that we always track frees or we might end up trying to add + // an allocation with a pointer which is already in the alloc maps. + set_malloc_suspended(spec._hide_dump_allocs); + + if (_backtrace != nullptr) { + dump_stream->print_cr("Stacks were collected via %s.", _backtrace_name); + } else { + dump_stream->print_cr("Stacks were collected via the fallback mechanism."); + } + + if (_track_free) { + dump_stream->print_raw_cr("Contains the currently allocated memory since enabling."); + } else { + dump_stream->print_raw_cr("Contains every allocation done since enabling."); + } + + bool uses_filter = is_non_empty_string(spec._filter); + + if (uses_filter) { + dump_stream->print_cr("Only printing stacks in which frames contain '%s'.", spec._filter); + } + + // We make a copy of each hash map, since we don't want to lock for the whole operation. + StatEntryCopy* entries[NR_OF_STACK_MAPS]; + int nr_of_entries[NR_OF_STACK_MAPS]; + + bool failed_alloc = false; + uint64_t total_count = 0; + uint64_t total_size = 0; + int total_entries = 0; + int total_non_empty_entries = 0; + int max_entries = MAX2(1, spec._dump_percentage > 0 ? INT_MAX : spec._max_entries); + int max_printed_entries = max_entries; + + if (uses_filter) { + max_entries = INT_MAX; + } + + elapsedTimer totalTime; + elapsedTimer lockedTime; + + totalTime.start(); + + for (int idx = 0; idx < NR_OF_STACK_MAPS; ++idx) { + int pos = 0; + int expected_size; + + { + StackMapData& map = _stack_maps_data[idx]; + Locker locker(&map._lock); + lockedTime.start(); + expected_size = map._size; + + entries[idx] = (StatEntryCopy*) real_malloc_funcs->malloc(sizeof(StatEntryCopy) * expected_size); + + if (entries[idx] != nullptr) { + StatEntry** orig = map._entries; + StatEntryCopy* copies = entries[idx]; + int nr_of_slots = map._mask + 1; + + for (int slot = 0; slot < nr_of_slots; ++slot) { + StatEntry* entry = orig[slot]; + + while (entry != nullptr) { + assert(pos < expected_size, "To many entries"); + + if (entry->count() > 0) { + copies[pos]._entry = entry; + copies[pos]._size = entry->size(); + copies[pos]._count = entry->count(); + + total_size += entry->size(); + total_count += entry->count(); + + pos += 1; + } + + entry = entry->next(); + } + } + + lockedTime.stop(); + assert(pos <= expected_size, "Size must be correct"); + } else { + nr_of_entries[idx] = 0; + failed_alloc = true; + lockedTime.stop(); + continue; + } + } + + // See if it makes sense to trim. We have to shave of enough and don't + // trim anyway after sorting. + if ((pos < expected_size - 16) && (pos < max_entries)) { + void* result = real_malloc_funcs->realloc(entries[idx], pos * sizeof(StatEntryCopy)); + + if (result != nullptr) { + entries[idx] = (StatEntryCopy*) result; + } + } + + nr_of_entries[idx] = pos; + total_entries += expected_size; + total_non_empty_entries += pos; + + // Now sort so we might be able to trim the array to only contain the + // maximum possible entries. + if (spec._sort_by_count) { + qsort(entries[idx], nr_of_entries[idx], sizeof(StatEntryCopy), sort_by_count); + } else { + qsort(entries[idx], nr_of_entries[idx], sizeof(StatEntryCopy), sort_by_size); + } + + // Free up some memory if possible. + if (nr_of_entries[idx] > max_entries) { + void* result = real_malloc_funcs->realloc(entries[idx], max_entries * sizeof(StatEntryCopy)); + + if (result == nullptr) { + // No problem, since the original memory is still there. Should not happen + // in reality. + } else { + entries[idx] = (StatEntryCopy*) result; + } + + nr_of_entries[idx] = max_entries; + } + } + + uint64_t size_limit = total_size; + uint64_t count_limit = total_count; + + if (spec._dump_percentage > 0) { + if (spec._sort_by_count) { + count_limit = (uint64_t) (0.01 * total_count * spec._dump_percentage); + } else { + size_limit = (uint64_t) (0.01 * total_size * spec._dump_percentage); + } + } + + AddressHashSet filter_cache(!spec._on_error); + + int curr_pos[NR_OF_STACK_MAPS]; + memset(curr_pos, 0, NR_OF_STACK_MAPS * sizeof(int)); + + uint64_t printed_size = 0; + uint64_t printed_count = 0; + int printed_entries = 0; + + for (int i = 0; i < max_entries; ++i) { + int max_pos = -1; + StatEntryCopy* max = nullptr; + + // Find the largest entry not currently printed. + if (spec._sort_by_count) { + for (int j = 0; j < NR_OF_STACK_MAPS; ++j) { + if (curr_pos[j] < nr_of_entries[j]) { + if ((max == nullptr) || (max->_count < entries[j][curr_pos[j]]._count)) { + max = &entries[j][curr_pos[j]]; + max_pos = j; + } + } + } + } else { + for (int j = 0; j < NR_OF_STACK_MAPS; ++j) { + if (curr_pos[j] < nr_of_entries[j]) { + if ((max == nullptr) || (max->_size < entries[j][curr_pos[j]]._size)) { + max = &entries[j][curr_pos[j]]; + max_pos = j; + } + } + } + } + + if (max == nullptr) { + // Done everything we can. + break; + } + + curr_pos[max_pos] += 1; + + if (dump_entry(dump_stream, max, i + 1, total_size, total_count, + total_non_empty_entries, spec._filter, &filter_cache)) { + printed_size += max->_size; + printed_count += max->_count; + printed_entries += 1; + + if (printed_entries >= max_printed_entries) { + break; + } + } + + if (printed_size > size_limit) { + break; + } + + if (printed_count > count_limit) { + break; + } + } + + for (int i = 0; i < NR_OF_STACK_MAPS; ++i) { + real_malloc_funcs->free(entries[i]); + } + + dump_stream->cr(); + dump_stream->print_cr("Printed %'d stacks", printed_entries); + + if (_track_free) { + dump_stream->print_cr("Total unique stacks: %'d (%'d including stacks with no alive allocations)", + total_non_empty_entries, total_entries); + } else { + dump_stream->print_cr("Total unique stacks: %'d", total_non_empty_entries); + } + + dump_stream->print_raw("Total allocated bytes: "); + print_mem(dump_stream, total_size); + dump_stream->cr(); + dump_stream->print_raw("Total allocation count: "); + print_count(dump_stream, total_count); + dump_stream->cr(); + dump_stream->print_raw("Total printed bytes: "); + print_mem(dump_stream, printed_size, total_size); + dump_stream->cr(); + dump_stream->print_raw("Total printed count: "); + print_count(dump_stream, printed_count, total_count); + dump_stream->cr(); + + totalTime.stop(); + + if (failed_alloc) { + dump_stream->print_cr("Failed to alloc memory during dump, so it might be incomplete!"); + } + + if (spec._internal_stats && _detailed_stats) { + uint64_t per_stack = _stack_walk_time / MAX2(_stack_walk_count, (uint64_t) 1); + msg_stream->cr(); + msg_stream->print_cr("Sampled %'" PRId64 " stacks, took %'" PRId64 " ns per stack on average.", + _stack_walk_count, per_stack); + msg_stream->print_cr("Sampling took %.2f seconds in total", _stack_walk_time * 1e-9); + msg_stream->print_cr("Tracked allocations : %'" PRId64, _tracked_ptrs); + msg_stream->print_cr("Untracked allocations: %'" PRId64, _not_tracked_ptrs); + msg_stream->print_cr("Untracked frees : %'" PRId64, _failed_frees); + + if ((_to_track_mask > 0) && (_tracked_ptrs > 0)) { + double frac = 100.0 * _tracked_ptrs / (_tracked_ptrs + _not_tracked_ptrs); + double rate = 100.0 / frac; + int target = (int) (0.5 + (_to_track_mask + 1) / (double) _to_track_limit); + msg_stream->print_cr("%.2f %% of the allocations were tracked, about every %.2f allocations " \ + "(target %d)", frac, rate, target); + } + } + + if (spec._internal_stats) { + print_allocation_stats(msg_stream, (HashMapData*) _stack_maps_data, + NR_OF_STACK_MAPS, "stack maps"); + + if (_track_free) { + print_allocation_stats(msg_stream, (HashMapData*) _alloc_maps_data, + NR_OF_ALLOC_MAPS, "alloc maps"); + } + + if (uses_filter) { + msg_stream->cr(); + msg_stream->print_raw_cr("Statistic for filter cache:"); + msg_stream->print("Allocated memory: "); + print_mem(dump_stream, filter_cache.allocated(), 0); + msg_stream->cr(); + msg_stream->print_cr("Load factor : %.3f", filter_cache.load()); + } + } + + msg_stream->cr(); + msg_stream->print_cr("Dumping done in %.3f s (%.3f s of that locked)", + totalTime.milliseconds() * 0.001, + lockedTime.milliseconds() * 0.001); + + set_malloc_suspended(false); + + if (used_rainy_day_fund) { + setup_hooks(&_malloc_stat_hooks, nullptr); + } + + return true; +} + +void MallocStatisticImpl::shutdown() { + _shutdown = true; + + if (_initialized) { + _enabled = false; + + if (register_hooks != nullptr) { + register_hooks(nullptr); + } + } +} + +static void dump_from_flags(bool on_error) { + DumpSpec spec; + char const* file = MallocTraceDumpOutput; + spec._on_error = on_error; + spec._filter = MallocTraceDumpFilter; + spec._sort_by_count = MallocTraceDumpSortByCount; + spec._max_entries = MallocTraceDumpMaxEntries; + spec._dump_percentage = MallocTraceDumpPercentage; + spec._hide_dump_allocs = MallocTraceDumpHideDumpAllocs; + spec._internal_stats = MallocTraceDumpInternalStats; + + if (is_non_empty_string(file)) { + if (strcmp("stdout", file) == 0) { + fdStream fds(1); + mallocStatImpl::MallocStatisticImpl::dump(&fds, &fds, spec); + } else if (strcmp("stderr", file) == 0) { + fdStream fds(2); + mallocStatImpl::MallocStatisticImpl::dump(&fds, &fds, spec); + } else { + char const* pid_tag = strstr(file, "@pid"); + + if (pid_tag != nullptr) { + size_t len = strlen(file); + size_t first = pid_tag - file; + char buf[32768]; + jio_snprintf(buf, sizeof(buf), "%.*s" UINTX_FORMAT "%s", + first, file, os::current_process_id(), pid_tag + 4); + fileStream fs(buf, "a"); + mallocStatImpl::MallocStatisticImpl::dump(&fs, &fs, spec); + } else { + fileStream fs(file, "a"); + mallocStatImpl::MallocStatisticImpl::dump(&fs, &fs, spec); + } + } + } else { + stringStream ss; + mallocStatImpl::MallocStatisticImpl::dump(&ss, &ss, spec); + } +} + +class MallocTraceDumpPeriodicTask : public PeriodicTask { +private: + int _left; + +public: + MallocTraceDumpPeriodicTask(uint64_t delay) : + PeriodicTask(MIN2((uint64_t) 2000000000, 1000 * delay)), + _left(MallocTraceDumpCount - 1) { + } + + virtual void task(); +}; + +void MallocTraceDumpPeriodicTask::task() { + dump_from_flags(false); + --_left; + + if (_left == 0) { + disenroll(); + } +} + +class MallocTraceDumpInitialTask : public PeriodicTask { + +public: + MallocTraceDumpInitialTask(uint64_t delay) : + PeriodicTask(MIN2((uint64_t) 2000000000, 1000 * delay)) { + } + + virtual void task(); +}; + +void MallocTraceDumpInitialTask::task() { + dump_from_flags(false); + + if (MallocTraceDumpCount > 1) { + uint64_t delay = MAX2((uint64_t) 1, parse_timespan(MallocTraceDumpInterval)); + + mallocStatImpl::MallocTraceDumpPeriodicTask* task = new mallocStatImpl::MallocTraceDumpPeriodicTask(delay); + task->enroll(); + } + + disenroll(); +} + +void enable_from_flags() { + TraceSpec spec; + stringStream ss; + + spec._stack_depth = (int) MallocTraceStackDepth; + spec._use_backtrace = MallocTraceUseBacktrace; + spec._only_nth = (int) MallocTraceOnlyNth; + spec._track_free = MallocTraceTrackFree; + spec._detailed_stats = MallocTraceDetailedStats; + + if (MallocTraceDumpOnError) { + spec._rainy_day_fund = (int) MallocTraceRainyDayFund; + } + + if (!MallocStatistic::enable(&ss, spec) && MallocTraceExitIfFail) { + fprintf(stderr, "Could not enable malloc trace via -XX:+MallocTraceAtStartup: %s", ss.base()); + os::exit(1); + } +} + +static void enable_delayed_dump() { + if (MallocTraceDumpCount > 0) { + uint64_t delay = MAX2((uint64_t) 1, parse_timespan(MallocTraceDumpDelay)); + mallocStatImpl::MallocTraceDumpInitialTask* task = new mallocStatImpl::MallocTraceDumpInitialTask(delay); + task->enroll(); + } +} + +class MallocTraceEnablePeriodicTask : public PeriodicTask { + +public: + MallocTraceEnablePeriodicTask(uint64_t delay) : + PeriodicTask(1000 * delay) { + } + + virtual void task(); +}; + +void MallocTraceEnablePeriodicTask::task() { + enable_from_flags(); + enable_delayed_dump(); +} + +} // namespace mallocStatImpl + +void MallocStatistic::initialize() { + // Don't enable this if the other malloc trace is on. +#if defined(LINUX) + if (EnableMallocTrace) { + return; + } +#endif + + // Remove the hooks from the preload env, so we don't + // preload mallochooks for spawned programs. + mallocStatImpl::remove_malloc_hooks_from_env(); + + // We have to make sure the child process of a fork doesn't run with + // enabled malloc hooks before forking. + pthread_atfork(nullptr, nullptr, mallocStatImpl::after_child_fork); + + mallocStatImpl::MallocStatisticImpl::initialize(); + + if (MallocTraceAtStartup) { +#define CHECK_TIMESPAN_ARG(argument) \ + char const* error_##argument; \ + parse_timespan(argument, &error_##argument); \ + if (error_##argument != nullptr) { \ + fprintf(stderr, "Could not parse argument '%s' of -XX:" #argument ": %s\n", argument, error_##argument); \ + os::exit(1); \ + } + + // Check interval specs now, so we don't fail later. + CHECK_TIMESPAN_ARG(MallocTraceEnableDelay); + CHECK_TIMESPAN_ARG(MallocTraceDumpDelay); + CHECK_TIMESPAN_ARG(MallocTraceDumpInterval); + + uint64_t delay = parse_timespan(MallocTraceEnableDelay); + + if (delay > 0) { + mallocStatImpl::MallocTraceEnablePeriodicTask* task = new mallocStatImpl::MallocTraceEnablePeriodicTask(delay); + task->enroll(); + } else { + mallocStatImpl::enable_from_flags(); + mallocStatImpl::enable_delayed_dump(); + } + } +} + +bool MallocStatistic::enable(outputStream* st, TraceSpec const& spec) { + return mallocStatImpl::MallocStatisticImpl::enable(st, spec); +} + +bool MallocStatistic::disable(outputStream* st) { + return mallocStatImpl::MallocStatisticImpl::disable(st); +} + +bool MallocStatistic::dump(outputStream* st, DumpSpec const& spec) { + const char* dump_file = spec._dump_file; + + if (is_non_empty_string(dump_file)) { + int fd; + + if (strcmp("stderr", dump_file) == 0) { + fd = 2; + } else if (strcmp("stdout", dump_file) == 0) { + fd = 1; + } else { + fd = ::open(dump_file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + + if (fd < 0) { + st->print_cr("Could not open '%s' for output.", dump_file); + + return false; + } + } + + fdStream dump_stream(fd); + bool result = mallocStatImpl::MallocStatisticImpl::dump(st, &dump_stream, spec); + + if ((fd != 1) && (fd != 2)) { + ::close(fd); + } + + return fd; + } + + return mallocStatImpl::MallocStatisticImpl::dump(st, st, spec); +} + +void MallocStatistic::emergencyDump() { + // Check enabled at all or already done. + if (!MallocTraceDumpOnError || mallocStatImpl::MallocStatisticImpl::rainy_day_fund_used()) { + return; + } + + mallocStatImpl::dump_from_flags(true); +} + +void MallocStatistic::shutdown() { + mallocStatImpl::MallocStatisticImpl::shutdown(); +} + +MallocTraceEnableDCmd::MallocTraceEnableDCmd(outputStream* output, bool heap) : + DCmdWithParser(output, heap), + _stack_depth("-stack-depth", "The maximum stack depth to track", "INT", false, "12"), + _use_backtrace("-use-backtrace", "If true we try to use the backtrace() method to sample " \ + "the stack traces.", "BOOLEAN", false, "false"), + _only_nth("-only-nth", "If > 1 we only track about every n'th allocation. Note that we round " \ + "the given number to the closest power of 2.", "INT", false, "1"), + _force("-force", "If the trace is already enabled, we disable it first.", "BOOLEAN", false, "false"), + _track_free("-track-free", "If true we also track frees, so we know the live memory consumption " \ + "and not just the total allocated amount. This costs some performance and memory.", + "BOOLEAN", false, "false"), + _detailed_stats("-detailed-stats", "Collect more detailed statistics. This will costs some " \ + "CPU time, but no memory.", "BOOLEAN", false, "false") { + _dcmdparser.add_dcmd_option(&_stack_depth); + _dcmdparser.add_dcmd_option(&_use_backtrace); + _dcmdparser.add_dcmd_option(&_only_nth); + _dcmdparser.add_dcmd_option(&_force); + _dcmdparser.add_dcmd_option(&_track_free); + _dcmdparser.add_dcmd_option(&_detailed_stats); +} + +void MallocTraceEnableDCmd::execute(DCmdSource source, TRAPS) { + // Need to switch to native or the long operations block GCs. + ThreadToNativeFromVM ttn(THREAD); + + TraceSpec spec; + spec._stack_depth = (int) _stack_depth.value(); + spec._use_backtrace = _use_backtrace.value(); + spec._only_nth = (int) _only_nth.value(); + spec._force = _force.value(); + spec._track_free = _track_free.value(); + spec._detailed_stats = _detailed_stats.value(); + + if (MallocStatistic::enable(_output, spec)) { + _output->print_raw_cr("Malloc statistic enabled"); + } +} + +MallocTraceDisableDCmd::MallocTraceDisableDCmd(outputStream* output, bool heap) : + DCmdWithParser(output, heap) { +} + +void MallocTraceDisableDCmd::execute(DCmdSource source, TRAPS) { + // Need to switch to native or the long operations block GCs. + ThreadToNativeFromVM ttn(THREAD); + + if (MallocStatistic::disable(_output)) { + _output->print_raw_cr("Malloc statistic disabled."); + } +} + +MallocTraceDumpDCmd::MallocTraceDumpDCmd(outputStream* output, bool heap) : + DCmdWithParser(output, heap), + _dump_file("-dump-file", "If given the dump command writes the result to the given file. " \ + "Note that the filename is interpreted by the target VM. You can use " \ + "'stdout' or 'stderr' as filenames to dump via stdout or stderr of " \ + "the target VM", "STRING", false), + _filter("-filter", "If given we only print a stack if it includes a function which contains the " \ + "given string as a substring.", "STRING", false), + _max_entries("-max-entries", "The maximum number of entries to dump.", "INT", false, "10"), + _dump_percentage("-percentage", "If > 0 we dump the given percentage of allocated bytes " \ + "(or allocated objects if sorted by count). In that case the -max-entries " \ + "option is ignored", "INT", false, "0"), + _sort_by_count("-sort-by-count", "If given the stacks are sorted according to the number " \ + "of allocations. Otherwise they are orted by the number of allocated bytes.", + "BOOLEAN", false), + _internal_stats("-internal-stats", "If given some internal statistics about the overhead of " \ + "the trace is included in the output", "BOOLEAN", false) { + _dcmdparser.add_dcmd_option(&_dump_file); + _dcmdparser.add_dcmd_option(&_filter); + _dcmdparser.add_dcmd_option(&_max_entries); + _dcmdparser.add_dcmd_option(&_dump_percentage); + _dcmdparser.add_dcmd_option(&_sort_by_count); + _dcmdparser.add_dcmd_option(&_internal_stats); +} + +void MallocTraceDumpDCmd::execute(DCmdSource source, TRAPS) { + // Need to switch to native or the long operations block GCs. + ThreadToNativeFromVM ttn(THREAD); + + DumpSpec spec; + spec._dump_file = _dump_file.value(); + spec._filter = _filter.value(); + spec._max_entries = _max_entries.value(); + spec._dump_percentage = _dump_percentage.value(); + spec._on_error = false; + spec._sort_by_count = _sort_by_count.value(); + spec._internal_stats = _internal_stats.value(); + + MallocStatistic::dump(_output, spec); +} + +} // namespace sap + +#endif // defined(LINUX) || defined(__APPLE__) + diff --git a/src/hotspot/os/posix/malloctrace/mallocTracePosix.hpp b/src/hotspot/os/posix/malloctrace/mallocTracePosix.hpp new file mode 100644 index 00000000000..f84c9fd1584 --- /dev/null +++ b/src/hotspot/os/posix/malloctrace/mallocTracePosix.hpp @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_POSIX_MALLOCTRACE_MALLOCTRACE_HPP +#define OS_POSIX_MALLOCTRACE_MALLOCTRACE_HPP + +#include "services/diagnosticCommand.hpp" +#include "utilities/globalDefinitions.hpp" + +#if defined(LINUX) || defined(__APPLE__) + +class outputStream; + +namespace sap { + +// The spec we use for configuring the trace +struct TraceSpec { + int _stack_depth; + bool _use_backtrace; + int _only_nth; + bool _force; + bool _track_free; + bool _detailed_stats; + int _rainy_day_fund; + + TraceSpec() : + _stack_depth(10), + _use_backtrace(true), + _only_nth(0), + _force(false), + _track_free(false), + _detailed_stats(false), + _rainy_day_fund(0) { + } +}; + +// The spec we use for configuring the dump. +struct DumpSpec { + const char* _dump_file; + const char* _filter; + int _max_entries; + bool _hide_dump_allocs; + bool _on_error; + bool _sort_by_count; + int _dump_percentage; + bool _internal_stats; + + DumpSpec() : + _dump_file(nullptr), + _filter(nullptr), + _max_entries(0), + _hide_dump_allocs(true), + _on_error(false), + _sort_by_count(false), + _dump_percentage(100), + _internal_stats(false) { + } +}; + +// Traces where allocations take place. Sums up the allocations by stack and total +// size. It is cheaper than a full trace, since it doesn't have to record frees +// and doesn't have to store data for each individual allocation. +class MallocStatistic : public AllStatic { + +public: + + // Called early to initialize the class. + static void initialize(); + + // Enables the tracing. Returns true if enabled. + static bool enable(outputStream* st, TraceSpec const& spec); + + // Disables the tracing. Returns true if disabled. + static bool disable(outputStream* st); + + // Dumps the statistic. + static bool dump(outputStream* st, DumpSpec const& spec); + + // Does the emergency dump. + static void emergencyDump(); + + // Shuts down the statistic on error. + static void shutdown(); +}; + +class MallocTraceEnableDCmd : public DCmdWithParser { + DCmdArgument _stack_depth; + DCmdArgument _use_backtrace; + DCmdArgument _only_nth; + DCmdArgument _force; + DCmdArgument _track_free; + DCmdArgument _detailed_stats; + +public: + static int num_arguments() { + return 6; + } + + MallocTraceEnableDCmd(outputStream* output, bool heap); + + static const char* name() { + return "MallocTrace.enable"; + } + + static const char* description() { + return "Enables tracing memory allocations"; + } + + static const char* impact() { + return "High"; + } + + static const JavaPermission permission() { + JavaPermission p = { "java.lang.management.ManagementPermission", "control", nullptr }; + return p; + } + + virtual void execute(DCmdSource source, TRAPS); +}; + +class MallocTraceDisableDCmd : public DCmdWithParser { + +public: + static int num_arguments() { + return 0; + } + + MallocTraceDisableDCmd(outputStream* output, bool heap); + + static const char* name() { + return "MallocTrace.disable"; + } + + static const char* description() { + return "Disables tracing memory allocations"; + } + + static const char* impact() { + return "Low"; + } + + static const JavaPermission permission() { + JavaPermission p = { "java.lang.management.ManagementPermission", "control", nullptr }; + return p; + } + + virtual void execute(DCmdSource source, TRAPS); +}; + +class MallocTraceDumpDCmd : public DCmdWithParser { +private: + + DCmdArgument _dump_file; + DCmdArgument _filter; + DCmdArgument _max_entries; + DCmdArgument _dump_percentage; + DCmdArgument _sort_by_count; + DCmdArgument _internal_stats; + +public: + static int num_arguments() { + return 6; + } + + MallocTraceDumpDCmd(outputStream* output, bool heap); + + static const char* name() { + return "MallocTrace.dump"; + } + + static const char* description() { + return "Dumps the currently running malloc trace"; + } + + static const char* impact() { + return "Low"; + } + + static const JavaPermission permission() { + JavaPermission p = { "java.lang.management.ManagementPermission", "control", nullptr }; + return p; + } + + virtual void execute(DCmdSource source, TRAPS); +}; + +} + +#endif // defined(LINUX) || defined(__APPLE__) + +#endif // OS_POSIX_MALLOCTRACE_MALLOCTRACE_HPP diff --git a/src/hotspot/os/windows/vitals_windows.cpp b/src/hotspot/os/windows/vitals_windows.cpp new file mode 100644 index 00000000000..dbf083bb708 --- /dev/null +++ b/src/hotspot/os/windows/vitals_windows.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2019, 2021 SAP SE. All rights reserved. + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "runtime/os.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "vitals/vitals_internals.hpp" + +#include + +namespace sapmachine_vitals { + +static Column* g_col_system_memoryload = NULL; +static Column* g_col_system_avail_phys = NULL; +static Column* g_col_process_working_set_size = NULL; +static Column* g_col_process_commit_charge = NULL; + +bool platform_columns_initialize() { + g_col_system_memoryload = + define_column("system", NULL, "mload", "Approximate percentage of physical memory that is in use.", true, MAX); + + // MEMORYSTATUSEX ullAvailPhys + g_col_system_avail_phys = + define_column("system", NULL, "avail-phys", "Amount of physical memory currently available.", true, MIN); + // PROCESS_MEMORY_COUNTERS_EX WorkingSetSize + g_col_process_working_set_size = + define_column("system", NULL, "wset", "Working set size", true); + + // PROCESS_MEMORY_COUNTERS_EX PrivateUsage + g_col_process_commit_charge = + define_column("system", NULL, "comch", "Commit charge", true); + + return true; +} + +static void set_value_in_sample(Column* col, Sample* sample, value_t val) { + if (col != NULL) { + int index = col->index(); + sample->set_value(index, val); + } +} + +void sample_platform_values(Sample* sample) { + + MEMORYSTATUSEX mse; + mse.dwLength = sizeof(mse); + if (::GlobalMemoryStatusEx(&mse)) { + set_value_in_sample(g_col_system_memoryload, sample, mse.dwMemoryLoad); + set_value_in_sample(g_col_system_avail_phys, sample, mse.ullAvailPhys); + } + + PROCESS_MEMORY_COUNTERS cnt; + cnt.cb = sizeof(cnt); + if (::GetProcessMemoryInfo(::GetCurrentProcess(), &cnt, sizeof(cnt))) { + set_value_in_sample(g_col_process_working_set_size, sample, cnt.WorkingSetSize); + set_value_in_sample(g_col_process_commit_charge, sample, cnt.PagefileUsage); + } + +} + +} // namespace sapmachine_vitals diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp index 16837da9cf2..4c12149670e 100644 --- a/src/hotspot/share/classfile/classLoaderData.cpp +++ b/src/hotspot/share/classfile/classLoaderData.cpp @@ -178,6 +178,11 @@ ClassLoaderData::ClassLoaderData(Handle h_class_loader, bool has_class_mirror_ho NOT_PRODUCT(_dependency_count = 0); // number of class loader dependencies JFR_ONLY(INIT_ID(this);) + + // SapMachine 2023-07-04: Vitals + if (EnableVitals) { + sapmachine_vitals::counters::inc_cld_count(has_class_mirror_holder); + } } ClassLoaderData::ChunkedHandleList::~ChunkedHandleList() { diff --git a/src/hotspot/share/classfile/classLoaderDataGraph.cpp b/src/hotspot/share/classfile/classLoaderDataGraph.cpp index 2046286651e..4ebcfba0f29 100644 --- a/src/hotspot/share/classfile/classLoaderDataGraph.cpp +++ b/src/hotspot/share/classfile/classLoaderDataGraph.cpp @@ -424,6 +424,11 @@ bool ClassLoaderDataGraph::do_unloading() { ClassUnloadingContext::context()->register_unloading_class_loader_data(data); + // SapMachine 2023-07-04: Vitals + if (EnableVitals) { + sapmachine_vitals::counters::dec_cld_count(data->has_class_mirror_holder()); + } + // Move dead CLD to unloading list. if (prev != nullptr) { prev->unlink_next(); diff --git a/src/hotspot/share/classfile/classLoaderDataGraph.inline.hpp b/src/hotspot/share/classfile/classLoaderDataGraph.inline.hpp index 6880194009c..6391a721ce2 100644 --- a/src/hotspot/share/classfile/classLoaderDataGraph.inline.hpp +++ b/src/hotspot/share/classfile/classLoaderDataGraph.inline.hpp @@ -32,6 +32,10 @@ #include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" +// SapMachine 2019-09-01: Vitals +#include "runtime/globals.hpp" +#include "vitals/vitals.hpp" + inline ClassLoaderData *ClassLoaderDataGraph::find_or_create(Handle loader) { guarantee(loader() != nullptr && oopDesc::is_oop(loader()), "Loader must be oop"); // Gets the class loader data out of the java/lang/ClassLoader object, if non-null @@ -53,20 +57,36 @@ size_t ClassLoaderDataGraph::num_array_classes() { void ClassLoaderDataGraph::inc_instance_classes(size_t count) { Atomic::add(&_num_instance_classes, count, memory_order_relaxed); + // SapMachine 2019-02-20: Vitals + if (EnableVitals) { + sapmachine_vitals::counters::inc_classes_loaded(count); + } } void ClassLoaderDataGraph::dec_instance_classes(size_t count) { size_t old_count = Atomic::fetch_then_add(&_num_instance_classes, -count, memory_order_relaxed); assert(old_count >= count, "Sanity"); + // SapMachine 2019-02-20: Vitals + if (EnableVitals) { + sapmachine_vitals::counters::inc_classes_unloaded(count); + } } void ClassLoaderDataGraph::inc_array_classes(size_t count) { Atomic::add(&_num_array_classes, count, memory_order_relaxed); + // SapMachine 2019-02-20: Vitals + if (EnableVitals) { + sapmachine_vitals::counters::inc_classes_loaded(count); + } } void ClassLoaderDataGraph::dec_array_classes(size_t count) { size_t old_count = Atomic::fetch_then_add(&_num_array_classes, -count, memory_order_relaxed); assert(old_count >= count, "Sanity"); + // SapMachine 2019-02-20: Vitals + if (EnableVitals) { + sapmachine_vitals::counters::inc_classes_unloaded(count); + } } bool ClassLoaderDataGraph::should_clean_metaspaces_and_reset() { diff --git a/src/hotspot/share/logging/logTag.hpp b/src/hotspot/share/logging/logTag.hpp index b64097311a4..fd55d5fda8f 100644 --- a/src/hotspot/share/logging/logTag.hpp +++ b/src/hotspot/share/logging/logTag.hpp @@ -212,6 +212,7 @@ class outputStream; LOG_TAG(valuebasedclasses) \ LOG_TAG(verification) \ LOG_TAG(verify) \ + LOG_TAG(vitals) /* SapMachine 2022-06-01: Vitals */ \ LOG_TAG(vmmutex) \ LOG_TAG(vmoperation) \ LOG_TAG(vmthread) \ diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp index 9588f32e6b3..c9a962baeee 100644 --- a/src/hotspot/share/prims/jvm.cpp +++ b/src/hotspot/share/prims/jvm.cpp @@ -3023,6 +3023,11 @@ JVM_ENTRY(void, JVM_StartThread(JNIEnv* env, jobject jthread)) JavaThread::name_for(JNIHandles::resolve_non_null(jthread))); // No one should hold a reference to the 'native_thread'. native_thread->smr_delete(); + + // SapMachine 2021-05-21: All ...OnOutOfMemoryError switches should work for + // thread creation failures too. + report_java_out_of_memory(os::native_thread_creation_failed_msg()); + if (JvmtiExport::should_post_resource_exhausted()) { JvmtiExport::post_resource_exhausted( JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_THREADS, diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index a56e1fcb998..8c4cac48447 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -2934,6 +2934,11 @@ jint Arguments::finalize_vm_init_args(bool patch_mod_javabase) { UNSUPPORTED_OPTION(ShowRegistersOnAssert); #endif // CAN_SHOW_REGISTERS_ON_ASSERT + // SapMachine 2021-05-21: Let ExitVMOnOutOfMemory be an alias for CrashOnOutOfMemoryError + if (ExitVMOnOutOfMemoryError && !CrashOnOutOfMemoryError) { + FLAG_SET_ERGO(CrashOnOutOfMemoryError, true); + } + return JNI_OK; } diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index 6f66a349595..53898fee93e 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -444,7 +444,8 @@ const int ObjectAlignmentInBytes = 8; product(bool, LogEvents, true, DIAGNOSTIC, \ "Enable the various ring buffer event logs") \ \ - product(int, LogEventsBufferEntries, 20, DIAGNOSTIC, \ + /* SapMachine 2019-05-28: More events */ \ + product(int, LogEventsBufferEntries, 75, DIAGNOSTIC, \ "Number of ring buffer event logs") \ range(1, NOT_LP64(1*K) LP64_ONLY(1*M)) \ \ @@ -504,6 +505,147 @@ const int ObjectAlignmentInBytes = 8; develop(bool, Verbose, false, \ "Print additional debugging information from other modes") \ \ + /* SapMachine 2019-02-20: Vitals */ \ + product(bool, EnableVitals, true, \ + "Enable sampling of vitals: memory, cpu utilization and various " \ + "VM core statistics; display via jcmd \"VM.vitals\".") \ + \ + product(uintx, VitalsSampleInterval, 10, \ + "Vitals sample rate interval in seconds (default 10)") \ + range(1, 24 * 3600) \ + \ + product(uintx, VitalsShortTermTableHours, 1, \ + "The size of the short term vitals table in hours") \ + range(1, 365 * 24) \ + \ + product(uintx, VitalsLongTermSampleIntervalMinutes, 60, \ + "Vitals sample rate interval in minutes for the long term table " \ + "(default 60)") \ + range(1, 365 * 24 * 60) \ + \ + product(uintx, VitalsLongTermTableDays, 14, \ + "The size of the long term vitals table in days") \ + range(1, 10 * 365) \ + \ + product(bool, VitalsLockFreeSampling, false, DIAGNOSTIC, \ + "When sampling vitals, omit any actions which require locking.") \ + \ + product(bool, DumpVitalsAtExit, false, \ + "Dump vitals at VM exit into two files, by default called " \ + "sapmachine_vitals_.txt and sapmachine_vitals_.csv. " \ + "Use -XX:VitalsFile option to change the file names.") \ + \ + product(bool, PrintVitalsAtExit, false, \ + "Prints vitals at VM exit to tty.") \ + \ + product(bool, StoreVitalsExtremas, true, \ + "If enabled we store the samples for extremum values of " \ + "selected types.") \ + \ + product(ccstr, VitalsFile, NULL, \ + "When DumpVitalsAtExit is set, the file name prefix for the " \ + "output files (default is sapmachine_vitals_).") \ + \ + /* SapMachine 2023-09-18: malloc trace */ \ + product(bool, UseMallocHooks, false, \ + "Preloads the malloc hooks library needed for the malloc trace. " \ + "This flag only works when using a JDK launcher. Otherwise the " \ + "library has to be preloaded by hand.") \ + \ + product(bool, MallocTraceAtStartup, false, \ + "If set the malloc trace is enabled at startup.") \ + \ + product(bool, MallocTraceExitIfFail, true, \ + "If set and enabling the malloc trace at startup fails, we " \ + "print an error message and exit. Otherwise the error is " \ + "silently ignored.") \ + \ + product(bool, MallocTraceTrackFree, true, \ + "If set the malloc trace also tracks deallocation of memory " \ + "if enabled at startup.") \ + \ + product(ccstr, MallocTraceEnableDelay, "0s", \ + "If > 0 seconds and MallocTraceAtStartup is enabled, we delay " \ + "the startup of tracking by the given amount of time. Can use " \ + "s, m, h or d to specify the delay.") \ + \ + product(uintx, MallocTraceStackDepth, 12, \ + "The maximum depth of stack traces for the malloc trace if " \ + "enabled at startup.") \ + \ + product(uintx, MallocTraceOnlyNth, 1, \ + "if > 1 we only track about every n'th allocation for the " \ + "malloc trace if enabled at startup.") \ + range(1, 1000) \ + \ + product(bool, MallocTraceUseBacktrace, true, \ + "If set we use the backtrace() call to sample the stacks of " \ + "the malloc trace if enabled at startup. Note that while this " \ + "creates better stack traces, it is also slower and not " \ + "supported on every system. If not supported, this option is " \ + "silently ignored.") \ + \ + product(ccstr, MallocTraceUnwindLibName, "libunwind.so.8", \ + "The path of the libunwind to load if it should be used to " \ + "create the stack traces and the backtrace() function cannot " \ + "be found. If libunwind is not on the library path, an " \ + "absolute path should be used.") \ + \ + product(bool, MallocTraceDetailedStats, false, \ + "If enabled we collect more detailed statistics for the malloc " \ + "trace if enabled at startup. This costs some performance.") \ + \ + product(bool, MallocTraceDumpOnError, false, \ + "If enabled and the malloc trace is enabled too we do an " \ + "emergency dump on native out-of-memory errors.") \ + \ + product(uintx, MallocTraceRainyDayFund, 1* M, \ + "The size of the rainy day fund to use when doing an emergency " \ + "dump.") \ + \ + product(ccstr, MallocTraceDumpFilter, "", \ + "If set, we only print stacks which contains functions which " \ + "match the given string.") \ + \ + product(bool, MallocTraceDumpInternalStats, false, \ + "If enabled we include internal statistics in the dump. ") \ + \ + product(uintx, MallocTraceDumpCount, 0, \ + "The number of dumps to perform.") \ + \ + product(ccstr, MallocTraceDumpDelay, "1h", \ + "The delay after the trace was enabled at which to start the " \ + "regular dumps. The format supports seconds, minutes, hours " \ + "and days, e.g. '1s 2m 3h 4d' or '20s'.") \ + \ + product(ccstr, MallocTraceDumpInterval, "1h", \ + "The interval for the dump for the dumps. The format supports " \ + "seconds, minutes, hours and days, e.g. '1s 2m 3h 4d' or '20s'.") \ + \ + product(bool, MallocTraceDumpSortByCount, false, \ + "If given we sort the output by allocation count instead of " \ + "allocation size.") \ + \ + product(uintx, MallocTraceDumpMaxEntries, 10, \ + "If > 0 it limits the number of entries printed. If no sort " \ + "is specified via -XX:MallocTraceDumpSort, we sort by " \ + "size.") \ + \ + product(uintx, MallocTraceDumpPercentage, 0, \ + "If > 0 we dump the given percentage of allocated bytes " \ + "(or allocated objects if sorted by count). In that case the " \ + "-XX:MallocTraceDumpMaxEntries option is ignored.") \ + \ + product(bool, MallocTraceDumpHideDumpAllocs, true, \ + "If enabled we don't track the allocation done for the dump.") \ + \ + product(ccstr, MallocTraceDumpOutput, "stderr", \ + "If set the dump is appended to the given file. 'stderr' and " \ + "'stdout' can be used for dumping to stderr or stdout. " \ + "Otherwise the dump is written to the given file name ( " \ + "the first occurrance of '@pid' is replaced by the pid of the " \ + "process).") \ + \ develop(bool, PrintMiscellaneous, false, \ "Print uncategorized debugging information (requires +Verbose)") \ \ @@ -557,11 +699,13 @@ const int ObjectAlignmentInBytes = 8; "from JVM " \ "(also see HeapDumpPath, HeapDumpGzipLevel)") \ \ + /* SapMachine 2024-05-10: HeapDumpPath for jcmd */ \ product(ccstr, HeapDumpPath, nullptr, MANAGEABLE, \ "When HeapDumpOnOutOfMemoryError, HeapDumpBeforeFullGC " \ - "or HeapDumpAfterFullGC is on, the path (filename or " \ - "directory) of the dump file (defaults to java_pid.hprof " \ - "in the working directory)") \ + "or HeapDumpAfterFullGC is on, or a heap dump is triggered by " \ + "jcmd GC.heap_dump without specifying a path, the path " \ + "(filename or directory) of the dump file. " \ + "(defaults to java_pid.hprof in the working directory)") \ \ product(int, HeapDumpGzipLevel, 0, MANAGEABLE, \ "When HeapDumpOnOutOfMemoryError, HeapDumpBeforeFullGC " \ @@ -586,7 +730,8 @@ const int ObjectAlignmentInBytes = 8; "Repeat compilation without installing code (number of times)") \ range(0, max_jint) \ \ - product(bool, PrintExtendedThreadInfo, false, \ + /* SapMachine 2018-08-29: Enable this per default */ \ + product(bool, PrintExtendedThreadInfo, true, \ "Print more information in thread dump") \ \ product(intx, ScavengeRootsInCode, 2, DIAGNOSTIC, \ @@ -837,9 +982,18 @@ const int ObjectAlignmentInBytes = 8; "JVM exits on the first occurrence of an out-of-memory error " \ "thrown from JVM") \ \ + /* SapMachine 2021-05-21: Changed comment (we do not core on OOM) */ \ product(bool, CrashOnOutOfMemoryError, false, \ - "JVM aborts, producing an error log and core/mini dump, on the " \ - "first occurrence of an out-of-memory error thrown from JVM") \ + "JVM aborts on the first occurrence of an out-of-memory error " \ + "thrown from JVM. A thread stack is dumped to stdout and an " \ + "error log produced. No core file will be produced unless " \ + "-XX:+CreateCoredumpOnCrash is explicitly specified on the " \ + "command line.") \ + \ + /* SapMachine 2021-05-21: Support ExitVMOnOutOfMemory to keep */ \ + /* command line parity with SAPJVM */ \ + product(bool, ExitVMOnOutOfMemoryError, false, \ + "Alias for CrashOnOutOfMemoryError") \ \ product(intx, UserThreadWaitAttemptsAtExit, 30, \ "The number of times to wait for user threads to stop executing " \ @@ -1034,8 +1188,9 @@ const int ObjectAlignmentInBytes = 8; "If an error occurs, save the error data to this file " \ "[default: ./hs_err_pid%p.log] (%p replaced with pid)") \ \ + /* SapMachine 2018-12-18: Enable this per default. */ \ product(bool, ExtensiveErrorReports, \ - PRODUCT_ONLY(false) NOT_PRODUCT(true), \ + PRODUCT_ONLY(true) NOT_PRODUCT(true), \ "Error reports are more extensive.") \ \ product(bool, DisplayVMOutputToStderr, false, \ diff --git a/src/hotspot/share/runtime/java.cpp b/src/hotspot/share/runtime/java.cpp index 674716a8598..b97ec7db755 100644 --- a/src/hotspot/share/runtime/java.cpp +++ b/src/hotspot/share/runtime/java.cpp @@ -102,6 +102,15 @@ #include "jvmci/jvmci.hpp" #endif +// SapMachine 2019-09-01: Vitals +#include "runtime/globals.hpp" +#include "vitals/vitals.hpp" + +// SapMachine 2021-09-01: malloc-trace +#ifdef LINUX +#include "malloctrace/mallocTrace.hpp" +#endif + GrowableArray* collected_profiled_methods; static int compare_methods(Method** a, Method** b) { @@ -357,6 +366,14 @@ void print_statistics() { CompilationMemoryStatistic::print_all_by_size(tty, false, 0); } + // SapMachine 2019-09-01: Vitals + if (DumpVitalsAtExit) { + sapmachine_vitals::dump_reports(); + } + if (PrintVitalsAtExit) { + sapmachine_vitals::print_report(tty); + } + ThreadsSMRSupport::log_statistics(); ClassLoader::print_counters(tty); @@ -489,6 +506,12 @@ void before_exit(JavaThread* thread, bool halt) { if (PrintMemoryMapAtExit) { MemMapPrinter::print_all_mappings(tty, false); } +#ifdef HAVE_GLIBC_MALLOC_HOOKS + // SapMachine 2021-09-01: malloc-trace + if (PrintMallocTraceAtExit) { + sap::MallocTracer::print(tty, true); + } +#endif // HAVE_GLIBC_MALLOC_HOOKS #endif if (JvmtiExport::should_post_thread_life()) { @@ -520,6 +543,11 @@ void before_exit(JavaThread* thread, bool halt) { } } + // SapMachine 2021-09-01: Shutdown vitals thread + if (EnableVitals) { + sapmachine_vitals::cleanup(); + } + #undef BEFORE_EXIT_NOT_RUN #undef BEFORE_EXIT_RUNNING #undef BEFORE_EXIT_DONE diff --git a/src/hotspot/share/runtime/threads.cpp b/src/hotspot/share/runtime/threads.cpp index 9800f85dfe4..29046a6f367 100644 --- a/src/hotspot/share/runtime/threads.cpp +++ b/src/hotspot/share/runtime/threads.cpp @@ -113,6 +113,17 @@ #include "jfr/jfr.hpp" #endif +// SapMachine 2019-02-20: Vitals +#include "vitals/vitals.hpp" +#ifdef LINUX +#include "vitals_linux_himemreport.hpp" +#endif + +// SapMachine 2023-08-15: malloc trace +#if defined(LINUX) || defined(__APPLE__) +#include "malloctrace/mallocTracePosix.hpp" +#endif + // Initialization after module runtime initialization void universe_post_module_init(); // must happen after call_initPhase2 @@ -579,6 +590,15 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) { return status; } + // SapMachine 2023-09-20: malloc trace2 +#if defined(LINUX) || defined(__APPLE__) + sap::MallocStatistic::initialize(); +#else + if (MallocTraceAtStartup || UseMallocHooks) { + warning("Malloc trace is not supported on this platform"); + } +#endif + JFR_ONLY(Jfr::on_create_vm_1();) // Should be done after the heap is fully created @@ -785,6 +805,16 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) { JFR_ONLY(Jfr::on_create_vm_3();) + // SapMachine 2019-02-20: Vitals + if (EnableVitals) { + sapmachine_vitals::initialize(); + } +#ifdef LINUX + if (HiMemReport) { + sapmachine_vitals::initialize_himem_report_facility(); + } +#endif // LINUX + #if INCLUDE_MANAGEMENT Management::initialize(THREAD); @@ -1021,6 +1051,9 @@ void Threads::add(JavaThread* p, bool force_daemon) { // Make new thread known to active EscapeBarrier EscapeBarrier::thread_added(p); + + // SapMachine 2019-02-20: Vitals + sapmachine_vitals::counters::inc_threads_created(1); } void Threads::remove(JavaThread* p, bool is_daemon) { @@ -1352,6 +1385,22 @@ void Threads::print_on(outputStream* st, bool print_stacks, PrintOnClosure cl(st); non_java_threads_do(&cl); + // SapMachine 2019-11-07: Vitals + const Thread* vitals_sampler_thread = sapmachine_vitals::samplerthread(); + if (vitals_sampler_thread != NULL) { + vitals_sampler_thread->print_on(st); + st->cr(); + } + +#ifdef LINUX + // SapMachine 2022-05-07: HiMemReport + const Thread* himem_reporter_thread = sapmachine_vitals::himem_reporter_thread(); + if (himem_reporter_thread != NULL) { + himem_reporter_thread->print_on(st); + st->cr(); + } +#endif + st->flush(); } @@ -1417,6 +1466,15 @@ void Threads::print_on_error(outputStream* st, Thread* current, char* buf, print_on_error(WatcherThread::watcher_thread(), st, current, buf, buflen, &found_current); print_on_error(AsyncLogWriter::instance(), st, current, buf, buflen, &found_current); + // SapMachine 2019-11-07: Vitals + print_on_error(const_cast(sapmachine_vitals::samplerthread()), + st, current, buf, buflen, &found_current); +#ifdef LINUX + // SapMachine 2022-05-07: HiMemReport + print_on_error(const_cast(sapmachine_vitals::himem_reporter_thread()), + st, current, buf, buflen, &found_current); +#endif + if (Universe::heap() != nullptr) { PrintOnErrorClosure print_closure(st, current, buf, buflen, &found_current); Universe::heap()->gc_threads_do(&print_closure); diff --git a/src/hotspot/share/services/diagnosticCommand.cpp b/src/hotspot/share/services/diagnosticCommand.cpp index faabe74a2ff..a8fb55c76b1 100644 --- a/src/hotspot/share/services/diagnosticCommand.cpp +++ b/src/hotspot/share/services/diagnosticCommand.cpp @@ -72,9 +72,18 @@ #ifdef LINUX #include "os_posix.hpp" #include "mallocInfoDcmd.hpp" +// SapMachine 2021-09-01: malloc-trace +#include "malloctrace/mallocTraceDCmd.hpp" #include "trimCHeapDCmd.hpp" #include #endif +// SapMachine 2023-08-15: malloc trace2 +#if defined(LINUX) || defined(__APPLE__) +#include "malloctrace/mallocTracePosix.hpp" +#endif + +// SapMachine 2019-02-20: Vitals +#include "vitals/vitalsDCmd.hpp" static void loadAgentModule(TRAPS) { ResourceMark rm(THREAD); @@ -110,6 +119,8 @@ void DCmd::register_dcmds(){ DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); + // SapMachine 2019-02-20: Vitals + DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); #if INCLUDE_SERVICES DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(DCmd_Source_Internal | DCmd_Source_AttachAPI, true, false)); DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); @@ -140,6 +151,15 @@ void DCmd::register_dcmds(){ DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true,false)); DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true,false)); + // SapMachine 2021-09-01: malloc-trace + DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); +#endif +#if defined(LINUX) || defined(__APPLE__) + // SapMachine 2023-08-15: malloc trace2 + DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); + DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); + DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); + #endif // LINUX DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); @@ -160,7 +180,8 @@ void DCmd::register_dcmds(){ // Debug on cmd (only makes sense with JVMTI since the agentlib needs it). #if INCLUDE_JVMTI - DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, true)); + // SapMachine 2022-12-12 Revert JDK-8226608, we should show the VM.start_java_debugging command in jcmd help + DCmdFactory::register_DCmdFactory(new DCmdFactoryImpl(full_export, true, false)); #endif // INCLUDE_JVMTI #if INCLUDE_CDS @@ -473,7 +494,8 @@ void FinalizerInfoDCmd::execute(DCmdSource source, TRAPS) { #if INCLUDE_SERVICES // Heap dumping/inspection supported HeapDumpDCmd::HeapDumpDCmd(outputStream* output, bool heap) : DCmdWithParser(output, heap), - _filename("filename","Name of the dump file", "STRING",true), + // SapMachine 2024-05-10: HeapDumpPath for jcmd + _filename("filename", "Name of the dump file", "STRING", false, "if no filename was specified, but -XX:HeapDumpPath= is set, path is taken"), _all("-all", "Dump all objects, including unreachable objects", "BOOLEAN", false, "false"), _gzip("-gz", "If specified, the heap dump is written in gzipped format " @@ -494,6 +516,8 @@ HeapDumpDCmd::HeapDumpDCmd(outputStream* output, bool heap) : void HeapDumpDCmd::execute(DCmdSource source, TRAPS) { jlong level = -1; // -1 means no compression. jlong parallel = HeapDumper::default_num_of_dump_threads(); + // SapMachine 2024-05-10: HeapDumpPath for jcmd + bool use_heapdump_path = false; if (_gzip.is_set()) { level = _gzip.value(); @@ -516,11 +540,27 @@ void HeapDumpDCmd::execute(DCmdSource source, TRAPS) { } } + // SapMachine 2024-05-10: HeapDumpPath for jcmd + if (!_filename.is_set()) { + if (HeapDumpPath != nullptr) { + // use HeapDumpPath (file or directory is possible) + use_heapdump_path = true; + } else { + output()->print_cr("Filename or -XX:HeapDumpPath must be set!"); + return; + } + } + // Request a full GC before heap dump if _all is false // This helps reduces the amount of unreachable objects in the dump // and makes it easier to browse. - HeapDumper dumper(!_all.value() /* request GC if _all is false*/); - dumper.dump(_filename.value(), output(), (int) level, _overwrite.value(), (uint)parallel); + // SapMachine 2024-05-10: HeapDumpPath for jcmd + if (use_heapdump_path) { + HeapDumper::dump_heap(!_all.value(), output(), (int)level, _overwrite.value(), (uint)parallel); + } else { + HeapDumper dumper(!_all.value() /* request GC if _all is false*/); + dumper.dump(_filename.value(), output(), (int)level, _overwrite.value(), (uint)parallel); + } } ClassHistogramDCmd::ClassHistogramDCmd(outputStream* output, bool heap) : diff --git a/src/hotspot/share/services/heapDumper.cpp b/src/hotspot/share/services/heapDumper.cpp index 6f113cfca95..a6cb92bd825 100644 --- a/src/hotspot/share/services/heapDumper.cpp +++ b/src/hotspot/share/services/heapDumper.cpp @@ -2736,7 +2736,8 @@ void HeapDumper::set_error(char const* error) { // Called by out-of-memory error reporting by a single Java thread // outside of a JVM safepoint void HeapDumper::dump_heap_from_oome() { - HeapDumper::dump_heap(true); + // SapMachine 2024-05-10: HeapDumpPath for jcmd + HeapDumper::dump_heap(false, true); } // Called by error reporting by a single Java thread outside of a JVM safepoint, @@ -2745,17 +2746,26 @@ void HeapDumper::dump_heap_from_oome() { // general use, however, this method will need modification to prevent // inteference when updating the static variables base_path and dump_file_seq below. void HeapDumper::dump_heap() { - HeapDumper::dump_heap(false); + // SapMachine 2024-05-10: HeapDumpPath for jcmd + HeapDumper::dump_heap(false, false); } -void HeapDumper::dump_heap(bool oome) { +// SapMachine 2024-05-10: HeapDumpPath for jcmd +void HeapDumper::dump_heap(bool gc_before_heap_dump, outputStream* out, int compression, bool overwrite, uint parallel_thread_num) { + HeapDumper::dump_heap(gc_before_heap_dump, false, out, compression, overwrite, parallel_thread_num); +} + +// SapMachine 2024-05-10: HeapDumpPath for jcmd +void HeapDumper::dump_heap(bool gc_before_heap_dump, bool oome, outputStream* out, int compression, bool overwrite, uint parallel_thread_num) { static char base_path[JVM_MAXPATHLEN] = {'\0'}; static uint dump_file_seq = 0; char* my_path; const int max_digit_chars = 20; const char* dump_file_name = "java_pid"; - const char* dump_file_ext = HeapDumpGzipLevel > 0 ? ".hprof.gz" : ".hprof"; + // SapMachine 2024-05-10: HeapDumpPath for jcmd + const int ziplevel = compression < 0 ? HeapDumpGzipLevel : compression; + const char* dump_file_ext = ziplevel > 0 ? ".hprof.gz" : ".hprof"; // The dump file defaults to java_pid.hprof in the current working // directory. HeapDumpPath= can be used to specify an alternative @@ -2820,8 +2830,10 @@ void HeapDumper::dump_heap(bool oome) { } dump_file_seq++; // increment seq number for next time we dump - HeapDumper dumper(false /* no GC before heap dump */, + // SapMachine 2024-05-10: HeapDumpPath for jcmd + HeapDumper dumper(gc_before_heap_dump /* GC before heap dump */, oome /* pass along out-of-memory-error flag */); - dumper.dump(my_path, tty, HeapDumpGzipLevel); + // SapMachine 2024-05-10: HeapDumpPath for jcmd + dumper.dump(my_path, out, ziplevel, overwrite, parallel_thread_num); os::free(my_path); } diff --git a/src/hotspot/share/services/heapDumper.hpp b/src/hotspot/share/services/heapDumper.hpp index a69cf72aaf0..fc14e858407 100644 --- a/src/hotspot/share/services/heapDumper.hpp +++ b/src/hotspot/share/services/heapDumper.hpp @@ -49,7 +49,8 @@ class HeapDumper : public StackObj { // internal timer. elapsedTimer* timer() { return &_t; } - static void dump_heap(bool oome); + // SapMachine 2024-05-10: HeapDumpPath for jcmd + static void dump_heap(bool gc_before_heap_dump, bool oome, outputStream* out = tty, int compression = -1, bool overwrite = false, uint parallel_thread_num = default_num_of_dump_threads()); public: HeapDumper(bool gc_before_heap_dump) : @@ -70,6 +71,9 @@ class HeapDumper : public StackObj { static void dump_heap_from_oome() NOT_SERVICES_RETURN; + // SapMachine 2024-05-10: HeapDumpPath for jcmd + static void dump_heap(bool gc_before_heap_dump, outputStream* out, int compression, bool overwrite, uint parallel_thread_num) NOT_SERVICES_RETURN; + // Parallel thread number for heap dump, initialize based on active processor count. static uint default_num_of_dump_threads() { return MAX2(1, (uint)os::initial_active_processor_count() * 3 / 8); diff --git a/src/hotspot/share/utilities/debug.cpp b/src/hotspot/share/utilities/debug.cpp index 9f839fc1a13..1bda3f94063 100644 --- a/src/hotspot/share/utilities/debug.cpp +++ b/src/hotspot/share/utilities/debug.cpp @@ -64,6 +64,15 @@ #include "utilities/unsigned5.hpp" #include "utilities/vmError.hpp" +// SapMachine 2021-05-21 +#include "runtime/globals.hpp" +#include "runtime/globals_extension.hpp" + +// SapMachine 2023-08-15: malloc trace +#if defined(LINUX) || defined(__APPLE__) +#include "malloctrace/mallocTracePosix.hpp" +#endif + #include #include @@ -219,6 +228,13 @@ void report_fatal(VMErrorType error_type, const char* file, int line, const char void report_vm_out_of_memory(const char* file, int line, size_t size, VMErrorType vm_err_type, const char* detail_fmt, ...) { + // SapMachine 2023-11-03: Check if we should to an emergency dump for the malloc trace. +#if defined(LINUX) || defined(__APPLE__) + if ((vm_err_type == OOM_MALLOC_ERROR) || (vm_err_type == OOM_MMAP_ERROR)) { + sap::MallocStatistic::emergencyDump(); + } +#endif + va_list detail_args; va_start(detail_args, detail_fmt); @@ -258,6 +274,22 @@ void report_java_out_of_memory(const char* message) { // commands multiple times we just do it once when the first threads reports // the error. if (Atomic::cmpxchg(&out_of_memory_reported, 0, 1) == 0) { + + // SapMachine 2021-05-21: If any one of the xxxOnOutOfMemoryError is specified, + // print stack to stdout. Do this before any subsequent handling - this is the + // most important information. + if ((HeapDumpOnOutOfMemoryError) || + (OnOutOfMemoryError && OnOutOfMemoryError[0]) || + CrashOnOutOfMemoryError || ExitOnOutOfMemoryError) { + VMError::print_stack(tty); + } + + // SapMachine 2021-05-21: If we crash due to CrashOnOutOfMemoryError, deactivate + // cores unless they had been explicitly enabled. + if (CrashOnOutOfMemoryError && FLAG_IS_DEFAULT(CreateCoredumpOnCrash)) { + FLAG_SET_ERGO(CreateCoredumpOnCrash, false); + } + // create heap dump before OnOutOfMemoryError commands are executed if (HeapDumpOnOutOfMemoryError) { tty->print_cr("java.lang.OutOfMemoryError: %s", message); diff --git a/src/hotspot/share/utilities/vmError.cpp b/src/hotspot/share/utilities/vmError.cpp index 0b3ccf9c747..31ef1af6367 100644 --- a/src/hotspot/share/utilities/vmError.cpp +++ b/src/hotspot/share/utilities/vmError.cpp @@ -69,12 +69,24 @@ #include "utilities/macros.hpp" #include "utilities/ostream.hpp" #include "utilities/vmError.hpp" +// SapMachine 2019-02-20: Vitals +#include "vitals/vitals.hpp" #if INCLUDE_JFR #include "jfr/jfr.hpp" #endif #if INCLUDE_JVMCI #include "jvmci/jvmci.hpp" #endif +#ifdef LINUX +// SapMachine 2019-02-20: Vitals +#include "vitals_linux_himemreport.hpp" +// SapMachine 2021-09-01: malloc-trace +#include "malloctrace/mallocTrace.hpp" +#endif +// SapMachine 2023-08-15: malloc trace +#if defined(LINUX) || defined(__APPLE__) +#include "malloctrace/mallocTracePosix.hpp" +#endif #ifndef PRODUCT #include @@ -1284,12 +1296,15 @@ void VMError::report(outputStream* st, bool _verbose) { Arguments::print_on(st); st->cr(); + // SapMachine 2021-09-07: + // - print all values, not only non-default + // - comments are unnecessary bloat STEP_IF("printing flags", _verbose) JVMFlag::printFlags( st, - true, // with comments + false, // with comments false, // no ranges - true); // skip defaults + false); // skip defaults st->cr(); STEP_IF("printing warning if internal testing API used", WhiteBox::used()) @@ -1321,6 +1336,23 @@ void VMError::report(outputStream* st, bool _verbose) { NativeHeapTrimmer::print_state(st); st->cr(); + // SapMachine 2019-02-20: Vitals + STEP("Vitals") + if (_verbose) { + sapmachine_vitals::print_info_t info; + sapmachine_vitals::default_settings(&info); + info.sample_now = true; + st->print_cr("Vitals:"); + sapmachine_vitals::print_report(st, &info); + } + +#ifdef LINUX + STEP("Vitals HiMemReport") + st->cr(); + sapmachine_vitals::print_himemreport_state(st); + st->cr(); +#endif // LINUX + STEP_IF("printing system", _verbose) st->print_cr("--------------- S Y S T E M ---------------"); st->cr(); @@ -1341,6 +1373,17 @@ void VMError::report(outputStream* st, bool _verbose) { st->print_cr("vm_info: %s", VM_Version::internal_vm_info_string()); st->cr(); + // SapMachine 2021-09-01: malloc-trace +#if defined(LINUX) && defined(HAVE_GLIBC_MALLOC_HOOKS) + STEP("printing Malloc Trace info") + + if (_verbose) { + st->print_cr("sapmachine malloc trace"); + sap::MallocTracer::print_on_error(st); + st->cr(); + } +#endif + // print a defined marker to show that error handling finished correctly. STEP_IF("printing end marker", _verbose) st->print_cr("END."); @@ -1497,6 +1540,21 @@ void VMError::print_vm_info(outputStream* st) { st->cr(); + // SapMachine 2019-02-20: Vitals + // STEP("Vitals") + sapmachine_vitals::print_info_t info; + sapmachine_vitals::default_settings(&info); + info.sample_now = true; + st->print_cr("Vitals:"); + sapmachine_vitals::print_report(st, &info); + +#ifdef LINUX + // STEP("Vitals HiMemReport") + st->cr(); + sapmachine_vitals::print_himemreport_state(st); + st->cr(); +#endif // LINUX + // STEP("printing system") st->print_cr("--------------- S Y S T E M ---------------"); st->cr(); @@ -1521,6 +1579,12 @@ void VMError::print_vm_info(outputStream* st) { st->print_cr("vm_info: %s", VM_Version::internal_vm_info_string()); st->cr(); +#if defined(LINUX) && defined(HAVE_GLIBC_MALLOC_HOOKS) + // SapMachine 2021-09-01: malloc-trace + st->print_cr("sapmachine malloc trace"); + sap::MallocTracer::print_on_error(st); +#endif + // print a defined marker to show that error handling finished correctly. // STEP("printing end marker") @@ -1620,6 +1684,11 @@ void VMError::report_and_die(int id, const char* message, const char* detail_fmt Thread* thread, address pc, void* siginfo, void* context, const char* filename, int lineno, size_t size) { +#if defined(LINUX) || defined(__APPLE__) + // SapMachine 2023-09-18: Make sure we don't track allocations anymore. + sap::MallocStatistic::shutdown(); +#endif + // A single scratch buffer to be used from here on. // Do not rely on it being preserved across function calls. static char buffer[O_BUFLEN]; @@ -2169,3 +2238,13 @@ VMErrorCallbackMark::~VMErrorCallbackMark() { assert(_thread->_vm_error_callbacks != nullptr, "Popped too far"); _thread->_vm_error_callbacks = _thread->_vm_error_callbacks->_next; } + +// SapMachine 2021-05-21: A wrapper for VMError::print_stack_trace(..), public, for printing stacks +// to tty on CrashOnOutOfMemoryError +void VMError::print_stack(outputStream* st) { + Thread* t = Thread::current_or_null_safe(); + char buf[1024]; + if (t != NULL && t->is_Java_thread()) { + VMError::print_stack_trace(st, (JavaThread*) t, buf, sizeof(buf), false); + } +} diff --git a/src/hotspot/share/utilities/vmError.hpp b/src/hotspot/share/utilities/vmError.hpp index dee8335afd5..79b8432a09f 100644 --- a/src/hotspot/share/utilities/vmError.hpp +++ b/src/hotspot/share/utilities/vmError.hpp @@ -225,6 +225,10 @@ class VMError : public AllStatic { // permissions. static int prepare_log_file(const char* pattern, const char* default_pattern, bool overwrite_existing, char* buf, size_t buflen); + // SapMachine 2021-05-21: A wrapper for VMError::print_stack_trace(..), public, for printing stacks + // to tty on CrashOnOutOfMemoryError + static void print_stack(outputStream* st); + }; class VMErrorCallback { diff --git a/src/hotspot/share/vitals/vitals.cpp b/src/hotspot/share/vitals/vitals.cpp new file mode 100644 index 00000000000..3860e0929c8 --- /dev/null +++ b/src/hotspot/share/vitals/vitals.cpp @@ -0,0 +1,1456 @@ +/* + * Copyright (c) 2019, 2023 SAP SE. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "jvm_io.h" + +#include "gc/shared/collectedHeap.hpp" +#include "classfile/classLoaderDataGraph.inline.hpp" +#include "code/codeCache.hpp" +#include "logging/log.hpp" +#include "memory/allocation.hpp" +#include "memory/metaspace.hpp" +#include "memory/metaspaceUtils.hpp" +#include "memory/universe.hpp" +#include "nmt/mallocTracker.hpp" +#include "nmt/memBaseline.hpp" +#include "nmt/memTracker.hpp" +#include "runtime/os.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/nonJavaThread.hpp" +#include "runtime/thread.hpp" +#include "runtime/threads.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" +#include "utilities/ostream.hpp" +#include "vitals/vitals.hpp" +#include "vitals/vitals_internals.hpp" +#include "vitals/vitalsLocker.hpp" + +#include +#include + +// Newer JDKS: NMT is always on and this macro does not exist +// Older JDKs: NMT can be off at compile time; in that case INCLUDE_NMT +// will be defined=0 via CFLAGS; or on, in that case it will be defined=1 in macros.hpp. +#ifndef INCLUDE_NMT +#define INCLUDE_NMT 1 +#endif + +namespace sapmachine_vitals { + +static Lock g_vitals_lock("VitalsLock"); + +namespace counters { + +static volatile size_t g_number_of_clds = 0; +static volatile size_t g_number_of_anon_clds = 0; +static volatile size_t g_classes_loaded = 0; +static volatile size_t g_classes_unloaded = 0; +static volatile size_t g_threads_created = 0; + +void inc_cld_count(bool is_anon_cld) { + Atomic::inc(&g_number_of_clds); + if (is_anon_cld) { + Atomic::inc(&g_number_of_anon_clds); + } +} + +void dec_cld_count(bool is_anon_cld) { + Atomic::dec(&g_number_of_clds); + if (is_anon_cld) { + Atomic::dec(&g_number_of_anon_clds); + } +} + +void inc_classes_loaded(size_t count) { + Atomic::add(&g_classes_loaded, count); +} + +void inc_classes_unloaded(size_t count) { + Atomic::add(&g_classes_unloaded, count); +} + +void inc_threads_created(size_t count) { + Atomic::add(&g_threads_created, count); +} + +} // namespace counters + +// helper function for the missing outputStream::put(int c, int repeat) +static void ostream_put_n(outputStream* st, int c, int repeat) { + for (int i = 0; i < repeat; i ++) { + st->put(c); + } +} + +/////// class Sample ///// + +int Sample::num_values() { return ColumnList::the_list()->num_columns(); } + +size_t Sample::size_in_bytes() { + assert(num_values() > 0, "not yet initialized"); + return sizeof(Sample) + sizeof(value_t) * (num_values() - 1); // -1 since Sample::values is size 1 to shut up compilers about zero length arrays +} + +// Note: this is not to be used for regular samples, which live in a preallocated table. +Sample* Sample::allocate() { + Sample* s = (Sample*) NEW_C_HEAP_ARRAY(char, size_in_bytes(), mtInternal); + s->reset(); + return s; +} + +void Sample::reset() { + for (int i = 0; i < num_values(); i ++) { + set_value(i, INVALID_VALUE); + } + DEBUG_ONLY(_num = -1;) + _timestamp = 0; +} + +void Sample::set_value(int index, value_t v) { + assert(index >= 0 && index < num_values(), "invalid index"); + _values[index] = v; +} + +void Sample::set_timestamp(time_t t) { + _timestamp = t; +} + +#ifdef ASSERT +void Sample::set_num(int n) { + _num = n; +} +#endif + +value_t Sample::value(int index) const { + assert(index >= 0 && index < num_values(), "invalid index"); + return _values[index]; +} + +static void print_text_with_dashes(outputStream* st, const char* text, int width) { + assert(width > 0, "Sanity"); + // Print the name centered within the width like this + // ----- system ------ + int extra_space = width - (int)strlen(text); + if (extra_space > 0) { + int left_space = extra_space / 2; + int right_space = extra_space - left_space; + ostream_put_n(st, '-', left_space); + st->print_raw(text); + ostream_put_n(st, '-', right_space); + } else { + ostream_put_n(st, '-', width); + } +} + +// Helper function for printing: +// Print to ostream, but only if ostream is given. In any case return number of +// characters printed (or which would have been printed). +static +ATTRIBUTE_PRINTF(2, 3) +int printf_helper(outputStream* st, const char *fmt, ...) { + // We only print numbers, so a small buffer is fine. + char buf[128]; + va_list args; + int len = 0; + va_start(args, fmt); + len = jio_vsnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + // jio_vsnprintf guarantees -1 on truncation, and always zero termination if buffersize > 0. + assert(len >= 0, "Error, possible truncation. Increase bufsize?"); + if (len < 0) { // Handle in release too: just print a clear marker + jio_snprintf(buf, sizeof(buf), "!ERR!"); + len = (int)::strlen(buf); + } + if (st != NULL) { + st->print_raw(buf); + } + return len; +} + +// length of time stamp +#define TIMESTAMP_LEN 19 +// number of spaces after time stamp +#define TIMESTAMP_DIVIDER_LEN 3 +static void print_timestamp(outputStream* st, time_t t) { + struct tm _tm; + if (os::localtime_pd(&t, &_tm) == &_tm) { + char buf[32]; + ::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &_tm); + st->print("%*s", TIMESTAMP_LEN, buf); + } +} + + +////// ColumnWidths : a helper class for pre-calculating column widths to make a table align nicely. +// Keeps an array of ints, dynamically sized (since each platform has a different number of columns), +// and offers methods of auto-sizeing them to fit given samples (via dry-printing). +class ColumnWidths { + int _widths[64]; // Don't allocate dynamically, since we might not have enough memory when we use it. + +public: + + ColumnWidths() { + // Assert including the non-active columns, so we spot possible problems earlier. + assert(sizeof(_widths) / sizeof(_widths[0]) >= (size_t) Legend::the_legend()->nr_of_columns(), "array too small"); + + // Allocate array; initialize with the minimum required column widths (which is the + // size required to print the column header fully) + const Column* c = ColumnList::the_list()->first(); + while (c != NULL) { + _widths[c->index()] = (int)::strlen(c->name()); + c = c->next(); + } + } + + // given a sample (and an optional preceding sample for delta values), + // update widths to accommodate sample values (uses dry-printing) + void update_from_sample(const Sample* sample, const Sample* last_sample, const print_info_t* pi, int add_width = 0) { + const Column* c = ColumnList::the_list()->first(); + while (c != NULL) { + const int idx = c->index(); + const value_t v = sample->value(idx); + value_t v2 = INVALID_VALUE; + int age = -1; + if (last_sample != NULL) { + v2 = last_sample->value(idx); + age = sample->timestamp() - last_sample->timestamp(); + } + int needed = c->calc_print_size(v, v2, age, pi) + add_width; + if (_widths[idx] < needed) { + _widths[idx] = needed; + } + c = c->next(); + } + } + + int at(int index) const { + return _widths[index]; + } +}; + + +////// Legend /////////////////////////////////////////////// + +Legend* Legend::_the_legend = NULL; + +bool Legend::initialize() { + _the_legend = new Legend(); + return _the_legend != NULL; +} + +stringStream _legend; +stringStream _footnote; +static Legend* _the_legend; + +Legend::Legend() : _last_added_cat(NULL), _nr_of_columns(0) {} + +void Legend::add_column_info(const char* const category, const char* const header, + const char* const name, const char* const description) { + // Print category label if this column opens a new category + if ((_last_added_cat == NULL) || (::strcmp(_last_added_cat, category) != 0)) { + print_text_with_dashes(&_legend, category, 30); + _legend.cr(); + } + _last_added_cat = category; + _nr_of_columns++; + // print column name and description + const int min_width_column_label = 16; + char buf[32]; + if (header != NULL) { + jio_snprintf(buf, sizeof(buf), "%s-%s", header, name); + } else { + jio_snprintf(buf, sizeof(buf), "%s", name); + } + _legend.print_cr("%*s: %s", min_width_column_label, buf, description); +} + +void Legend::add_footnote(const char* text) { + _footnote.print_cr("%s", text); +} + +void Legend::print_on(outputStream* st) const { + st->print_raw(_legend.base()); + st->cr(); + st->print_raw(_footnote.base()); + st->print_cr("(Vitals version %X, pid: %d)", vitals_version, os::current_process_id()); +} + +////// ColumnList: a singleton class holding all information about all columns + +ColumnList* ColumnList::_the_list = NULL; + +bool ColumnList::initialize() { + _the_list = new ColumnList(); + return _the_list != NULL; +} + +void ColumnList::add_column(Column* c) { + assert(c->index() == -1, "Do not add twice."); + Column* c_last = _last; + if (_last != NULL) { + _last->_next = c; + _last = c; + } else { + _first = _last = c; + } + // fix indices (describe position of column within table/category/header + c->_idx = c->_idx_cat = c->_idx_hdr = 0; + if (c_last != NULL) { + c->_idx = c_last->_idx + 1; + if (::strcmp(c->category(), c_last->category()) == 0) { // same category as last column? + c->_idx_cat = c_last->_idx_cat + 1; + } + if (c->header() != NULL && c_last->header() != NULL && + ::strcmp(c_last->header(), c->header()) == 0) { // have header and same as last column? + c->_idx_hdr = c_last->_idx_hdr + 1; + } + } + _num_columns ++; +} + +//////////////////// + +static void print_category_line(outputStream* st, const ColumnWidths* widths, const print_info_t* pi) { + + assert(pi->csv == false, "Not in csv mode"); + ostream_put_n(st, ' ', TIMESTAMP_LEN + TIMESTAMP_DIVIDER_LEN); + + const Column* c = ColumnList::the_list()->first(); + assert(c != NULL, "no columns?"); + const char* last_category_text = NULL; + int width = 0; + + while(c != NULL) { + if (c->index_within_category_section() == 0) { + if (width > 0) { + // Print category label centered over the last n columns, surrounded by dashes. + print_text_with_dashes(st, last_category_text, width - 1); + st->put(' '); + } + width = 0; + } + width += widths->at(c->index()); + width += 1; // divider between columns + last_category_text = c->category(); + c = c->next(); + } + print_text_with_dashes(st, last_category_text, width - 1); + st->cr(); +} + +static void print_header_line(outputStream* st, const ColumnWidths* widths, const print_info_t* pi) { + + assert(pi->csv == false, "Not in csv mode"); + ostream_put_n(st, ' ', TIMESTAMP_LEN + TIMESTAMP_DIVIDER_LEN); + + const Column* c = ColumnList::the_list()->first(); + assert(c != NULL, "no columns?"); + const char* last_header_text = NULL; + int width = 0; + + while(c != NULL) { + if (c->index_within_header_section() == 0) { // First in header section + if (width > 0) { + if (last_header_text != NULL) { + // Print header label centered over the last n columns, surrounded by dashes. + print_text_with_dashes(st, last_header_text, width - 1); + st->put(' '); // divider + } else { + // the last n columns had no header. Just fill with blanks. + ostream_put_n(st, ' ', width); + } + } + width = 0; + } + width += widths->at(c->index()); + width += 1; // divider between columns + last_header_text = c->header(); + c = c->next(); + } + if (width > 0 && last_header_text != NULL) { + print_text_with_dashes(st, last_header_text, width - 1); + } + st->cr(); +} + +static void print_column_names(outputStream* st, const ColumnWidths* widths, const print_info_t* pi) { + + // Leave space for timestamp column + if (pi->csv == false) { + ostream_put_n(st, ' ', TIMESTAMP_LEN + TIMESTAMP_DIVIDER_LEN); + } else { + st->print_raw("time,"); + } + + const Column* c = ColumnList::the_list()->first(); + const Column* previous = NULL; + while (c != NULL) { + if (pi->csv == false) { + st->print("%-*s ", widths->at(c->index()), c->name()); + } else { // csv mode + // csv: use comma as delimiter, don't pad, and precede name with category/header + // (limited to 4 chars). + if (c->category() != NULL) { + st->print("%.4s-", c->category()); + } + if (c->header() != NULL) { + st->print("%.4s-", c->header()); + } + st->print("%s,", c->name()); + } + previous = c; + c = c->next(); + } + st->cr(); +} + +// Print a human readable size. +// byte_size: size, in bytes, to be printed. +// scale: K,M,G or 0 (dynamic) +// width: printing width. +static int print_memory_size(outputStream* st, size_t byte_size, size_t scale) { + + // If we forced a unit via scale=.. argument, we suppress display of the unit + // since we already know which unit is used. That saves horizontal space and + // makes automatic processing of the data easier. + bool dynamic_mode = false; + + if (scale == 0) { + dynamic_mode = true; + // Dynamic mode. Choose scale for this value. + if (byte_size == 0) { + scale = K; + } else { + if (byte_size >= G) { + scale = G; + } else if (byte_size >= M) { + scale = M; + } else { + scale = K; + } + } + } + + const char* display_unit = ""; + if (dynamic_mode) { + switch(scale) { + case K: display_unit = "k"; break; + case M: display_unit = "m"; break; + case G: display_unit = "g"; break; + default: + ShouldNotReachHere(); + } + } + + // How we display stuff: + // scale=1 (manually set) - print exact byte values without unit + // scale=0 (default, dynamic mode) - print values < 1024KB as "..k", <1024MB as "..m", "..g" above that + // - to distinguish between 0 and "almost 0" print very small values as "<1K" + // - print "k", "m" values with precision 0, "g" values with precision 1. + // scale=k,m or g (manually set) - print value divided by scale and without unit. No smart printing. + // Used mostly for automated processing, lets keep parsing simple. + + int l = 0; + if (scale == 1) { + // scale = 1 - print exact bytes + l = printf_helper(st, SIZE_FORMAT, byte_size); + + } else { + const float display_value = (float) byte_size / scale; + if (dynamic_mode) { + // dynamic scale + const int precision = scale >= G ? 1 : 0; + if (byte_size > 0 && byte_size < 1 * K) { + // very small but not zero. + assert(scale == K, "Sanity"); + l = printf_helper(st, "<1%s", display_unit); + } else { + l = printf_helper(st, "%.*f%s", precision, display_value, display_unit); + } + } else { + // fixed scale K, M or G + const int precision = 0; + l = printf_helper(st, "%.*f%s", precision, display_value, display_unit); + } + } + + return l; + +} + +///////// class Column and childs /////////// + +Column::Column(const char* category, const char* header, const char* name, const char* description, Extremum extremum) + : _category(category), + _header(header), // may be NULL + _name(name), + _description(description), + _extremum(extremum), + _next(NULL), _idx(-1), + _idx_cat(-1), _idx_hdr(-1) +{} + +void Column::print_value(outputStream* st, value_t value, value_t last_value, + int last_value_age, int min_width, const print_info_t* pi, char const* marker) const { + + // We print all values right aligned. + int needed = calc_print_size(value, last_value, last_value_age, pi); + if (pi->csv == false && min_width > needed) { + // In ascii (non csv) mode, pad to minimum width + ostream_put_n(st, ' ', min_width - needed); + } + // csv values shall be enclosed in quotes. + if (pi->csv) { + st->put('"'); + } + do_print(st, value, last_value, last_value_age, pi); + st->print_raw(marker); + if (pi->csv) { + st->put('"'); + } +} + +// Returns the number of characters this value needs to be printed. +int Column::calc_print_size(value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const { + return do_print(NULL, value, last_value, last_value_age, pi); +} + +int Column::do_print(outputStream* st, value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const { + if (value == INVALID_VALUE) { + if (pi->raw) { + if (st != NULL) { + return printf_helper(st, "%s", "?"); + } + return 1; + } else { + return 0; + } + } else { + if (pi->raw) { + return printf_helper(st, UINT64_FORMAT, value); + } else { + return do_print0(st, value, last_value, last_value_age, pi); + } + } +} + +int PlainValueColumn::do_print0(outputStream* st, value_t value, + value_t last_value, int last_value_age, const print_info_t* pi) const +{ + return printf_helper(st, UINT64_FORMAT, value); +} + +int DeltaValueColumn::do_print0(outputStream* st, value_t value, + value_t last_value, int last_value_age, const print_info_t* pi) const { + if (last_value > value) { + // we assume the underlying value to be monotonically raising, and that + // any negative delta would be just a fluke (e.g. counter overflows) + // we do not want to show + return 0; + } + if (last_value != INVALID_VALUE) { + return printf_helper(st, INT64_FORMAT, (int64_t)(value - last_value)); + } + return 0; +} + +int MemorySizeColumn::do_print0(outputStream* st, value_t value, + value_t last_value, int last_value_age, const print_info_t* pi) const { + return print_memory_size(st, value, pi->scale); +} + +int DeltaMemorySizeColumn::do_print0(outputStream* st, value_t value, + value_t last_value, int last_value_age, const print_info_t* pi) const { + if (last_value != INVALID_VALUE) { + return print_memory_size(st, value - last_value, pi->scale); + } + return 0; +} + +////////////// sample printing /////////////////////////// + +// Print one sample. +static void print_one_sample(outputStream* st, const Sample* sample, + const Sample* last_sample, const ColumnWidths* widths, const print_info_t* pi, int marked_index = -1, char const* mark = NULL) { + + // Print timestamp and divider + if (pi->csv) { + st->print("\""); + } + print_timestamp(st, sample->timestamp()); + if (pi->csv) { + st->print("\""); + } + + if (pi->csv == false) { + ostream_put_n(st, ' ', TIMESTAMP_DIVIDER_LEN); + } else { + st->put(','); + } + + const Column* c = ColumnList::the_list()->first(); + while (c != NULL) { + const int idx = c->index(); + const value_t v = sample->value(idx); + value_t v2 = INVALID_VALUE; + int age = -1; + if (last_sample != NULL) { + v2 = last_sample->value(idx); + age = sample->timestamp() - last_sample->timestamp(); + } + const int min_width = widths->at(idx) - (marked_index >= 0 ? 1 : 0); + c->print_value(st, v, v2, age, min_width, pi, + marked_index == idx ? mark : (marked_index >= 0 && !pi->csv ? " " : "")); + st->put(pi->csv ? ',' : ' '); + c = c->next(); + } + st->cr(); +} + +////////////// Class SampleTable ///////////////////////// + +// A fixed sized fifo buffer of n samples +class SampleTable : public CHeapObj { + + const int _num_entries; + int _head; // Index of the last sample written; -1 if none have been written yet + bool _did_wrap; + Sample* _samples; + +#ifdef ASSERT + void verify() const { + assert(_samples != NULL, "sanity"); + assert(_head >= 0 && _head < _num_entries, "sanity"); + } +#endif + + size_t sample_offset_in_bytes(int idx) const { + assert(idx >= 0 && idx <= _num_entries, "invalid index: %d", idx); + return Sample::size_in_bytes() * idx; + } + +public: + + SampleTable(int num_entries) + : _num_entries(num_entries), + _head(-1), + _did_wrap(false), + _samples(NULL) + { + _samples = (Sample*) NEW_C_HEAP_ARRAY(char, Sample::size_in_bytes() * _num_entries, mtInternal); +#ifdef ASSERT + for (int i = 0; i < _num_entries; i ++) { + sample_at(i)->reset(); + } +#endif + } + + bool is_empty() const { return _head == -1; } + + const Sample* sample_at(int index) const { return (Sample*)((uint8_t*)_samples + sample_offset_in_bytes(index)); } + Sample* sample_at(int index) { return (Sample*)((uint8_t*)_samples + sample_offset_in_bytes(index)); } + + void add_sample(const Sample* sample) { + // Advance head + _head ++; + if (_head == _num_entries) { + _did_wrap = true; + _head = 0; + } + // Copy sample + ::memcpy(sample_at(_head), sample, Sample::size_in_bytes()); + DEBUG_ONLY(verify()); + } + + // Given a valid sample index, return the previous index or -1 if this is the oldest sample. + int get_previous_index(int idx) const { + assert(idx >= 0 && idx <= _num_entries, "index oob: %d", idx); + assert(_did_wrap == true || idx <= _head, "index invalid: %d", idx); + int prev = idx - 1; + if (prev == -1 && _did_wrap) { + prev = _num_entries - 1; + } + if (prev == _head) { + prev = -1; + } + return prev; + } + + class Closure { + public: + virtual void do_sample(const Sample* sample, const Sample* previous_sample) = 0; + }; + + void call_closure_for_sample_at(Closure* closure, int idx) const { + const Sample* sample = sample_at(idx); + int idx2 = get_previous_index(idx); + const Sample* previous_sample = idx2 == -1 ? NULL : sample_at(idx2); + closure->do_sample(sample, previous_sample); + } + + void walk_table_locked(Closure* closure, bool youngest_to_oldest = true) const { + + if (_head == -1) { + return; + } + + DEBUG_ONLY(verify();) + + if (youngest_to_oldest) { // youngest to oldest + for (int pos = _head; pos >= 0; pos--) { + call_closure_for_sample_at(closure, pos); + } + if (_did_wrap) { + for (int pos = _num_entries - 1; pos > _head; pos--) { + call_closure_for_sample_at(closure, pos); + } + } + } else { // oldest to youngest + if (_did_wrap) { + for (int pos = _head + 1; pos < _num_entries; pos ++) { + call_closure_for_sample_at(closure, pos); + } + } + for (int pos = 0; pos <= _head; pos ++) { + call_closure_for_sample_at(closure, pos); + } + } + } + +}; + +class MeasureColumnWidthsClosure : public SampleTable::Closure { + const print_info_t* const _pi; + ColumnWidths* const _widths; + +public: + MeasureColumnWidthsClosure(const print_info_t* pi, ColumnWidths* widths) : + _pi(pi), _widths(widths) {} + + void do_sample(const Sample* sample, const Sample* previous_sample) { + _widths->update_from_sample(sample, previous_sample, _pi); + } +}; + +class PrintSamplesClosure : public SampleTable::Closure { + outputStream* const _st; + const print_info_t* const _pi; + const ColumnWidths* const _widths; + +public: + + PrintSamplesClosure(outputStream* st, const print_info_t* pi, const ColumnWidths* widths) : + _st(st), _pi(pi), _widths(widths) {} + + void do_sample(const Sample* sample, const Sample* previous_sample) { + print_one_sample(_st, sample, previous_sample, _widths, _pi); + } +}; + +// sampleTables is a combination of two tables: a short term table and a long term table. +// It takes care to feed new samples into these tables at the appropriate intervals. +class SampleTables: public CHeapObj { + + static int short_term_tablesize() { return (VitalsShortTermTableHours * 3600 / VitalsSampleInterval) + 1; } + static int long_term_tablesize() { return (VitalsLongTermTableDays * 24 * 60 / VitalsLongTermSampleIntervalMinutes) + 1; } + + SampleTable _short_term_table; + SampleTable _long_term_table; + SampleTable _extremum_samples; + SampleTable _last_extremum_samples; + + int _count; + int _large_table_count; + + static void print_table(const SampleTable* table, outputStream* st, + const ColumnWidths* widths, const print_info_t* pi) { + if (table->is_empty()) { + st->print_cr("(no samples)"); + return; + } + PrintSamplesClosure prclos(st, pi, widths); + table->walk_table_locked(&prclos, !pi->reverse_ordering); + } + + static void print_headers(outputStream* st, const ColumnWidths* widths, const print_info_t* pi) { + if (pi->csv == false) { + print_category_line(st, widths, pi); + print_header_line(st, widths, pi); + } + print_column_names(st, widths, pi); + } + + // Helper, print a time span given in seconds- + static void print_time_span(outputStream* st, int secs) { + const int mins = secs / 60; + const int hrs = secs / (60 * 60); + const int days = secs / (60 * 60 * 24); + if (days > 1) { + st->print_cr("Last %d days:", days); + } else if (hrs > 1) { + st->print_cr("Last %d hours:", hrs); + } else if (mins > 1) { + st->print_cr("Last %d minutes:", mins); + } else { + st->print_cr("Last %d seconds:", secs); + } + } + +public: + + SampleTables() + : _short_term_table(short_term_tablesize()), + _long_term_table(long_term_tablesize()), + _extremum_samples(Sample::num_values()), + _last_extremum_samples(Sample::num_values()), + _count(0), + _large_table_count(MAX2(1, (int) (VitalsLongTermSampleIntervalMinutes * 60 / VitalsSampleInterval))) + {} + + void add_sample(const Sample* sample) { + AutoLock autolock(&g_vitals_lock); + // Nothing we do in here blocks: the sample values are already taken, + // we only modify existing data structures (no memory is allocated either). + _short_term_table.add_sample(sample); + // Increment before feeding longterm table, in order for it to show up in reports + // only after an initial long term table interval has passed + _count++; + // Feed long term table + if ((_count % _large_table_count) == 0) { + _long_term_table.add_sample(sample); + } + + // Update exetremum samples if needed. + if (StoreVitalsExtremas) { + static Sample* last_sample = NULL; + + if (last_sample == NULL) { + // Nothing to do yet. We need at least two samples, since some types need the + // previous sample to print. Just allocate the space for the last sample as a marker + // for seeing the first sample. + last_sample = (Sample*) NEW_C_HEAP_ARRAY(char, Sample::size_in_bytes(), mtInternal); + } else if (_extremum_samples.is_empty()) { + // We already have a last sample and this is the second sample we see. + // We can initialize the two tables now to store the last sample and this sample + // for all extremas. + for (int i = 0; i < Sample::num_values(); ++i) { + _last_extremum_samples.add_sample(last_sample); + _extremum_samples.add_sample(sample); + } + } else { + // Iterate columns and update if needed. + for (Column const* column = ColumnList::the_list()->first(); column != NULL; column = column->next()) { + if (column->extremum() != NONE) { + int idx = column->index(); + Sample* extremum_sample = _extremum_samples.sample_at(idx); + + bool should_log = (column->extremum() == MAX) && (sample->value(idx) > extremum_sample->value(idx)); + should_log |= (column->extremum() == MIN) && (sample->value(idx) < extremum_sample->value(idx)); + should_log &= sample->value(idx) != INVALID_VALUE; + should_log |= extremum_sample->value(idx) == INVALID_VALUE; + + if (should_log) { + Sample* last_extremum_sample = _last_extremum_samples.sample_at(idx); + ::memcpy(last_extremum_sample, last_sample, Sample::size_in_bytes()); + ::memcpy(extremum_sample, sample, Sample::size_in_bytes()); + } + } + } + } + + // Remember the last sample. + ::memcpy(last_sample, sample, Sample::size_in_bytes()); + } + } + + void print_all(outputStream* st, const print_info_t* pi, const Sample* sample_now) { + + { // lock start + AutoLock autolock(&g_vitals_lock); + + if (sample_now != NULL) { + ColumnWidths widths; + MeasureColumnWidthsClosure mcwclos(pi, &widths); + widths.update_from_sample(sample_now, NULL, pi); + st->print_cr("Now:"); + print_headers(st, &widths, pi); + print_one_sample(st, sample_now, NULL, &widths, pi); + st->cr(); + } + + if (!_short_term_table.is_empty()) { + ColumnWidths widths; + MeasureColumnWidthsClosure mcwclos(pi, &widths); + _short_term_table.walk_table_locked(&mcwclos); + + if (pi->csv == false) { + print_time_span(st, VitalsShortTermTableHours * 3600); + } + print_headers(st, &widths, pi); + print_table(&_short_term_table, st, &widths, pi); + st->cr(); + } + + if (!_long_term_table.is_empty()) { + ColumnWidths widths; + MeasureColumnWidthsClosure mcwclos(pi, &widths); + _long_term_table.walk_table_locked(&mcwclos); + print_time_span(st, VitalsLongTermTableDays * 24 * 3600); + print_headers(st, &widths, pi); + print_table(&_long_term_table, st, &widths, pi); + st->cr(); + } + + if (StoreVitalsExtremas && !_extremum_samples.is_empty() && !_last_extremum_samples.is_empty()) { + st->print_cr("Samples at extremes (+ marks a maximum, - marks a minimum)"); + + ColumnWidths widths; + MeasureColumnWidthsClosure mcwclos(pi, &widths); + + for (Column const* column = ColumnList::the_list()->first(); column != NULL; column = column->next()) { + if (column->extremum() != NONE) { + Sample* extremum_sample = _extremum_samples.sample_at(column->index()); + Sample* last_extremum_sample = _last_extremum_samples.sample_at(column->index()); + widths.update_from_sample(extremum_sample, last_extremum_sample, pi, 1); + } + } + + print_headers(st, &widths, pi); // Need more space for the mark to display. + + for (Column const* column = ColumnList::the_list()->first(); column != NULL; column = column->next()) { + if (column->extremum() != NONE) { + Sample* extremum_sample = _extremum_samples.sample_at(column->index()); + Sample* last_extremum_sample = _last_extremum_samples.sample_at(column->index()); + print_one_sample(st, extremum_sample, last_extremum_sample, &widths, pi, column->index(), + column->extremum() == MIN ? "-" : "+"); + } + } + } + + st->cr(); + + } // lock end + } +}; + +static SampleTables* g_all_tables = NULL; + +/////////////// SAMPLING ////////////////////// + +// Samples all values, but leaves timestamp unchanged +static void sample_values(Sample* sample, bool avoid_locking) { + time_t t; + ::time(&t); + sample->set_timestamp(t); + DEBUG_ONLY(sample->set_num(-1);) + sample_jvm_values(sample, avoid_locking); + sample_platform_values(sample); +} + +class SamplerThread: public NamedThread { + + Sample* _sample; + bool _stop; + int _samples_taken; + int _jump_cooldown; + + static int get_sample_interval_ms() { + return (int)VitalsSampleInterval * 1000; + } + + void take_sample() { + _sample->reset(); + DEBUG_ONLY(_sample->set_num(_samples_taken);) + _samples_taken ++; + sample_values(_sample, VitalsLockFreeSampling); + g_all_tables->add_sample(_sample); + } + +public: + + SamplerThread() + : NamedThread(), + _sample(NULL), + _stop(false), + _samples_taken(0), + _jump_cooldown(0) + { + _sample = Sample::allocate(); + this->set_name("vitals sampler thread"); + } + + virtual void run() { + record_stack_base_and_size(); + for (;;) { + take_sample(); + os::naked_sleep(get_sample_interval_ms()); + if (_stop) { + break; + } + } + } + + void stop() { + _stop = true; + } + +}; + +static SamplerThread* g_sampler_thread = NULL; + +static bool initialize_sampler_thread() { + g_sampler_thread = new SamplerThread(); + if (g_sampler_thread != NULL) { + if (os::create_thread(g_sampler_thread, os::os_thread)) { + os::start_thread(g_sampler_thread); + } + return true; + } + return false; +} + + +/////////////////////////////////////// +/////// JVM-specific columns ////////// + +static Column* g_col_heap_committed = NULL; +static Column* g_col_heap_used = NULL; + +static Column* g_col_metaspace_committed = NULL; +static Column* g_col_metaspace_used = NULL; + +static bool g_show_classspace_columns = false; +static Column* g_col_classspace_committed = NULL; +static Column* g_col_classspace_used = NULL; + +static Column* g_col_metaspace_cap_until_gc = NULL; + +static Column* g_col_codecache_committed = NULL; + +static bool g_show_nmt_columns = false; +static Column* g_col_nmt_malloc = NULL; +static Column* g_col_nmt_mmap = NULL; +static Column* g_col_nmt_gc_overhead = NULL; +static Column* g_col_nmt_other = NULL; +static Column* g_col_nmt_overhead = NULL; + +static Column* g_col_number_of_java_threads = NULL; +static Column* g_col_number_of_java_threads_non_demon = NULL; + +static bool g_show_size_thread_stacks_col = false; +static Column* g_col_size_thread_stacks = NULL; + +static Column* g_col_number_of_java_threads_created = NULL; + +static Column* g_col_number_of_clds = NULL; +static Column* g_col_number_of_anon_clds = NULL; + +static Column* g_col_number_of_classes = NULL; +static Column* g_col_number_of_class_loads = NULL; +static Column* g_col_number_of_class_unloads = NULL; + +static bool is_nmt_enabled() { +#if INCLUDE_NMT + // Note: JDK version dependency: Before JDK18, NMT had the ability to shut down operations + // at any point in time, and therefore we also had NMT_minimal tracking level. Therefore, + // to stay version independent, we never must compare against NMT_off or NMT_minimal directly, + // only always against NMT_summary/detail. And to be very safe, don't assume a numerical + // value either, since older JDKs had NMT_unknown as a very high numerical value. + const NMT_TrackingLevel lvl = MemTracker::tracking_level(); + return (lvl == NMT_summary || lvl == NMT_detail); +#else + return false; +#endif +} + +static bool add_jvm_columns() { + // Order matters! + const char* const jvm_cat = "jvm"; + + Legend::the_legend()->add_footnote(" [delta]: values refer to the previous measurement."); + Legend::the_legend()->add_footnote(" [nmt]: only shown if NMT is available and activated"); + Legend::the_legend()->add_footnote(" [cs]: only shown on 64-bit if class space is active"); + Legend::the_legend()->add_footnote(" [linux]: only on Linux"); + + g_col_heap_committed = + define_column(jvm_cat, "heap", "comm", "Java Heap Size, committed", true); + g_col_heap_used = + define_column(jvm_cat, "heap", "used", "Java Heap Size, used", true); + + g_col_metaspace_committed = + define_column(jvm_cat, "meta", "comm", "Meta Space Size (class+nonclass), committed", true); + g_col_metaspace_used = + define_column(jvm_cat, "meta", "used", "Meta Space Size (class+nonclass), used", true); + + // Class space columns only shown if class space is active + g_show_classspace_columns = Metaspace::using_class_space(); + g_col_classspace_committed = + define_column(jvm_cat, "meta", "csc", "Class Space Size, committed [cs]", g_show_classspace_columns); + g_col_classspace_used = + define_column(jvm_cat, "meta", "csu", "Class Space Size, used [cs]", g_show_classspace_columns); + + g_col_metaspace_cap_until_gc = + define_column(jvm_cat, "meta", "gctr", "GC threshold", true); + + g_col_codecache_committed = + define_column(jvm_cat, NULL, "code", "Code cache, committed", true); + + // NMT columns only shown if NMT is at least at summary level + g_show_nmt_columns = is_nmt_enabled(); + g_col_nmt_malloc = + define_column(jvm_cat, "nmt", "mlc", "Memory malloced by hotspot [nmt]", g_show_nmt_columns); + g_col_nmt_mmap = + define_column(jvm_cat, "nmt", "map", "Memory mapped by hotspot [nmt]", g_show_nmt_columns); + g_col_nmt_gc_overhead = + define_column(jvm_cat, "nmt", "gc", "NMT \"gc\" (GC-overhead, malloc and mmap) [nmt]", g_show_nmt_columns); + g_col_nmt_other = + define_column(jvm_cat, "nmt", "oth", "NMT \"other\" (typically DBB or Unsafe.allocateMemory) [nmt]", g_show_nmt_columns); + g_col_nmt_overhead = + define_column(jvm_cat, "nmt", "ovh", "NMT overhead [nmt]", g_show_nmt_columns); + + g_col_number_of_java_threads = + define_column(jvm_cat, "jthr", "num", "Number of java threads", true); + g_col_number_of_java_threads_non_demon = + define_column(jvm_cat, "jthr", "nd", "Number of non-demon java threads", true); + g_col_number_of_java_threads_created = + define_column(jvm_cat, "jthr", "cr", "Threads created [delta]", true); + + // Displaying thread stack size for now only implemented on Linux, and requires NMT + g_show_size_thread_stacks_col = LINUX_ONLY(g_show_nmt_columns) NOT_LINUX(false); + g_col_size_thread_stacks = + define_column(jvm_cat, "jthr", "st", "Total reserved size of java thread stacks [nmt] [linux]", + g_show_size_thread_stacks_col); + + g_col_number_of_clds = + define_column(jvm_cat, "cldg", "num", "Classloader Data", true); + g_col_number_of_anon_clds = + define_column(jvm_cat, "cldg", "anon", "Anonymous CLD", true); + + g_col_number_of_classes = + define_column(jvm_cat, "cls", "num", "Classes (instance + array)", true); + + g_col_number_of_class_loads = + define_column(jvm_cat, "cls", "ld", "Class loaded [delta]", true); + + g_col_number_of_class_unloads = + define_column(jvm_cat, "cls", "uld", "Classes unloaded [delta]", true); + + return true; +} + + +////////// class ValueSampler and childs ///////////////// + +template +static void set_value_in_sample(const Column* col, Sample* sample, T t) { + if (col != NULL) { + int idx = col->index(); + assert(ColumnList::the_list()->is_valid_column_index(idx), "Invalid column index"); + sample->set_value(idx, (value_t)t); + } +} + +struct nmt_values_t { + // How much memory, in total, was committed via mmap + value_t mapped_total; + // How much memory, in total, was malloced + value_t malloced_total; + // How many allocations from malloc, in total + value_t malloced_num; + // thread stack size; depending on NMT version this would + // be reserved (I believe up to and including jdk 8) or committed (9+) + value_t thread_stacks_committed; + // NMT "GC" category (both malloced and mapped) + value_t gc_overhead; + // NMT "other" category (both malloced and mapped). Usually dominated by DBB allocated with allocateDirect(), + // and Unsafe.allocateMemory. + value_t other_memory; + // NMT overhead (both malloced and mapped) + value_t overhead; +}; + +static bool get_nmt_values(nmt_values_t* out) { +#if INCLUDE_NMT + if (is_nmt_enabled()) { + MutexLocker locker(MemTracker::query_lock()); + MemBaseline baseline; + baseline.baseline(true); + MallocMemorySnapshot* mlc_snapshot = baseline.malloc_memory_snapshot(); + VirtualMemorySnapshot vm_snapshot; + VirtualMemorySummary::snapshot(&vm_snapshot); + out->malloced_total = mlc_snapshot->total(); + out->mapped_total = vm_snapshot.total_committed(); + out->thread_stacks_committed = + vm_snapshot.by_type(mtThreadStack)->committed(); + out->thread_stacks_committed = + vm_snapshot.by_type(MEMFLAGS::mtThreadStack)->committed() + + mlc_snapshot->by_type(MEMFLAGS::mtThreadStack)->malloc_size(); + out->gc_overhead = + vm_snapshot.by_type(MEMFLAGS::mtGC)->committed() + + mlc_snapshot->by_type(MEMFLAGS::mtGC)->malloc_size(); + out->other_memory = + vm_snapshot.by_type(MEMFLAGS::mtOther)->committed() + + mlc_snapshot->by_type(MEMFLAGS::mtOther)->malloc_size(); + out->overhead = + vm_snapshot.by_type(MEMFLAGS::mtNMT)->committed() + + mlc_snapshot->by_type(MEMFLAGS::mtNMT)->malloc_size() + + mlc_snapshot->malloc_overhead(); + out->malloced_num = + mlc_snapshot->total_count(); + return true; + } +#endif // INCLUDE_NMT + return false; +} + +void sample_jvm_values(Sample* sample, bool avoid_locking) { + + // Note: if avoid_locking=true, skip values which need JVM-side locking. + + nmt_values_t nmt_vals; + bool have_nmt_values = false; + if (!avoid_locking) { + have_nmt_values = get_nmt_values(&nmt_vals); + } + + // Heap + if (!avoid_locking) { + size_t heap_cap = 0; + size_t heap_used = 0; + const CollectedHeap* const heap = Universe::heap(); + if (heap != NULL) { + MutexLocker hl(Heap_lock); + heap_cap = Universe::heap()->capacity(); + heap_used = Universe::heap()->used(); + } + set_value_in_sample(g_col_heap_committed, sample, heap_cap); + set_value_in_sample(g_col_heap_used, sample, heap_used); + } + + // Metaspace + set_value_in_sample(g_col_metaspace_committed, sample, MetaspaceUtils::committed_bytes()); + set_value_in_sample(g_col_metaspace_used, sample, MetaspaceUtils::used_bytes()); + + if (Metaspace::using_class_space()) { + set_value_in_sample(g_col_classspace_committed, sample, MetaspaceUtils::committed_bytes(Metaspace::ClassType)); + set_value_in_sample(g_col_classspace_used, sample, MetaspaceUtils::used_bytes(Metaspace::ClassType)); + } + + set_value_in_sample(g_col_metaspace_cap_until_gc, sample, MetaspaceGC::capacity_until_GC()); + + // Code cache + value_t codecache_committed = INVALID_VALUE; + if (!avoid_locking) { + MutexLocker lck(CodeCache_lock, Mutex::_no_safepoint_check_flag); + codecache_committed = CodeCache::capacity(); + } + set_value_in_sample(g_col_codecache_committed, sample, codecache_committed); + + // NMT integration + if (have_nmt_values) { + set_value_in_sample(g_col_nmt_malloc, sample, nmt_vals.malloced_total); + set_value_in_sample(g_col_nmt_mmap, sample, nmt_vals.mapped_total); + set_value_in_sample(g_col_nmt_gc_overhead, sample, nmt_vals.gc_overhead); + set_value_in_sample(g_col_nmt_other, sample, nmt_vals.other_memory); + set_value_in_sample(g_col_nmt_overhead, sample, nmt_vals.overhead); + } + + // Java threads + set_value_in_sample(g_col_number_of_java_threads, sample, Threads::number_of_threads()); + set_value_in_sample(g_col_number_of_java_threads_non_demon, sample, Threads::number_of_non_daemon_threads()); + set_value_in_sample(g_col_number_of_java_threads_created, sample, counters::g_threads_created); + + // Java thread stack size + if (have_nmt_values) { + set_value_in_sample(g_col_size_thread_stacks, sample, nmt_vals.thread_stacks_committed); + } + + // CLDG + set_value_in_sample(g_col_number_of_clds, sample, counters::g_number_of_clds); + set_value_in_sample(g_col_number_of_anon_clds, sample, counters::g_number_of_anon_clds); + + // Classes + set_value_in_sample(g_col_number_of_classes, sample, + ClassLoaderDataGraph::num_instance_classes() + ClassLoaderDataGraph::num_array_classes()); + set_value_in_sample(g_col_number_of_class_loads, sample, counters::g_classes_loaded); + set_value_in_sample(g_col_number_of_class_unloads, sample, counters::g_classes_unloaded); +} + +bool initialize() { + + static bool initialized = false; + assert(initialized == false, "Vitals already initialized"); + initialized = true; + + log_info(vitals)("Vitals v%x", vitals_version); + + bool success = ColumnList::initialize(); + success = success && Legend::initialize(); + + // Order matters. First platform columns, then jvm columns. + success = success && platform_columns_initialize(); + success = success && add_jvm_columns(); + + // -- Now the number of columns is known (and fixed). -- + + g_all_tables = new SampleTables(); + success = success && (g_all_tables != NULL); + + success = success && initialize_sampler_thread(); + + if (success) { + log_info(vitals)("Vitals initialized."); + log_debug(vitals)("Vitals sample interval: " UINTX_FORMAT " seconds.", VitalsSampleInterval); + } else { + log_warning(vitals)("Failed to initialize Vitals."); + } + + return success; + +} + +void cleanup() { + if (g_sampler_thread != NULL) { + g_sampler_thread->stop(); + } +} + +void default_settings(print_info_t* out) { + out->raw = false; + out->csv = false; + out->no_legend = false; + out->reverse_ordering = false; + out->scale = 0; + out->sample_now = false; +} + +void print_report(outputStream* st, const print_info_t* pinfo) { + + if (ColumnList::the_list() == NULL) { + st->print_cr(" (unavailable)"); + return; + } + + print_info_t info; + if (pinfo != NULL) { + info = *pinfo; + } else { + default_settings(&info); + } + + if (info.csv == false) { + st->cr(); + } + + // Print legend at the top (omit if suppressed on command line, or in csv mode). + if (info.no_legend == false && info.csv == false) { + Legend::the_legend()->print_on(st); + if (info.scale != 0) { + const char* display_unit = NULL; + switch (info.scale) { + case 1: display_unit = "bytes"; break; + case K: display_unit = "KB"; break; + case M: display_unit = "MB"; break; + case G: display_unit = "GB"; break; + default: ShouldNotReachHere(); + } + st->print_cr("[mem] values are in %s.", display_unit); + } + st->cr(); + } + + // If we are to sample the current values at print time, do that and print them too. + // Note: we omit the "Now" sample for csv output. + Sample* sample_now = NULL; + if (info.sample_now && !info.csv) { + sample_now = Sample::allocate(); + sample_values(sample_now, true /* never lock for now sample - be safe */ ); + } + + g_all_tables->print_all(st, &info, sample_now); + + os::free(sample_now); +} + +// Dump both textual and csv style reports to two files, "sapmachine_vitals_.txt" and "sapmachine_vitals_.csv". +// If these files exist, they are overwritten. +void dump_reports() { + + static const char* file_prefix = "sapmachine_vitals_"; + char vitals_file_name[1024]; + + if (VitalsFile != NULL) { + os::snprintf(vitals_file_name, sizeof(vitals_file_name), "%s.txt", VitalsFile); + } else { + os::snprintf(vitals_file_name, sizeof(vitals_file_name), "%s%d.txt", file_prefix, os::current_process_id()); + } + + // Note: we print two reports, both in reverse order (oldest to youngest). One in text form, one as csv. + + ::printf("Dumping Vitals to %s\n", vitals_file_name); + { + fileStream fs(vitals_file_name); + static const sapmachine_vitals::print_info_t settings = { + false, // raw + false, // csv + false, // no_legend + true, // reverse_ordering + 0, // scale + true // sample_now + }; + print_report(&fs, &settings); + } + + if (VitalsFile != NULL) { + os::snprintf(vitals_file_name, sizeof(vitals_file_name), "%s.csv", VitalsFile); + } else { + os::snprintf(vitals_file_name, sizeof(vitals_file_name), "%s%d.csv", file_prefix, os::current_process_id()); + } + ::printf("Dumping Vitals csv to %s\n", vitals_file_name); + { + fileStream fs(vitals_file_name); + static const sapmachine_vitals::print_info_t settings = { + false, // raw + true, // csv + false, // no_legend + true, // reverse_ordering + 1 * K, // scale + false // sample_now + }; + print_report(&fs, &settings); + } +} + +// For printing in thread lists only. +const Thread* samplerthread() { return g_sampler_thread; } + +} // namespace sapmachine_vitals diff --git a/src/hotspot/share/vitals/vitals.hpp b/src/hotspot/share/vitals/vitals.hpp new file mode 100644 index 00000000000..6054f1349b2 --- /dev/null +++ b/src/hotspot/share/vitals/vitals.hpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019, 2022 SAP SE. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef HOTSPOT_SHARE_VITALS_VITALS_HPP +#define HOTSPOT_SHARE_VITALS_VITALS_HPP + +#include "utilities/globalDefinitions.hpp" + +// Configure for different JDK versions +#define JDK_MAINLINE +//#define JDK17u +//#define JDK11u + +class outputStream; +class Thread; + +namespace sapmachine_vitals { + + bool initialize(); + void cleanup(); + + struct print_info_t { + bool raw; + bool csv; + // Omit printing a legend. + bool no_legend; + // Reverse printing order (default: youngest-to-oldest; reversed: oldest-to-youngest) + bool reverse_ordering; + + size_t scale; + + // if true, sample and print the current values too. If false, + // just print the sample tables. + bool sample_now; + + }; + + void default_settings(print_info_t* out); + + // Print report to stream. Leave print_info NULL for default settings. + void print_report(outputStream* st, const print_info_t* print_info = NULL); + + // Dump both textual and csv style reports to two files, "vitals_.txt" and "vitals_.csv". + // If these files exist, they are overwritten. + void dump_reports(); + + // For printing in thread lists only. + const Thread* samplerthread(); + + namespace counters { + void inc_cld_count(bool is_anon_cld); + void dec_cld_count(bool is_anon_cld); + void inc_classes_loaded(size_t count); + void inc_classes_unloaded(size_t count); + void inc_threads_created(size_t count); + }; + +}; + +#endif /* HOTSPOT_SHARE_VITALS_VITALS_HPP */ diff --git a/src/hotspot/share/vitals/vitalsDCmd.cpp b/src/hotspot/share/vitals/vitalsDCmd.cpp new file mode 100644 index 00000000000..3a320a9eb9c --- /dev/null +++ b/src/hotspot/share/vitals/vitalsDCmd.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019, 2023 SAP SE. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/resourceArea.hpp" +#include "utilities/ostream.hpp" +#include "utilities/globalDefinitions.hpp" +#include "vitals/vitals.hpp" +#include "vitals/vitalsDCmd.hpp" + +namespace sapmachine_vitals { + +VitalsDCmd::VitalsDCmd(outputStream* output, bool heap) + : DCmdWithParser(output, heap), + _scale("scale", "Memory usage in which to scale. Valid values are: k, m, g (fixed scale) " + "or \"dynamic\" for a dynamically chosen scale.", + "STRING", false, "dynamic"), + _csv("csv", "csv format.", "BOOLEAN", false, "false"), + _no_legend("no-legend", "Omit legend.", "BOOLEAN", false, "false"), + _reverse("reverse", "Reverse printing order.", "BOOLEAN", false, "false"), + _raw("raw", "Print raw values.", "BOOLEAN", false, "false"), + _sample_now("now", "Sample now values", "BOOLEAN", false, "false") +{ + _dcmdparser.add_dcmd_option(&_scale); + _dcmdparser.add_dcmd_option(&_csv); + _dcmdparser.add_dcmd_option(&_no_legend); + _dcmdparser.add_dcmd_option(&_reverse); + _dcmdparser.add_dcmd_option(&_raw); + _dcmdparser.add_dcmd_option(&_sample_now); +} + +static bool scale_from_name(const char* scale, size_t* out) { + if (strcasecmp(scale, "dynamic") == 0) { + *out = 0; + } else if (strcasecmp(scale, "1") == 0 || strcasecmp(scale, "b") == 0) { + *out = 1; + } else if (strcasecmp(scale, "kb") == 0 || strcasecmp(scale, "k") == 0) { + *out = K; + } else if (strcasecmp(scale, "mb") == 0 || strcasecmp(scale, "m") == 0) { + *out = M; + } else if (strcasecmp(scale, "gb") == 0 || strcasecmp(scale, "g") == 0) { + *out = G; + } else { + return false; // Invalid value + } + return true; +} + +void VitalsDCmd::execute(DCmdSource source, TRAPS) { + sapmachine_vitals::print_info_t info; + sapmachine_vitals::default_settings(&info); + if (!scale_from_name(_scale.value(), &(info.scale))) { + output()->print_cr("Invalid scale: \"%s\".", _scale.value()); + return; + } + info.csv = _csv.value(); + info.no_legend = _no_legend.value(); + info.reverse_ordering = _reverse.value(); + info.raw = _raw.value(); + info.sample_now = _sample_now.value(); + + output()->print_cr("Vitals:"); + if (info.sample_now && info.csv) { + output()->print_cr("(\"now\" ignored in csv mode)"); + } + sapmachine_vitals::print_report(output(), &info); +} + +} // namespace sapmachine_vitals diff --git a/src/hotspot/share/vitals/vitalsDCmd.hpp b/src/hotspot/share/vitals/vitalsDCmd.hpp new file mode 100644 index 00000000000..c971fc6feca --- /dev/null +++ b/src/hotspot/share/vitals/vitalsDCmd.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019, 2023 SAP SE. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef HOTSPOT_SHARE_VITALS_VITALSDCMD_HPP +#define HOTSPOT_SHARE_VITALS_VITALSDCMD_HPP + +#include "services/diagnosticCommand.hpp" + +namespace sapmachine_vitals { + +class VitalsDCmd : public DCmdWithParser { +protected: + DCmdArgument _scale; + DCmdArgument _csv; + DCmdArgument _no_legend; + DCmdArgument _reverse; + DCmdArgument _raw; + DCmdArgument _sample_now; +public: + static int num_arguments() { return 6; } + VitalsDCmd(outputStream* output, bool heap); + static const char* name() { + return "VM.vitals"; + } + static const char* description() { + return "Print Vitals."; + } + static const char* impact() { + return "Low."; + } + static const JavaPermission permission() { + JavaPermission p = { "java.lang.management.ManagementPermission", "monitor", NULL }; + return p; + } + virtual void execute(DCmdSource source, TRAPS); +}; + +} // namespace sapmachine_vitals + +#endif // HOTSPOT_SHARE_VITALS_VITALSDCMD_HPP diff --git a/src/hotspot/share/vitals/vitalsLocker.cpp b/src/hotspot/share/vitals/vitalsLocker.cpp new file mode 100644 index 00000000000..8958bd76c7f --- /dev/null +++ b/src/hotspot/share/vitals/vitalsLocker.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019, 2022 SAP SE. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "vitals/vitalsLocker.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/debug.hpp" + +#ifndef _WIN32 +#include +#endif + +namespace sapmachine_vitals { + +#ifdef _WIN32 + +Lock::Lock(const char* name) : _name(name) { + ::InitializeCriticalSection(&_lock); +} + +void Lock::lock() { + ::EnterCriticalSection(&_lock); +} + +void Lock::unlock() { + ::LeaveCriticalSection(&_lock); +} + +#else + +Lock::Lock(const char* name) : _name(name), _lock(PTHREAD_MUTEX_INITIALIZER) {} + +void Lock::lock() { + int rc = ::pthread_mutex_lock(&_lock); + assert(rc == 0, "%s: failed to grab lock (%d).", _name, errno); +} + +void Lock::unlock() { + int rc = ::pthread_mutex_unlock(&_lock); + assert(rc == 0, "%s: failed to release lock (%d).", _name, errno); +} + +#endif + + +}; // namespace sapmachine_vitals diff --git a/src/hotspot/share/vitals/vitalsLocker.hpp b/src/hotspot/share/vitals/vitalsLocker.hpp new file mode 100644 index 00000000000..7f01fdb31c0 --- /dev/null +++ b/src/hotspot/share/vitals/vitalsLocker.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2021, 2022 SAP SE. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef HOTSPOT_SHARE_VITALS_VITALSLOCKER_HPP +#define HOTSPOT_SHARE_VITALS_VITALSLOCKER_HPP + +// SapMachine 2021-10-14: I need a simple critical section. I don't +// need hotspot mutex error checking here, and I want to be independent of +// upstream changes to hotspot mutexes. + +#ifdef _WIN32 + #include +#else + #include +#endif + +namespace sapmachine_vitals { + +class Lock { + const char* const _name; +#ifdef _WIN32 + CRITICAL_SECTION _lock; +#else + pthread_mutex_t _lock; +#endif + +public: + Lock(const char* name); + void lock(); + void unlock(); +}; + +class AutoLock { + Lock* const _lock; +public: + AutoLock(Lock* lock) + : _lock(lock) + { + _lock->lock(); + } + ~AutoLock() { + _lock->unlock(); + } +}; + +}; + +#endif /* HOTSPOT_SHARE_VITALS_VITALS_HPP */ diff --git a/src/hotspot/share/vitals/vitals_internals.hpp b/src/hotspot/share/vitals/vitals_internals.hpp new file mode 100644 index 00000000000..5eab4047ed8 --- /dev/null +++ b/src/hotspot/share/vitals/vitals_internals.hpp @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2019, 2023 SAP SE. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef HOTSPOT_SHARE_VITALS_VITALS_INTERNALS_HPP +#define HOTSPOT_SHARE_VITALS_VITALS_INTERNALS_HPP + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" +#include "vitals/vitals.hpp" + +namespace sapmachine_vitals { + + static const int vitals_version = 0x220600; + + typedef uint64_t value_t; +#define INVALID_VALUE ((value_t)UINT64_MAX) + + class Sample { + DEBUG_ONLY(int _num;) + time_t _timestamp; + value_t _values[1]; // var sized + public: + static int num_values(); + static size_t size_in_bytes(); + static Sample* allocate(); + + void reset(); + void set_value(int index, value_t v); + void set_timestamp(time_t t); + DEBUG_ONLY(void set_num(int n);) + + value_t value(int index) const; + time_t timestamp() const { return _timestamp; } + DEBUG_ONLY(int num() const { return _num; }) + }; + + class ColumnList; + + enum Extremum { + NONE, + MAX, + MIN + }; + + class Column: public CHeapObj { + friend class ColumnList; + + const char* const _category; + const char* const _header; // optional. May be NULL. + const char* const _name; + const char* const _description; + const Extremum _extremum; + + // The following members are fixed by ColumnList when the Column is added to it. + Column* _next; // next column in table + int _idx; // position in table + int _idx_cat; // position in category + int _idx_hdr; // position under its header (if any, 0 otherwise) + + int do_print(outputStream* os, value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const; + + protected: + + Column(const char* category, const char* header, const char* name, const char* description, Extremum extremum); + + // Child classes implement this. + // output stream can be NULL; in that case, method shall return number of characters it would have printed. + virtual int do_print0(outputStream* os, value_t value, + value_t last_value, int last_value_age, const print_info_t* pi) const = 0; + + public: + + const char* category() const { return _category; } + const char* header() const { return _header; } + const char* name() const { return _name; } + const char* description() const { return _description; } + Extremum extremum() const { return _extremum; } + + void print_value(outputStream* os, value_t value, value_t last_value, + int last_value_age, int min_width, const print_info_t* pi, char const* marker) const; + + // Returns the number of characters this value needs to be printed. + int calc_print_size(value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const; + + // Returns the index (the position in the table) of this column. + int index() const { return _idx; } + int index_within_category_section() const { return _idx_cat; } + int index_within_header_section() const { return _idx_hdr; } + + const Column* next () const { return _next; } + + virtual bool is_memory_size() const { return false; } + + static Extremum extremum_default() { return NONE; } + }; + + // Some standard column types + + class PlainValueColumn: public Column { + int do_print0(outputStream* os, value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const; + public: + PlainValueColumn(const char* category, const char* header, const char* name, const char* description, Extremum extremum) + : Column(category, header, name, description, extremum) + {} + }; + + class DeltaValueColumn: public Column { + int do_print0(outputStream* os, value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const; + public: + // only_positive: only positive deltas are shown, negative deltas are supressed + DeltaValueColumn(const char* category, const char* header, const char* name, const char* description, Extremum extremum) + : Column(category, header, name, description, extremum) + {} + }; + + class MemorySizeColumn: public Column { + int do_print0(outputStream* os, value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const; + public: + MemorySizeColumn(const char* category, const char* header, const char* name, const char* description, Extremum extremum) + : Column(category, header, name, description, extremum) + {} + bool is_memory_size() const { return true; } + static Extremum extremum_default() { return MAX; } + }; + + class DeltaMemorySizeColumn: public Column { + int do_print0(outputStream* os, value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const; + public: + DeltaMemorySizeColumn(const char* category, const char* header, const char* name, const char* description, Extremum extremum) + : Column(category, header, name, description, extremum) + {} + }; + + class TimeStampColumn: public Column { + int do_print0(outputStream* os, value_t value, value_t last_value, + int last_value_age, const print_info_t* pi) const; + public: + TimeStampColumn(const char* category, const char* header, const char* name, const char* description, Extremum extremum) + : Column(category, header, name, description, extremum) + {} + }; + + ////// Legend: handles the legend + + class Legend: public CHeapObj { + stringStream _legend; + stringStream _footnote; + static Legend* _the_legend; + // needed during building the legend + const char* _last_added_cat; + int _nr_of_columns; + public: + Legend(); + void add_column_info(const char* const category, const char* const header, + const char* const name, const char* const description); + void add_footnote(const char* text); + void print_on(outputStream* st) const; + int nr_of_columns() { return _nr_of_columns; } + static Legend* the_legend () { return _the_legend; } + static bool initialize(); + }; + + ////// ColumnList: a singleton class holding all information about all columns + + class ColumnList: public CHeapObj { + + Column* _first, *_last; + int _num_columns; + + static ColumnList* _the_list; + + public: + + ColumnList() + : _first(NULL), _last(NULL), _num_columns(0) + {} + + const Column* first() const { return _first; } + int num_columns() const { return _num_columns; } + + void add_column(Column* column); + + static ColumnList* the_list () { return _the_list; } + + static bool initialize(); + +#ifdef ASSERT + bool is_valid_column_index(int idx) { + return idx >= 0 && idx < _num_columns; + } +#endif + + }; + + // Convenient method to define and register a possibly deactivated column + // (a deactivated column is not shown in the table, but still shown in the legend, to + // given the user a hint about it) + template + Column* define_column ( + const char* const category, const char* const header, + const char* const name, const char* const description, + bool is_active, Extremum extremum = ColumnType::extremum_default()) + { + Column* c = NULL; + if (is_active) { + c = new ColumnType(category, header, name, description, extremum); + ColumnList::the_list()->add_column(c); + } + Legend::the_legend()->add_column_info(category, header, name, description); + return c; + } + + // Ask platform to add platform specific columns + bool platform_columns_initialize(); + + void sample_platform_values(Sample* sample); + void sample_jvm_values(Sample* sample, bool avoid_locking); + +}; // namespace sapmachine_vitals + +#endif /* HOTSPOT_SHARE_VITALS_VITALS_INTERNALS_HPP */ diff --git a/src/java.base/macosx/native/libjli/java_md_macosx.m b/src/java.base/macosx/native/libjli/java_md_macosx.m index 279a7abef1e..5d857451c22 100644 --- a/src/java.base/macosx/native/libjli/java_md_macosx.m +++ b/src/java.base/macosx/native/libjli/java_md_macosx.m @@ -376,6 +376,32 @@ static void MacOSXStartup(int argc, char *argv[]) { } JLI_Snprintf(jvmcfg, so_jvmcfg, "%s%slib%sjvm.cfg", jrepath, FILESEP, FILESEP); + + /* SapMachine 2023-09-18: New malloc trace */ + if (ShouldPreloadLibMallocHooks(*pargc, *pargv)) { + char const* env_name = "DYLD_INSERT_LIBRARIES"; + char const* libpath = "/lib/libmallochooks.dylib"; + char const* old_env = getenv(env_name); + char* env_entry; + + if ((old_env == NULL) || (old_env[0] == '0')) { + size_t size = JLI_StrLen(jrepath) + JLI_StrLen(libpath) + JLI_StrLen(env_name) + 2; + env_entry = JLI_MemAlloc(size); + + snprintf(env_entry, size, "%s=%s%s", env_name, jrepath, libpath); + } else { + size_t size = JLI_StrLen(jrepath) + JLI_StrLen(libpath) + JLI_StrLen(env_name) + + 3 + JLI_StrLen(old_env); + env_entry = JLI_MemAlloc(size); + + snprintf(env_entry, size, "%s=%s%s:%s", env_name, jrepath, libpath, old_env); + } + + if (putenv(env_entry) == 0) { + execv(execname, argv); + } + } + /* Find the specified JVM type */ if (ReadKnownVMs(jvmcfg, JNI_FALSE) < 1) { JLI_ReportErrorMessage(CFG_ERROR7); diff --git a/src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java b/src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java index 153de588938..9c0645df5e8 100644 --- a/src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java +++ b/src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java @@ -81,6 +81,9 @@ static int getUserKeepAliveSeconds(String type) { userKeepAliveProxy = getUserKeepAliveSeconds("proxy"); } + // SapMachine 2024-04-12: Provide additional key field for KeepAliveCache entries (for FRUN) + public static final ThreadLocal connectionID = new ThreadLocal<>(); + /* maximum # keep-alive connections to maintain at once * This should be 2 by the HTTP spec, but because we don't support pipe-lining * a larger value is more appropriate. So we now set a default of 5, and the value @@ -367,6 +370,11 @@ private void readObject(ObjectInputStream stream) } class KeepAliveKey { + // SapMachine 2024-04-12: Provide additional key field for KeepAliveCache entries (for FRUN) + @SuppressWarnings("removal") + private static boolean useKeyExtension = AccessController.doPrivileged( + (PrivilegedAction)()->Boolean.getBoolean("com.sap.jvm.UseHttpKeepAliveCacheKeyExtension")); + private final String protocol; private final String host; private final int port; @@ -378,10 +386,25 @@ class KeepAliveKey { * @param url the URL containing the protocol, host and port information */ public KeepAliveKey(URL url, Object obj) { + // SapMachine 2024-04-12: Provide additional key field for KeepAliveCache entries (for FRUN) + final record KeyObject(String connectionID, Object obj) { + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other instanceof KeyObject ok) { + return (connectionID == null ? ok.connectionID == null : connectionID.equals(ok.connectionID)) && obj == ok.obj; + } else { + return false; + } + } + }; + this.protocol = url.getProtocol(); this.host = url.getHost(); this.port = url.getPort(); - this.obj = obj; + // SapMachine 2024-04-12: Provide additional key field for KeepAliveCache entries (for FRUN) + this.obj = useKeyExtension ? new KeyObject(KeepAliveCache.connectionID.get(), obj) : obj; } /** @@ -395,7 +418,8 @@ public boolean equals(Object obj) { return host.equals(kae.host) && (port == kae.port) && protocol.equals(kae.protocol) - && this.obj == kae.obj; + // SapMachine 2024-04-12: Provide additional key field for KeepAliveCache entries (for FRUN) + && useKeyExtension ? this.obj.equals(kae.obj) : this.obj == kae.obj; } /** diff --git a/src/java.base/share/conf/security/java.security b/src/java.base/share/conf/security/java.security index 5188362033d..e20457e822f 100644 --- a/src/java.base/share/conf/security/java.security +++ b/src/java.base/share/conf/security/java.security @@ -1267,7 +1267,7 @@ jceks.key.serialFilter = java.base/java.lang.Enum;java.base/java.security.KeyRep #keystore.pkcs12.macIterationCount = 10000 # -# Enhanced exception message information +# Enhanced exception message information (see SapMachine comment below!!) # # By default, exception messages should not include potentially sensitive # information such as file names, host names, or port numbers. This property @@ -1293,7 +1293,12 @@ jceks.key.serialFilter = java.base/java.lang.Enum;java.base/java.security.KeyRep # The property setting in this file can be overridden by a system property of # the same name, with the same syntax and possible values. # -#jdk.includeInExceptions=hostInfo,jar +# The SapMachine team considers these enhanced information in exception messages +# not an undue safety risk. The SapMachine team is of the opinion that the +# potential benefit of enhanced information in exception messages for support +# engineers outweighs the risk. Therefore this feature is switched on by default +# in SapMachine. +jdk.includeInExceptions=hostInfo,jar # # Disabled mechanisms for the Simple Authentication and Security Layer (SASL) diff --git a/src/java.base/share/data/cacerts/sapglobalrootca b/src/java.base/share/data/cacerts/sapglobalrootca new file mode 100644 index 00000000000..740d7eee566 --- /dev/null +++ b/src/java.base/share/data/cacerts/sapglobalrootca @@ -0,0 +1,43 @@ +Owner: CN=SAP Global Root CA, O=SAP AG, L=Walldorf, C=DE +Issuer: CN=SAP Global Root CA, O=SAP AG, L=Walldorf, C=DE +Serial number: 5d03d93d31615d8f488b3970c78f1b99 +Valid from: Thu Apr 26 15:41:55 GMT 2012 until: Mon Apr 26 15:46:27 GMT 2032 +Signature algorithm name: SHA256withRSA +Subject Public Key Algorithm: 4096-bit RSA key +Version: 3 +-----BEGIN CERTIFICATE----- +MIIGTDCCBDSgAwIBAgIQXQPZPTFhXY9Iizlwx48bmTANBgkqhkiG9w0BAQsFADBO +MQswCQYDVQQGEwJERTERMA8GA1UEBwwIV2FsbGRvcmYxDzANBgNVBAoMBlNBUCBB +RzEbMBkGA1UEAwwSU0FQIEdsb2JhbCBSb290IENBMB4XDTEyMDQyNjE1NDE1NVoX +DTMyMDQyNjE1NDYyN1owTjELMAkGA1UEBhMCREUxETAPBgNVBAcMCFdhbGxkb3Jm +MQ8wDQYDVQQKDAZTQVAgQUcxGzAZBgNVBAMMElNBUCBHbG9iYWwgUm9vdCBDQTCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOrxJKFFA1eTrZg1Ux8ax6n/ +LQRHZlgLc2FZpfyAgwvkt71wLkPLiTOaRb3Bd1dyydpKcwJLy0dzGkunzNkPRSFz +bKy2IPS0RS45hUCCPzhGnqQM6TcDYWeWpSUvygqujgb/cAG0mSJpvzAD3SMDQ+VJ +Az5Ryq4IrP7LkfCb63LKZxLsHEkEcNKoGPsSsd4LTwuEIyM3ZHcCoA97m6hvgLWV +GLzLIQMEblkswqX29z7JZH+zJopoqZB6eEogE2YpExkw52PufytEslDY3dyVubjp +GlvD4T03F2zm6CYleMwgWbATLVYvk2I9WfqPAP+ln2IU9DZzegSMTWHCE+jizaiq +b5f5s7m8f+cz7ndHSrz8KD/S9iNdWpuSlknHDrh+3lFTX/uWNBRs5mC/cdejcqS1 +v6erflyIfqPWWO6PxhIs49NL9Lix3ou6opJo+m8K757T5uP/rQ9KYALIXvl2uFP7 +0CqI+VGfossMlSXa1keagraW8qfplz6ffeSJQWO/+zifbfsf0tzUAC72zBuO0qvN +E7rSbqAfpav/o010nKP132gbkb4uOkUfZwCuvZjA8ddsQ4udIBRj0hQlqnPLJOR1 +PImrAFC3PW3NgaDEo9QAJBEp5jEJmQghNvEsmzXgABebwLdI9u0VrDz4mSb6TYQC +XTUaSnH3zvwAv8oMx7q7AgMBAAGjggEkMIIBIDAOBgNVHQ8BAf8EBAMCAQYwEgYD +VR0TAQH/BAgwBgEB/wIBATAdBgNVHQ4EFgQUg8dB/Q4mTynBuHmOhnrhv7XXagMw +gdoGA1UdIASB0jCBzzCBzAYKKwYBBAGFNgRkATCBvTAmBggrBgEFBQcCARYaaHR0 +cDovL3d3dy5wa2kuY28uc2FwLmNvbS8wgZIGCCsGAQUFBwICMIGFHoGCAEMAZQBy +AHQAaQBmAGkAYwBhAHQAZQAgAFAAbwBsAGkAYwB5ACAAYQBuAGQAIABDAGUAcgB0 +AGkAZgBpAGMAYQB0AGkAbwBuACAAUAByAGEAYwB0AGkAYwBlACAAUwB0AGEAdABl +AG0AZQBuAHQAIABvAGYAIABTAEEAUAAgAEEARzANBgkqhkiG9w0BAQsFAAOCAgEA +0HpCIaC36me6ShB3oHDexA2a3UFcU149nZTABPKT+yUCnCQPzvK/6nJUc5I4xPfv +2Q8cIlJjPNRoh9vNSF7OZGRmWQOFFrPWeqX5JA7HQPsRVURjJMeYgZWMpy4t1Tof +lF13u6OY6xV6A5kQZIISFj/dOYLT3+O7wME5SItL+YsNh6BToNU0xAZt71Z8JNdY +VJb2xSPMzn6bNXY8ioGzHlVxfEvzMqebV0KY7BTXR3y/Mh+v/RjXGmvZU6L/gnU7 +8mTRPgekYKY8JX2CXTqgfuW6QSnJ+88bHHMhMP7nPwv+YkPcsvCPBSY08ykzFATw +SNoKP1/QFtERVUwrUXt3Cufz9huVysiy23dEyfAglgCCRWA+ZlaaXfieKkUWCJaE +Kw/2Jqz02HDc7uXkFLS1BMYjr3WjShg1a+ulYvrBhNtseRoZT833SStlS/jzZ8Bi +c1dt7UOiIZCGUIODfcZhO8l4mtjh034hdARLF0sUZhkVlosHPml5rlxh+qn8yJiJ +GJ7CUQtNCDBVGksVlwew/+XnesITxrDjUMu+2297at7wjBwCnO93zr1/wsx1e2Um +Xn+IfM6K/pbDar/y6uI9rHlyWu4iJ6cg7DAPJ2CCklw/YHJXhDHGwheO/qSrKtgz +PGHZoN9jcvvvWDLUGtJkEotMgdFpEA2XWR83H4fVFVc= +-----END CERTIFICATE----- diff --git a/src/java.base/share/native/libjli/java.c b/src/java.base/share/native/libjli/java.c index 1b4ece834d6..121fa20b358 100644 --- a/src/java.base/share/native/libjli/java.c +++ b/src/java.base/share/native/libjli/java.c @@ -920,6 +920,71 @@ CheckJvmType(int *pargc, char ***argv, jboolean speculative) { return jvmtype; } +// SapMachine 2023-09-18: new malloc trace +jboolean ShouldPreloadLibMallocHooks(int argc, char **argv) { +#if defined(__APPLE__) || defined(LINUX) + jboolean uses_new_trace = JNI_FALSE; + jboolean uses_old_trace = JNI_FALSE; +#if defined(__APPLE__) + char const* env_name = "DYLD_INSERT_LIBRARIES"; + char const* libpath = "libmallochooks.dylib"; +#else + char const* env_name = "LD_PRELOAD"; + char const* libpath = "libmallochooks.so"; +#endif + + char const* old_env = getenv(env_name); + + /* Check if we have already preloaded the lib. We don't catch + all possble ways in which it could have been added (e.g. symlinks), + but this should be no problem. */ + if (old_env != NULL) { + size_t len = JLI_StrLen(libpath); + char const* pos = old_env; + + while ((pos = JLI_StrStr(pos, libpath)) != NULL) { + if ((pos[len] == ':') || (pos[len] == '\0')) { + if ((pos == old_env) || (pos[-1] == '/') || (pos[-1] == ':')) { + // Already preloaded, so we don't have to. + return JNI_FALSE; + } + } + } + } + + for (int i = 1; i < argc; i++) { + char const* arg = argv[i]; + + if ((JLI_StrCmp("-XX:+UseMallocHooks", arg) == 0) || + (JLI_StrCmp("-J-XX:+UseMallocHooks", arg) == 0)) { + uses_new_trace = JNI_TRUE; + continue; + } + + if ((JLI_StrCmp("-XX:+EnableMallocTrace", arg) == 0) || + (JLI_StrCmp("-J-XX:+EnableMallocTrace", arg) == 0)) { + uses_old_trace = JNI_TRUE; + continue; + } + + if (!IsJavaArgs()) { + if (IsWhiteSpaceOption(arg)) { + i += 1; + continue; + } + + if (arg[0] != '-') { + break; + } + } + } + + return (uses_new_trace && !uses_old_trace) ? JNI_TRUE : JNI_FALSE; +#endif + + return JNI_FALSE; +} + /* copied from HotSpot function "atomll()" */ static int parse_size(const char *s, jlong *result) { diff --git a/src/java.base/share/native/libjli/java.h b/src/java.base/share/native/libjli/java.h index f768b58a001..a7086cc6554 100644 --- a/src/java.base/share/native/libjli/java.h +++ b/src/java.base/share/native/libjli/java.h @@ -172,6 +172,9 @@ void AddOption(char *str, void *info); jboolean IsWhiteSpaceOption(const char* name); jlong CurrentTimeMicros(); +// SapMachine 2023-09-18: new malloc trace +jboolean ShouldPreloadLibMallocHooks(int argc, char **argv); + // Utility function defined in args.c int isTerminalOpt(char *arg); jboolean IsJavaw(); diff --git a/src/java.base/unix/native/libjli/java_md.c b/src/java.base/unix/native/libjli/java_md.c index 2a1a4e7795a..8069443535c 100644 --- a/src/java.base/unix/native/libjli/java_md.c +++ b/src/java.base/unix/native/libjli/java_md.c @@ -346,6 +346,36 @@ CreateExecutionEnvironment(int *pargc, char ***pargv, mustsetenv = RequiresSetenv(jvmpath); JLI_TraceLauncher("mustsetenv: %s\n", mustsetenv ? "TRUE" : "FALSE"); + /* SapMachine 2023-09-18: new malloc trace */ +#if defined(LINUX) + if (ShouldPreloadLibMallocHooks(*pargc, *pargv) == JNI_TRUE) { + char const* env_name = "LD_PRELOAD"; + char const* libpath = "/lib/libmallochooks.so"; + char const* old_env = getenv(env_name); + char* env_entry; + + if ((old_env == NULL) || (old_env[0] == '0')) { + size_t size = JLI_StrLen(jrepath) + JLI_StrLen(libpath) + JLI_StrLen(env_name) + 2; + env_entry = JLI_MemAlloc(size); + + snprintf(env_entry, size, "%s=%s%s", env_name, jrepath, libpath); + } else { + size_t size = JLI_StrLen(jrepath) + JLI_StrLen(libpath) + JLI_StrLen(env_name) + + 3 + JLI_StrLen(old_env); + env_entry = JLI_MemAlloc(size); + + snprintf(env_entry, size, "%s=%s%s:%s", env_name, jrepath, libpath, old_env); + } + + if (putenv(env_entry) == 0) { + /* We exec here, since the code below might return without exec. + * This can lead to double exec in the worst case, but we don't care. + */ + execve(execname, argv, environ); + } + } +#endif + if (mustsetenv == JNI_FALSE) { return; } diff --git a/src/java.base/unix/native/libmallochooks/mallochooks.c b/src/java.base/unix/native/libmallochooks/mallochooks.c new file mode 100644 index 00000000000..c42ac40c15c --- /dev/null +++ b/src/java.base/unix/native/libmallochooks/mallochooks.c @@ -0,0 +1,583 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__APPLE__) +#include +#else +#include +#endif + +#include "mallochooks.h" + +// The log level. 0 is none, 1 is basic logging. +#define LOG_LEVEL 0 + +// If > 0 we sync after each write. +#define SYNC_WRITE 0 + +void write_safe(int fd, char const* buf, size_t len) { + int errno_backup = errno; + size_t left = len; + ssize_t result; + + while ((result = write(fd, buf, left)) > 0) { + buf += result; + left -= result; + } + +#if SYNC_WRITE > 0 + fsync(fd); +#endif + + errno = errno_backup; +} + +static void print_error(char const* msg) { + write_safe(2, msg, strlen(msg)); +} + + +static void unexpected_call() { + print_error("Uninitialized function called. libmallochooks.so must be the first preloaded library.\n"); + exit(1); +} + +// The tag for malloc functions which should be loaded by dl_sym. +#define LOAD_DYNAMIC ((void*) unexpected_call) + +#if defined(__APPLE__) + +static real_malloc_funcs_t impl = { + malloc, + calloc, + realloc, + free, + posix_memalign, + NULL, + NULL, + valloc, + NULL, + (malloc_size_func_t*) malloc_size +}; + +#define REPLACE_NAME(x) x##_interpose + +#elif defined(__GLIBC__) + +void* __libc_malloc(size_t size); +void* __libc_calloc(size_t elems, size_t size); +void* __libc_realloc(void* ptr, size_t size); +void __libc_free(void* ptr); +void* __libc_memalign(size_t align, size_t size); +void* __libc_valloc(size_t size); +void* __libc_pvalloc(size_t size); + +static real_malloc_funcs_t impl = { + __libc_malloc, + __libc_calloc, + __libc_realloc, + __libc_free, + (posix_memalign_func_t*) LOAD_DYNAMIC, + __libc_memalign, + (aligned_alloc_func_t*) LOAD_DYNAMIC, + __libc_valloc, + __libc_pvalloc, + malloc_usable_size +}; + +#elif defined(MUSL_LIBC) + +static void* calloc_by_malloc(size_t elems, size_t size); +static int posix_memalign_by_aligned_alloc(void** ptr, size_t align, size_t size); +static void* memalign_by_aligned_alloc(size_t align, size_t size); + +static real_malloc_funcs_t impl = { + (malloc_func_t*) LOAD_DYNAMIC, + calloc_by_malloc, + (realloc_func_t*) LOAD_DYNAMIC, + (free_func_t*) LOAD_DYNAMIC, + posix_memalign_by_aligned_alloc, + memalign_by_aligned_alloc, + (aligned_alloc_func_t*) LOAD_DYNAMIC, + NULL, + NULL, + malloc_usable_size +}; + +/* musl calloc would call the redirected malloc, so we call the right malloc here. */ +static void* calloc_by_malloc(size_t elems, size_t size) { + /* Check for overflow */ + if (size > 0 && (elems > ((size_t) -1) / size)) { + errno = ENOMEM; + return NULL; + } + + void* result = impl.malloc(elems * size); + + if (result != NULL) { + bzero(result, elems * size); + } + + return result; +} + +/* musl posix_memalign would call the redirected aligned_alloc, so we call the right aligned_alloc here. */ +static int posix_memalign_by_aligned_alloc(void** ptr, size_t align, size_t size) { + if (align < sizeof(void *)) { + return EINVAL; + } + + void* result = impl.aligned_alloc(align, size); + + if (ptr != NULL) { + *ptr = result; + + return 0; + } + + return errno; +} + +/* musl memalign would call the redirected aligned_alloc, so we call the right aligned_alloc here. */ +static void* memalign_by_aligned_alloc(size_t align, size_t size) { + return impl.aligned_alloc(align, size); +} + +#else +#error "Unexpected platform" +#endif + +#if LOG_LEVEL > 0 + +static void print(char const* str); +static void print_ptr(void* ptr); +static void print_size(size_t size); + +#else + +#define print_ptr(x) +#define print_size(x) +#define print(x) + +#endif + +#ifndef REPLACE_NAME +#define REPLACE_NAME(x) x +#endif + +static void assign_function(void** dest, char const* symbol) { + if (*dest != LOAD_DYNAMIC) { + print("Don't need to load '"); + print(symbol); + print("'\n"); + + return; + } + + print("Resolving '"); + print(symbol); + print("'\n"); + + *dest = dlsym(RTLD_NEXT, symbol); + + if (*dest == NULL) { + print_error(symbol); + print_error(" not found!\n"); + exit(1); + } + + print("Found at "); + print_ptr(*dest); + print("\n"); +} + +#define LIB_INIT __attribute__((constructor)) +#define EXPORT __attribute__((visibility("default"))) + +static void LIB_INIT init(void) { + assign_function((void**) &impl.malloc, "malloc"); + assign_function((void**) &impl.calloc, "calloc"); + assign_function((void**) &impl.realloc, "realloc"); + assign_function((void**) &impl.free, "free"); + assign_function((void**) &impl.memalign, "memalign"); + assign_function((void**) &impl.posix_memalign, "posix_memalign"); + assign_function((void**) &impl.aligned_alloc, "aligned_alloc"); + assign_function((void**) &impl.valloc, "valloc"); + assign_function((void**) &impl.pvalloc, "pvalloc"); +} + +static registered_hooks_t empty_registered_hooks = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL +}; + +static registered_hooks_t* volatile registered_hooks = &empty_registered_hooks; + +EXPORT registered_hooks_t* malloc_hooks_register_hooks(registered_hooks_t* hooks) { + registered_hooks_t* old_hooks = registered_hooks; + + if (hooks == NULL) { + print("Deregistered hooks\n"); + registered_hooks = &empty_registered_hooks; + } else { + print("Registered hooks\n"); + registered_hooks = hooks; + } + + return old_hooks == &empty_registered_hooks ? NULL : old_hooks; +} + +EXPORT registered_hooks_t* malloc_hooks_active_hooks() { + if (registered_hooks == &empty_registered_hooks) { + return NULL; + } + + return (registered_hooks_t*) registered_hooks; +} + +EXPORT real_malloc_funcs_t* malloc_hooks_get_real_malloc_funcs() { + return &impl; +} + +#if LOG_LEVEL > 0 + +#define LOG_FUNC(func) \ + print(#func); + +#define LOG_ALIGN(align) \ + print(" alignment "); \ + print_size(align); + +#define LOG_PTR(ptr) \ + print(" "); \ + print_ptr(ptr); + +#define LOG_PTR_WITH_SIZE(ptr) \ + LOG_PTR(ptr); \ + if (ptr != NULL) { \ + size_t size = impl.malloc_size(ptr); \ + if (size > 0) { \ + print(" (size "); \ + print_size(size); \ + print(")"); \ + } \ + } + +#define LOG_ELEMS(elems) \ + print(" #elems "); \ + print_size(elems); + +#define LOG_SIZE(size) \ + print(" size "); \ + print_size(size); + +#define LOG_ALLOCATION_RESULT(result) \ + if (result == NULL) { \ + print(" failed with errno "); \ + print_size(errno); \ + } else { \ + print(" allocated at"); \ + LOG_PTR_WITH_SIZE(result); \ + } + +#define LOG_RESULT(result) \ + print(" result "); \ + print_size(result); + +#define LOG_HOOK \ + print(hook ? " with hook\n" : " without hook\n"); + +#else + +#define LOG_FUNC(func) +#define LOG_ALIGN(align) +#define LOG_PTR(ptr) +#define LOG_PTR_WITH_SIZE(ptr) +#define LOG_ELEMS(elems) +#define LOG_SIZE(size) +#define LOG_ALLOCATION_RESULT(result) +#define LOG_RESULT(result) +#define LOG_HOOK + +#endif + +EXPORT void* REPLACE_NAME(malloc)(size_t size) { + malloc_hook_t* hook = registered_hooks->malloc_hook; + void* result; + + LOG_FUNC(malloc); + LOG_SIZE(size); + + if (hook != NULL) { + result = hook(size, __builtin_return_address(0)); + } else { + result = impl.malloc(size); + } + + LOG_ALLOCATION_RESULT(result); + LOG_HOOK; + + return result; +} + +EXPORT void* REPLACE_NAME(calloc)(size_t elems, size_t size) { + calloc_hook_t* hook = registered_hooks->calloc_hook; + void* result; + + LOG_FUNC(calloc); + LOG_ELEMS(elems); + LOG_SIZE(size); + + if (hook != NULL) { + result = hook(elems, size, __builtin_return_address(0)); + } else { + result = impl.calloc(elems, size); + } + + LOG_ALLOCATION_RESULT(result); + LOG_HOOK; + + return result; +} + +EXPORT void* REPLACE_NAME(realloc)(void* ptr, size_t size) { + realloc_hook_t* hook = registered_hooks->realloc_hook; + void* result; + + LOG_FUNC(realloc); + LOG_PTR_WITH_SIZE(ptr); + LOG_SIZE(size); + + if (hook != NULL) { + result = hook(ptr, size, __builtin_return_address(0)); + } else { + result = impl.realloc(ptr, size); + } + + LOG_ALLOCATION_RESULT(result); + LOG_HOOK; + + return result; +} + +EXPORT void REPLACE_NAME(free)(void* ptr) { + free_hook_t* hook = registered_hooks->free_hook; + + LOG_FUNC(free); + LOG_PTR_WITH_SIZE(ptr); + + if (hook != NULL) { + hook(ptr, __builtin_return_address(0)); + } else { + impl.free(ptr); + } + + LOG_HOOK; +} + +EXPORT int REPLACE_NAME(posix_memalign)(void** ptr, size_t align, size_t size) { + posix_memalign_hook_t* hook = registered_hooks->posix_memalign_hook; + int result; + + LOG_FUNC(posix_memalign); + LOG_ALIGN(align); + LOG_SIZE(size); + + if (hook != NULL) { + result = hook(ptr, align, size, __builtin_return_address(0)); + } else { + result = impl.posix_memalign(ptr, align, size); + } + + LOG_ALLOCATION_RESULT(*ptr); + LOG_RESULT(result); + LOG_HOOK; + + return result; +} + +#if !defined(__APPLE__) +EXPORT void* REPLACE_NAME(memalign)(size_t align, size_t size) { + memalign_hook_t* hook = registered_hooks->memalign_hook; + void* result; + + LOG_FUNC(memalign); + LOG_ALIGN(align); + LOG_SIZE(size); + + if (hook != NULL) { + result = hook(align, size, __builtin_return_address(0)); + } else { + result = impl.memalign(align, size); + } + + LOG_ALLOCATION_RESULT(result); + LOG_HOOK; + + return result; +} +#endif + +EXPORT void* REPLACE_NAME(aligned_alloc)(size_t align, size_t size) { + memalign_hook_t* hook = registered_hooks->aligned_alloc_hook; + void* result; + + LOG_FUNC(aligned_alloc); + LOG_ALIGN(align); + LOG_SIZE(size); + + if (hook != NULL) { + result = hook(align, size, __builtin_return_address(0)); + } else { + result = impl.aligned_alloc(align, size); + } + + LOG_ALLOCATION_RESULT(result); + LOG_HOOK; + + return result; +} + +#if !defined(MUSL_LIBC) +EXPORT void* REPLACE_NAME(valloc)(size_t size) { + valloc_hook_t* hook = registered_hooks->valloc_hook; + void* result; + + LOG_FUNC(valloc); + LOG_SIZE(size); + + if (hook != NULL) { + result = hook(size, __builtin_return_address(0)); + } else { + result = impl.valloc(size); + } + + LOG_ALLOCATION_RESULT(result); + LOG_HOOK; + + return result; +} +#endif + +#if !defined(MUSL_LIBC) +EXPORT void* REPLACE_NAME(pvalloc)(size_t size) { + pvalloc_hook_t* hook = registered_hooks->pvalloc_hook; + void* result; + + LOG_FUNC(pvalloc); + LOG_SIZE(size); + + if (hook != NULL) { + result = hook(size, __builtin_return_address(0)); + } else { + result = impl.pvalloc(size); + } + + LOG_ALLOCATION_RESULT(result); + LOG_HOOK; + + return result; +} +#endif + +#if defined(__APPLE__) + +#define DYLD_INTERPOSE(_replacement,_replacee) \ + __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee \ + __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee }; + +DYLD_INTERPOSE(REPLACE_NAME(malloc), malloc) +DYLD_INTERPOSE(REPLACE_NAME(calloc), calloc) +DYLD_INTERPOSE(REPLACE_NAME(realloc), realloc) +DYLD_INTERPOSE(REPLACE_NAME(free), free) +DYLD_INTERPOSE(REPLACE_NAME(posix_memalign), posix_memalign) + +// We compile for 10.12 but aligned_alloc is only available in 10.15 and up +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability-new" +DYLD_INTERPOSE(REPLACE_NAME(aligned_alloc), aligned_alloc) +#pragma clang diagnostic pop + +DYLD_INTERPOSE(REPLACE_NAME(valloc), valloc) + +#endif + + +// D E B U G C O D E + + +#if LOG_LEVEL > 0 + +#define DEBUG_FD 2 + +static void print(char const* str) { + write_safe(DEBUG_FD, str, strlen(str)); +} + +static void print_ptr(void* ptr) { + char buf[18]; + int shift = 64; + buf[0] = '0'; + buf[1] = 'x'; + char* p = buf + 2; + print("0x"); + + do { + shift -= 4; + *p = "0123456789abcdef"[((((size_t) ptr) >> shift) & 15)]; + ++p; + } while (shift > 0); + + write_safe(DEBUG_FD, buf, p - buf); +} + +static void print_size(size_t size) { + char buf[20]; + size_t pos = sizeof(buf); + + do { + buf[--pos] = '0' + (size % 10); + size /= 10; + } while (size > 0); + + write_safe(DEBUG_FD, buf + pos, sizeof(buf) - pos); +} +#endif + diff --git a/src/java.base/unix/native/libmallochooks/mallochooks.h b/src/java.base/unix/native/libmallochooks/mallochooks.h new file mode 100644 index 00000000000..3ad02fe95ef --- /dev/null +++ b/src/java.base/unix/native/libmallochooks/mallochooks.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef __SAPMACHINE_MALLOC_HOOK +#define __SAPMACHINE_MALLOC_HOOK + +typedef void* malloc_func_t(size_t size); +typedef void* calloc_func_t(size_t elems, size_t size); +typedef void* realloc_func_t(void* ptr, size_t size); +typedef void free_func_t(void* ptr); +typedef int posix_memalign_func_t(void** ptr, size_t align, size_t size); +typedef void* memalign_func_t(size_t align, size_t size); +typedef void* aligned_alloc_func_t(size_t align, size_t size); +typedef void* valloc_func_t(size_t size); +typedef void* pvalloc_func_t(size_t size); + +typedef size_t malloc_size_func_t(void* ptr); +typedef void* malloc_hook_t(size_t size, void* caller); +typedef void* calloc_hook_t(size_t elems, size_t size, void* caller); +typedef void* realloc_hook_t(void* ptr, size_t size, void* caller); +typedef void free_hook_t(void* ptr, void* caller); +typedef int posix_memalign_hook_t(void** ptr, size_t align, size_t size, void* caller); +typedef void* memalign_hook_t(size_t align, size_t size, void* caller); +typedef void* aligned_alloc_hook_t(size_t align, size_t size, void* caller); +typedef void* valloc_hook_t(size_t size, void* caller); +typedef void* pvalloc_hook_t(size_t size, void* caller); + +typedef struct { + malloc_hook_t* malloc_hook; + calloc_hook_t* calloc_hook; + realloc_hook_t* realloc_hook; + free_hook_t* free_hook; + posix_memalign_hook_t* posix_memalign_hook; + memalign_hook_t* memalign_hook; + aligned_alloc_hook_t* aligned_alloc_hook; + valloc_hook_t* valloc_hook; + pvalloc_hook_t* pvalloc_hook; +} registered_hooks_t; + +typedef struct { + malloc_func_t* malloc; + calloc_func_t* calloc; + realloc_func_t* realloc; + free_func_t* free; + posix_memalign_func_t* posix_memalign; + memalign_func_t* memalign; + aligned_alloc_func_t* aligned_alloc; + valloc_func_t* valloc; + pvalloc_func_t* pvalloc; + malloc_size_func_t* malloc_size; +} real_malloc_funcs_t; + +typedef registered_hooks_t* register_hooks_t(registered_hooks_t* registered_hooks); +typedef registered_hooks_t* active_hooks_t(); +typedef real_malloc_funcs_t* get_real_malloc_funcs_t(); + +#define REGISTER_HOOKS_NAME "malloc_hooks_register_hooks" +#define ACTIVE_HOOKS_NAME "malloc_hooks_active_hooks" +#define GET_REAL_MALLOC_FUNCS_NAME "malloc_hooks_get_real_malloc_funcs" + +#endif + diff --git a/src/java.base/windows/native/launcher/icons/awt.ico b/src/java.base/windows/native/launcher/icons/awt.ico index 8d2c9571ed9..a8507d716d8 100644 Binary files a/src/java.base/windows/native/launcher/icons/awt.ico and b/src/java.base/windows/native/launcher/icons/awt.ico differ diff --git a/src/java.desktop/macosx/data/macosxicons/JavaApp.icns b/src/java.desktop/macosx/data/macosxicons/JavaApp.icns index fc60195cd0e..9c93b462982 100644 Binary files a/src/java.desktop/macosx/data/macosxicons/JavaApp.icns and b/src/java.desktop/macosx/data/macosxicons/JavaApp.icns differ diff --git a/src/jdk.jdwp.agent/share/native/libdt_filesocket/fileSocketTransport.c b/src/jdk.jdwp.agent/share/native/libdt_filesocket/fileSocketTransport.c new file mode 100644 index 00000000000..b47f938b8a2 --- /dev/null +++ b/src/jdk.jdwp.agent/share/native/libdt_filesocket/fileSocketTransport.c @@ -0,0 +1,410 @@ +/* + * Copyright (c) 2018, 2024 SAP SE. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include +#include +#include +#include "jni.h" +#include "jdwpTransport.h" +#include "fileSocketTransport.h" + +#ifdef _WIN32 +#include +#include +#define MAX_FILE_SOCKET_PATH_LEN UNIX_PATH_MAX +#else +#include +#include +#include +#define MAX_FILE_SOCKET_PATH_LEN sizeof(((struct sockaddr_un *) 0)->sun_path) +#endif + +#define MAX_DATA_SIZE 1000 +#define HANDSHAKE "JDWP-Handshake" + +/* Since the jdwp agent sometimes kills the VM outright when + * the connection fails, we always fake a successful + * connection and instead fail in the read/write packet methods, + * which does not cause the VM to exit. + */ +static jboolean fake_open = JNI_FALSE; +static jboolean initialized = JNI_FALSE; +static JavaVM *jvm; +static char path[MAX_FILE_SOCKET_PATH_LEN]; +static jdwpTransportCallback *callback; +static char last_error[2048]; +static struct jdwpTransportNativeInterface_ nif; +static jdwpTransportEnv single_env = (jdwpTransportEnv) &nif; + +void fileSocketTransport_logError(char const* format, ...) { + char* tmp = (*callback->alloc)(sizeof(last_error)); + va_list ap; + + if (tmp != NULL) { + va_start(ap, format); + vsnprintf(tmp, sizeof(last_error) - 1, format, ap); + tmp[sizeof(last_error) - 1] = '\0'; + va_end(ap); + + printf("Error: %s\n", tmp); + + memcpy(last_error, tmp, sizeof(last_error)); + (*callback->free)(tmp); + } else { + printf("Could not get memory to print error.\n"); + } +} + +static jdwpTransportError JNICALL fileSocketTransport_GetCapabilities(jdwpTransportEnv* env, JDWPTransportCapabilities *capabilities_ptr) { + JDWPTransportCapabilities result; + + memset(&result, 0, sizeof(result)); + result.can_timeout_attach = JNI_FALSE; + result.can_timeout_accept = JNI_FALSE; + result.can_timeout_handshake = JNI_FALSE; + + *capabilities_ptr = result; + + return JDWPTRANSPORT_ERROR_NONE; +} + +static jdwpTransportError JNICALL fileSocketTransport_SetTransportConfiguration(jdwpTransportEnv* env, jdwpTransportConfiguration *config) { + return JDWPTRANSPORT_ERROR_NONE; +} + +static jdwpTransportError JNICALL fileSocketTransport_Close(jdwpTransportEnv* env) { + if (fileSocketTransport_HasValidHandle()) { + fileSocketTransport_CloseImpl(); + } + + fake_open = JNI_FALSE; + return JDWPTRANSPORT_ERROR_NONE; +} + +static jdwpTransportError JNICALL fileSocketTransport_Attach(jdwpTransportEnv* env, const char* address, jlong attach_timeout, jlong handshake_timeout) { + /* We don't support attach. */ + fileSocketTransport_logError("Only server=y mode is supported by dt_filesocket"); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; +} + +static jdwpTransportError JNICALL fileSocketTransport_StartListening(jdwpTransportEnv* env, const char* address, char** actual_address) { + /* Only make sure we have no open connection. */ + fileSocketTransport_Close(env); + + if (address == NULL) { + fileSocketTransport_logError("Default address not supported"); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; + } + + *actual_address = (*callback->alloc)((int)strlen(address) + 1); + + if (*actual_address == NULL) { + return JDWPTRANSPORT_ERROR_OUT_OF_MEMORY; + } else { + strcpy(*actual_address, address); + } + + if (strlen(address) < MAX_FILE_SOCKET_PATH_LEN) { + strcpy(path, address); + } else { + fileSocketTransport_logError("Address too long: %s", address); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; + } + + return JDWPTRANSPORT_ERROR_NONE; +} + +static jdwpTransportError JNICALL fileSocketTransport_StopListening(jdwpTransportEnv* env) { + return JDWPTRANSPORT_ERROR_NONE; +} + +static jboolean JNICALL fileSocketTransport_IsOpen(jdwpTransportEnv* env) { + return fake_open || fileSocketTransport_HasValidHandle(); +} + +static int fileSocketTransport_ReadFully(char* buf, int len) { + int read = 0; + + while (len > 0) { + int n = fileSocketTransport_ReadImpl(buf, len); + + if (n < 0) { + return n; + } else if (n == 0) { + break; + } + + buf += n; + len -= n; + read += n; + } + + return read; +} + +static int fileSocketTransport_WriteFully(char* buf, int len) { + int written = 0; + + while (len > 0) { + int n = fileSocketTransport_WriteImpl(buf, len); + + if (n < 0) { + return n; + } else if (n == 0) { + break; + } + + buf += n; + len -= n; + written += n; + } + + return written; +} + +static jdwpTransportError JNICALL fileSocketTransport_Accept(jdwpTransportEnv* env, jlong accept_timeout, jlong handshake_timeout) { + fileSocketTransport_AcceptImpl(path); + + if (!fileSocketTransport_HasValidHandle()) { + fake_open = JNI_TRUE; + } else { + char buf[sizeof(HANDSHAKE)]; + fileSocketTransport_ReadFully(buf, (int) strlen(HANDSHAKE)); + fileSocketTransport_WriteFully(HANDSHAKE, (int) strlen(HANDSHAKE)); + + if (strcmp(buf, HANDSHAKE) != 0) { + fake_open = JNI_TRUE; + } + } + + return JDWPTRANSPORT_ERROR_NONE; +} + +static jdwpTransportError JNICALL fileSocketTransport_ReadPacket(jdwpTransportEnv* env, jdwpPacket *packet) { + jint length, data_len, n; + + if (!fileSocketTransport_HasValidHandle()) { + fake_open = JNI_FALSE; + return JDWPTRANSPORT_ERROR_IO_ERROR; + } else if (fake_open) { + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + + if (packet == NULL) { + fileSocketTransport_logError("Packet is null while reading"); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; + } + + /* Taken mostly from socketTransport.c */ + n = fileSocketTransport_ReadFully((char *) &length, sizeof(jint)); + + if (n == 0) { + packet->type.cmd.len = 0; + return JDWPTRANSPORT_ERROR_NONE; + } + + if (n != sizeof(jint)) { + fileSocketTransport_logError("Only read %d instead of %d bytes for length field", (int) n, (int) sizeof(jint)); + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + + length = (jint) ntohl(length); + packet->type.cmd.len = length; + n = fileSocketTransport_ReadFully((char *)&(packet->type.cmd.id), sizeof(jint)); + + if (n < (int)sizeof(jint)) { + fileSocketTransport_logError("Only read %d instead of %d bytes for command id", (int) n, (int) sizeof(jint)); + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + + packet->type.cmd.id = (jint) ntohl(packet->type.cmd.id); + n = fileSocketTransport_ReadFully((char *)&(packet->type.cmd.flags), sizeof(jbyte)); + + if (n < (int) sizeof(jbyte)) { + fileSocketTransport_logError("Only read %d bytes for flags", (int) n); + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + + if (packet->type.cmd.flags & JDWPTRANSPORT_FLAGS_REPLY) { + n = fileSocketTransport_ReadFully((char *)&(packet->type.reply.errorCode), sizeof(jbyte)); + if (n < (int)sizeof(jshort)) { + fileSocketTransport_logError("Only read %d bytes for error code", (int) n); + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + } else { + n = fileSocketTransport_ReadFully((char *)&(packet->type.cmd.cmdSet), sizeof(jbyte)); + if (n < (int)sizeof(jbyte)) { + fileSocketTransport_logError("Only read %d bytes for command set", (int) n); + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + + n = fileSocketTransport_ReadFully((char *)&(packet->type.cmd.cmd), sizeof(jbyte)); + if (n < (int)sizeof(jbyte)) { + fileSocketTransport_logError("Only read %d bytes for command", (int) n); + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + } + + data_len = length - ((sizeof(jint) * 2) + (sizeof(jbyte) * 3)); + + if (data_len < 0) { + fileSocketTransport_logError("Inavlid data length %d of read packet", (int) data_len); + return JDWPTRANSPORT_ERROR_IO_ERROR; + } else if (data_len == 0) { + packet->type.cmd.data = NULL; + } else { + packet->type.cmd.data = (*callback->alloc)(data_len); + + if (packet->type.cmd.data == NULL) { + return JDWPTRANSPORT_ERROR_OUT_OF_MEMORY; + } + + n = fileSocketTransport_ReadFully((char *) packet->type.cmd.data, data_len); + + if (n < data_len) { + fileSocketTransport_logError("Only read %d bytes for JDWP payload but expected %d", (int) n, (int) data_len); + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + } + + return JDWPTRANSPORT_ERROR_NONE; +} + +static jdwpTransportError JNICALL fileSocketTransport_WritePacket(jdwpTransportEnv* env, const jdwpPacket* packet) { + jint len, data_len, id, n; + char header[JDWP_HEADER_SIZE + MAX_DATA_SIZE]; + jbyte *data; + + if (!fileSocketTransport_HasValidHandle()) { + fake_open = JNI_FALSE; + return JDWPTRANSPORT_ERROR_IO_ERROR; + } else if (fake_open) { + return JDWPTRANSPORT_ERROR_IO_ERROR; + } + + /* Taken mostly from sockectTransport.c */ + if (packet == NULL) { + fileSocketTransport_logError("Packet is null when writing"); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; + } + + len = packet->type.cmd.len; + data_len = len - JDWP_HEADER_SIZE; + + if (data_len < 0) { + fileSocketTransport_logError("Packet to write has illegal data length %d", (int) data_len); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; + } + + /* prepare the header for transmission */ + len = (jint) htonl(len); + id = (jint) htonl(packet->type.cmd.id); + + memcpy(header + 0, &len, 4); + memcpy(header + 4, &id, 4); + header[8] = packet->type.cmd.flags; + + if (packet->type.cmd.flags & JDWPTRANSPORT_FLAGS_REPLY) { + jshort errorCode = htons(packet->type.reply.errorCode); + memcpy(header + 9, &errorCode, 2); + } else { + header[9] = packet->type.cmd.cmdSet; + header[10] = packet->type.cmd.cmd; + } + + data = packet->type.cmd.data; + + /* Do one send for short packets, two for longer ones */ + if (data_len <= MAX_DATA_SIZE) { + memcpy(header + JDWP_HEADER_SIZE, data, data_len); + if ((n = fileSocketTransport_WriteFully((char *) &header, JDWP_HEADER_SIZE + data_len)) != + JDWP_HEADER_SIZE + data_len) { + fileSocketTransport_logError("Could only write %d bytes instead of %d of the packet", (int) n, (int) (JDWP_HEADER_SIZE + data_len)); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; + } + } else { + memcpy(header + JDWP_HEADER_SIZE, data, MAX_DATA_SIZE); + if ((n = fileSocketTransport_WriteFully((char *) &header, JDWP_HEADER_SIZE + MAX_DATA_SIZE)) != + JDWP_HEADER_SIZE + MAX_DATA_SIZE) { + fileSocketTransport_logError("Could only write %d bytes instead of %d of the packet", (int) n, (int) (JDWP_HEADER_SIZE + MAX_DATA_SIZE)); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; + } + /* Send the remaining data bytes right out of the data area. */ + if ((n = fileSocketTransport_WriteFully((char *) data + MAX_DATA_SIZE, + data_len - MAX_DATA_SIZE)) != data_len - MAX_DATA_SIZE) { + fileSocketTransport_logError("Could only write %d bytes instead of %d of the packet", (int) n, (int) (data_len - MAX_DATA_SIZE)); + return JDWPTRANSPORT_ERROR_ILLEGAL_ARGUMENT; + } + } + + return JDWPTRANSPORT_ERROR_NONE; +} + +static jdwpTransportError JNICALL fileSocketTransport_GetLastError(jdwpTransportEnv* env, char** error) { + *error = (*callback->alloc)(sizeof(last_error)); + + if (*error == NULL) { + return JDWPTRANSPORT_ERROR_OUT_OF_MEMORY; + } + + memcpy(*error, last_error, sizeof(last_error)); + (*error)[sizeof(last_error) - 1] = '\0'; + return JDWPTRANSPORT_ERROR_NONE; +} + +JNIEXPORT jint JNICALL +jdwpTransport_OnLoad(JavaVM *vm, jdwpTransportCallback* callbacks, jint version, jdwpTransportEnv** env) +{ + if (version < JDWPTRANSPORT_VERSION_1_0 ||version > JDWPTRANSPORT_VERSION_1_1) { + return JNI_EVERSION; + } + + if (initialized) { + return JNI_EEXIST; + } + + initialized = JNI_TRUE; + jvm = vm; + callback = callbacks; + + /* initialize interface table */ + nif.GetCapabilities = fileSocketTransport_GetCapabilities; + nif.Attach = fileSocketTransport_Attach; + nif.StartListening = fileSocketTransport_StartListening; + nif.StopListening = fileSocketTransport_StopListening; + nif.Accept = fileSocketTransport_Accept; + nif.IsOpen = fileSocketTransport_IsOpen; + nif.Close = fileSocketTransport_Close; + nif.ReadPacket = fileSocketTransport_ReadPacket; + nif.WritePacket = fileSocketTransport_WritePacket; + nif.GetLastError = fileSocketTransport_GetLastError; + if (version >= JDWPTRANSPORT_VERSION_1_1) { + nif.SetTransportConfiguration = fileSocketTransport_SetTransportConfiguration; + } + *env = (jdwpTransportEnv*) &single_env; + + return JNI_OK; +} diff --git a/src/jdk.jdwp.agent/share/native/libdt_filesocket/fileSocketTransport.h b/src/jdk.jdwp.agent/share/native/libdt_filesocket/fileSocketTransport.h new file mode 100644 index 00000000000..29016886a2d --- /dev/null +++ b/src/jdk.jdwp.agent/share/native/libdt_filesocket/fileSocketTransport.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018, 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef FILE_SOCKET_TRANSPORT_H +#define FILE_SOCKET_TRANSPORT_H + +#include "jni.h" + +void fileSocketTransport_logError(char const* format, ...); +jboolean fileSocketTransport_HasValidHandle(); +void fileSocketTransport_CloseImpl(); +void fileSocketTransport_AcceptImpl(char const* name); +int fileSocketTransport_ReadImpl(char* buffer, int size); +int fileSocketTransport_WriteImpl(char* buffer, int size); + +#endif diff --git a/src/jdk.jdwp.agent/share/native/libjdwp/debugInit.c b/src/jdk.jdwp.agent/share/native/libjdwp/debugInit.c index fe990e97a65..dbc2a2cb22b 100644 --- a/src/jdk.jdwp.agent/share/native/libjdwp/debugInit.c +++ b/src/jdk.jdwp.agent/share/native/libjdwp/debugInit.c @@ -841,6 +841,8 @@ printUsage(void) "launch= run debugger on event none\n" "onthrow= debug on throw none\n" "onuncaught=y|n debug on any uncaught? n\n" + // SapMachine 2022-12-12 Revert JDK-8226608, we should show the onjcmd=y|n option in jdwp usage + "onjcmd=y|n start debug via jcmd? n\n" "timeout= for listen/attach in milliseconds n\n" "includevirtualthreads=y|n List of all threads includes virtual threads as well as platform threads.\n" " n\n" diff --git a/src/jdk.jdwp.agent/unix/native/libdt_filesocket/fileSocketTransport_md.c b/src/jdk.jdwp.agent/unix/native/libdt_filesocket/fileSocketTransport_md.c new file mode 100644 index 00000000000..575eced1dd8 --- /dev/null +++ b/src/jdk.jdwp.agent/unix/native/libdt_filesocket/fileSocketTransport_md.c @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2018, 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include +#include + +#include "jni.h" +#include "fileSocketTransport.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define UNIX_PATH_MAX sizeof(((struct sockaddr_un *) 0)->sun_path) + +static int server_socket = -1; +static int connection_socket = -1; + +static void closeSocket(int* socket) { + if (*socket != -1) { +#if defined(_AIX) + int rv; + + do { + rv = close(*socket); + } while ((rv != 0) && (errno == EINTR)); +#else + close(*socket); +#endif + + *socket = -1; + } +} + +static char file_to_delete[UNIX_PATH_MAX]; +static volatile int file_to_delete_valid; + +static void memoryBarrier() { +#if defined(__linux__) || defined(__APPLE__) + __sync_synchronize(); +#elif defined(_AIX) + __sync(); +#else +#error "Unknown platform" +#endif +} + +static jboolean deleteFile(char const* name) { + return (access(name, F_OK) == -1) || (unlink(name) == 0) ? JNI_TRUE : JNI_FALSE; +} + +static void registerFileToDelete(char const* name) { + if (file_to_delete_valid == 0) { + if ((name != NULL) && (strlen(name) + 1 <= sizeof(file_to_delete))) { + strcpy(file_to_delete, name); + memoryBarrier(); + file_to_delete_valid = 1; + memoryBarrier(); + } + } else { + // Should never change. + assert(strcmp(name, file_to_delete) == 0); + } +} + +static void cleanupSocketOnExit(void) { + memoryBarrier(); + + if (file_to_delete_valid) { + memoryBarrier(); + deleteFile(file_to_delete); + } +} + +jboolean fileSocketTransport_HasValidHandle() { + return connection_socket == -1 ? JNI_FALSE : JNI_TRUE; +} + +void fileSocketTransport_CloseImpl() { + closeSocket(&server_socket); + closeSocket(&connection_socket); +} + +void logAndCleanupFailedAccept(char const* error_msg, char const* name) { + fileSocketTransport_logError("%s: socket %s: %s", error_msg, name, strerror(errno)); + fileSocketTransport_CloseImpl(); +} + +void fileSocketTransport_AcceptImpl(char const* name) { + static int already_called = 0; + + if (!already_called) { + registerFileToDelete(name); + atexit(cleanupSocketOnExit); + already_called = 1; + } + + if (server_socket == -1) { + socklen_t len = sizeof(struct sockaddr_un); + struct sockaddr_un addr; + int addr_size = sizeof(addr); + + memset((void*)&addr, 0, len); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, name, sizeof(addr.sun_path) - 1); + +#ifdef _AIX + addr.sun_len = strlen(addr.sun_path); + addr_size = SUN_LEN(&addr); +#endif + + server_socket = socket(PF_UNIX, SOCK_STREAM, 0); + + if (server_socket == -1) { + logAndCleanupFailedAccept("Could not create domain socket", name); + return; + } + + if (!deleteFile(name)) { + logAndCleanupFailedAccept("Could not remove file to create new file socket", name); + return; + } + + if (bind(server_socket, (struct sockaddr*)&addr, addr_size) == -1) { + logAndCleanupFailedAccept("Could not bind file socket", name); + return; + } + + if (chmod(name, (S_IREAD | S_IWRITE) & ~(S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) == -1) { + logAndCleanupFailedAccept("Chmod on file socket failed", name); + return; + } + + if (chown(name, geteuid(), getegid()) == -1) { + logAndCleanupFailedAccept("Chown on file socket failed", name); + return; + } + + if (listen(server_socket, 1) == -1) { + logAndCleanupFailedAccept("Could not listen on file socket", name); + return; + } + } + + do { + connection_socket = accept(server_socket, NULL, NULL); + } while ((connection_socket == -1) && (errno == EINTR)); + + /* We can remove the file since we are connected (or it failed). */ + deleteFile(name); + closeSocket(&server_socket); + + if (connection_socket == -1) { + logAndCleanupFailedAccept("Could not accept on file socket", name); + return; + } + + uid_t other_user = (uid_t)-1; + gid_t other_group = (gid_t)-1; + + /* Check if the connected user is the same as the user running the VM. */ +#if defined(__linux__) + struct ucred cred_info; + socklen_t optlen = sizeof(cred_info); + + if (getsockopt(connection_socket, SOL_SOCKET, SO_PEERCRED, (void*) &cred_info, &optlen) == -1) { + logAndCleanupFailedAccept("Failed to get socket option SO_PEERCRED of file socket", name); + return; + } + + other_user = cred_info.uid; + other_group = cred_info.gid; +#elif defined(__APPLE__) + if (getpeereid(connection_socket, &other_user, &other_group) != 0) { + logAndCleanupFailedAccept("Failed to get peer id of file socket", name); + return; + } +#elif defined(_AIX) + struct peercred_struct cred_info; + socklen_t optlen = sizeof(cred_info); + + if (getsockopt(connection_socket, SOL_SOCKET, SO_PEERID, (void*)&cred_info, &optlen) == -1) { + logAndCleanupFailedAccept("Failed to get socket option SO_PEERID of file socket", name); + return; + } + + other_user = cred_info.euid; + other_group = cred_info.egid; +#else +#error "Unknown platform" +#endif + + /* Allow root too */ + if (other_user != 0) { + if (other_user != geteuid()) { + fileSocketTransport_logError("Cannot allow user %d to connect to file socket %s of user %d", + (int)other_user, name, (int)geteuid()); + fileSocketTransport_CloseImpl(); + } else if (other_group != getegid()) { + fileSocketTransport_logError("Cannot allow user %d (group %d) to connect to file socket " + "%s of user %d (group %d)", (int)other_user, (int)other_group, + name, (int)geteuid(), (int)getegid()); + fileSocketTransport_CloseImpl(); + } + } +} + +int fileSocketTransport_ReadImpl(char* buffer, int size) { + int result; + + do { + result = read(connection_socket, buffer, size); + } while ((result < 0) && (errno == EINTR)); + + if (result < 0) { + fileSocketTransport_logError("Read failed with result %d: %s", result, strerror(errno)); + } + + return result; +} + +int fileSocketTransport_WriteImpl(char* buffer, int size) { + int result; + + do { + result = write(connection_socket, buffer, size); + } while ((result < 0) && (errno == EINTR)); + + if (result < 0) { + fileSocketTransport_logError("Write failed with result %d: %s", result, strerror(errno)); + } + + return result; +} diff --git a/src/jdk.jdwp.agent/windows/native/libdt_filesocket/fileSocketTransport_md.c b/src/jdk.jdwp.agent/windows/native/libdt_filesocket/fileSocketTransport_md.c new file mode 100644 index 00000000000..51d4da80eba --- /dev/null +++ b/src/jdk.jdwp.agent/windows/native/libdt_filesocket/fileSocketTransport_md.c @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2018, 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include +#include + +#include "jni.h" +#include "fileSocketTransport.h" + + +#include +#include +#include + +/* Make sure winsock is initialized on Windows */ +BOOL WINAPI DllMain(HINSTANCE hinst, DWORD reason, LPVOID reserved) +{ + WSADATA wsadata; + + if ((reason == DLL_PROCESS_ATTACH) && (WSAStartup(MAKEWORD(2, 2), &wsadata) != 0)) { + return JNI_FALSE; + } + + if (reason == DLL_PROCESS_DETACH) { + WSACleanup(); + } + + return TRUE; +} + +static SOCKET server_socket = INVALID_SOCKET; +static SOCKET connection_socket = INVALID_SOCKET; + +static void closeSocket(SOCKET* socket) { + if (*socket != INVALID_SOCKET) { + closesocket(*socket); + *socket = INVALID_SOCKET; + } +} + +static char file_to_delete[UNIX_PATH_MAX]; +static volatile int file_to_delete_valid; + +static jboolean deleteFile(char const* name) { + if ((!DeleteFile(name)) && (GetLastError() != ERROR_FILE_NOT_FOUND)) { + return JNI_FALSE; + } + + return JNI_TRUE; +} + +static void registerFileToDelete(char const* name) { + if (file_to_delete_valid == 0) { + if ((name != NULL) && (strlen(name) + 1 <= sizeof(file_to_delete))) { + strcpy(file_to_delete, name); + MemoryBarrier(); + file_to_delete_valid = 1; + MemoryBarrier(); + } + } else { + // Should never change. + assert(strcmp(name, file_to_delete) == 0); + } +} + +static char* getErrorMsg(char* buf, size_t len) { + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, WSAGetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + buf, (DWORD)len, NULL); + return buf; +} + +static void cleanupSocketOnExit(void) { + MemoryBarrier(); + + if (file_to_delete_valid) { + MemoryBarrier(); + deleteFile(file_to_delete); + } +} + +jboolean fileSocketTransport_HasValidHandle() { + return connection_socket == INVALID_SOCKET ? JNI_FALSE : JNI_TRUE; +} + +void fileSocketTransport_CloseImpl() { + closeSocket(&server_socket); + closeSocket(&connection_socket); +} + +void logAndCleanupFailedAccept(char const* error_msg, char const* name) { + char buf[256]; + fileSocketTransport_logError("%s: socket %s: %s", error_msg, name, getErrorMsg(buf, sizeof(buf))); + fileSocketTransport_CloseImpl(); +} + +void fileSocketTransport_AcceptImpl(char const* name) { + static int already_called = 0; + + if (!already_called) { + registerFileToDelete(name); + atexit(cleanupSocketOnExit); + already_called = 1; + } + + if (server_socket == INVALID_SOCKET) { + int len = (int) sizeof(struct sockaddr_un); + struct sockaddr_un addr; + int addr_size = sizeof(addr); + + memset((void*)&addr, 0, len); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, name, sizeof(addr.sun_path) - 1); + + server_socket = socket(PF_UNIX, SOCK_STREAM, 0); + + if (server_socket == INVALID_SOCKET) { + logAndCleanupFailedAccept("Could not create domain socket", name); + return; + } + + if (!deleteFile(name)) { + logAndCleanupFailedAccept("Could not remove file to create new file socket", name); + return; + } + + if (bind(server_socket, (struct sockaddr*)&addr, addr_size) == -1) { + logAndCleanupFailedAccept("Could not bind file socket", name); + return; + } + + if (listen(server_socket, 1) == -1) { + logAndCleanupFailedAccept("Could not listen on file socket", name); + return; + } + } + + do { + connection_socket = accept(server_socket, NULL, NULL); + } while ((connection_socket == INVALID_SOCKET) && (WSAGetLastError() == WSAEINTR)); + + /* We can remove the file since we are connected (or it failed). */ + deleteFile(name); + closeSocket(&server_socket); + + if (connection_socket == INVALID_SOCKET) { + logAndCleanupFailedAccept("Could not accept on file socket", name); + return; + } + + ULONG peer_pid; + DWORD size; + + if (WSAIoctl(connection_socket, SIO_AF_UNIX_GETPEERPID, NULL, 0, &peer_pid, sizeof(peer_pid), &size, NULL, NULL) != 0) { + logAndCleanupFailedAccept("Could not determine connected processed", name); + return; + } + + HANDLE peer_proc = NULL; + HANDLE self_token = NULL; + HANDLE peer_token = NULL; + PTOKEN_USER self_user = NULL; + PTOKEN_USER peer_user = NULL; + TOKEN_ELEVATION peer_elevation; + + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &self_token)) { + logAndCleanupFailedAccept("Could not get own token", name); + } else if (GetTokenInformation(self_token, TokenUser, NULL, 0, &size)) { + logAndCleanupFailedAccept("Could not get own token user size", name); + } else if ((self_user = (PTOKEN_USER)malloc((size_t)size)) == NULL) { + logAndCleanupFailedAccept("Could not alloc own token user size", name); + } else if (!GetTokenInformation(self_token, TokenUser, self_user, size, &size)) { + logAndCleanupFailedAccept("Could not get own token user", name); + } else if ((peer_proc = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, peer_pid)) == NULL) { + logAndCleanupFailedAccept("Could not open peer process", name); + } else if (!OpenProcessToken(peer_proc, TOKEN_QUERY | TOKEN_QUERY_SOURCE, &peer_token)) { + logAndCleanupFailedAccept("Could not get peer token", name); + } else if (GetTokenInformation(peer_token, TokenUser, NULL, 0, &size)) { + logAndCleanupFailedAccept("Could not get peer token user size", name); + } else if ((peer_user = (PTOKEN_USER)malloc((size_t)size)) == NULL) { + logAndCleanupFailedAccept("Could not alloc peer token user size", name); + } else if (!GetTokenInformation(peer_token, TokenUser, peer_user, size, &size)) { + logAndCleanupFailedAccept("Could not get peer token information", name); + } else if (!GetTokenInformation(peer_token, TokenElevation, &peer_elevation, sizeof(peer_elevation), &size)) { + logAndCleanupFailedAccept("Could not get peer token information", name); + } else if (!peer_elevation.TokenIsElevated && !EqualSid(self_user->User.Sid, peer_user->User.Sid)) { + fileSocketTransport_logError("Connecting process is not the same user nor admin"); + fileSocketTransport_CloseImpl(); + } + + if (peer_token != NULL) { + CloseHandle(peer_token); + } + + if (self_token != NULL) { + CloseHandle(self_token); + } + + if (peer_proc != NULL) { + CloseHandle(peer_proc); + } + + free(self_user); + free(peer_user); +} + +int fileSocketTransport_ReadImpl(char* buffer, int size) { + int result; + + do { + result = recv(connection_socket, buffer, size, 0); + } while ((result < 0) && (WSAGetLastError() == WSAEINTR)); + + if (result < 0) { + char buf[256]; + fileSocketTransport_logError("Read failed with result %d: %s", result, getErrorMsg(buf, sizeof(buf))); + } + + return result; +} + +int fileSocketTransport_WriteImpl(char* buffer, int size) { + int result; + + do { + result = send(connection_socket, buffer, size, 0); + } while ((result < 0) && (WSAGetLastError() == WSAEINTR)); + + if (result < 0) { + char buf[256]; + fileSocketTransport_logError("Write failed with result %d: %s", result, getErrorMsg(buf, sizeof(buf))); + } + + return result; +} diff --git a/src/jdk.jfr/share/conf/jfr/gc.jfc b/src/jdk.jfr/share/conf/jfr/gc.jfc new file mode 100755 index 00000000000..2ef37b29ff1 --- /dev/null +++ b/src/jdk.jfr/share/conf/jfr/gc.jfc @@ -0,0 +1,1158 @@ + + + + + + + true + 1000 ms + + + + false + everyChunk + + + + false + 1000 ms + + + + false + everyChunk + + + + false + 1000 ms + + + + false + 10 s + + + + false + 10 s + + + + false + false + + + + false + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + + + + false + + + + false + false + 20 ms + + + + false + false + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + + + + false + false + 0 ms + + + + false + false + 0 ms + + + + false + + + + false + + + + false + + + + false + + + + false + false + + + + false + false + 0 ms + + + + false + false + + + + false + false + 0 ms + + + + false + false + 0 ms + + + + false + + + + false + + + + true + beginChunk + + + + true + beginChunk + + + + false + 20 ms + + + + false + 20 ms + + + + false + 10 ms + + + + false + 10 ms + + + + false + 10 ms + + + + false + 10 ms + + + + false + 10 ms + + + + true + 10 ms + + + + true + false + + + + false + everyChunk + + + + true + beginChunk + + + + false + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + false + everyChunk + + + + true + everyChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + false + + + + true + everyChunk + + + + true + everyChunk + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + false + + + + true + false + + + + true + + + + true + 0 ms + + + + true + 0 ms + true + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + false + 0 ms + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + false + + + + true + + + + false + everyChunk + + + + false + + + + true + everyChunk + + + + false + + + + true + false + 0 ns + + + + true + 1000 ms + + + + true + 1000 ms + + + + false + beginChunk + + + + false + 1000 ms + + + + false + 1000 ms + + + + false + 60 s + + + + false + + + + false + + + + true + + + + false + beginChunk + + + + false + everyChunk + + + + true + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + 30 s + + + + true + 30 s + + + + true + 30 s + + + + true + 30 s + + + + true + beginChunk + + + + false + 10 s + + + + true + 1000 ms + + + + true + 10 s + + + + true + beginChunk + + + + true + endChunk + + + + true + false + + + + false + 5 s + + + + false + 10 s + + + + true + beginChunk + + + + true + everyChunk + + + + true + everyChunk + + + + false + false + + + + false + false + + + + true + 150/s + false + + + + false + everyChunk + + + + false + false + 0 ms + + + + false + false + 0 ms + + + + false + endChunk + + + + false + endChunk + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + + + + false + + + + true + beginChunk + + + + false + false + + + + false + false + + + + false + false + + + + false + false + + + + false + false + + + + false + false + + + + true + false + + + + false + 1000 ms + + + + true + + + + true + + + + false + 0 ns + + + + true + + + + true + + + + true + false + 0 ms + + + + true + false + 1 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + false + false + + + + false + 0 ns + false + + + + true + 5 s + + + + true + 1 s + false + + + + true + endChunk + + + + false + endChunk + + + + false + endChunk + + + + false + false + forRemoval + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 20 ms + + 20 ms + + 20 ms + + false + + + diff --git a/src/jdk.jfr/share/conf/jfr/gc_details.jfc b/src/jdk.jfr/share/conf/jfr/gc_details.jfc new file mode 100755 index 00000000000..cd8cf0660f4 --- /dev/null +++ b/src/jdk.jfr/share/conf/jfr/gc_details.jfc @@ -0,0 +1,1158 @@ + + + + + + + true + 1000 ms + + + + true + everyChunk + + + + true + 1000 ms + + + + true + everyChunk + + + + true + 1000 ms + + + + true + 10 s + + + + true + 10 s + + + + true + true + + + + true + + + + true + true + 20 ms + + + + true + true + 20 ms + + + + false + false + + + + false + + + + false + false + 20 ms + + + + false + false + + + + true + true + 20 ms + + + + true + true + 20 ms + + + + true + true + 20 ms + + + + true + true + + + + false + false + 0 ms + + + + false + false + 0 ms + + + + false + + + + false + + + + false + + + + false + + + + true + true + + + + false + false + 0 ms + + + + false + false + + + + false + false + 0 ms + + + + false + false + 0 ms + + + + false + + + + false + + + + true + beginChunk + + + + true + beginChunk + + + + false + 20 ms + + + + false + 20 ms + + + + true + 10 ms + + + + false + 10 ms + + + + false + 10 ms + + + + false + 10 ms + + + + false + 10 ms + + + + true + 10 ms + + + + true + true + + + + true + everyChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + everyChunk + + + + true + everyChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + + + + true + everyChunk + + + + true + everyChunk + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + false + + + + true + false + + + + true + + + + true + 0 ms + + + + true + 0 ms + true + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + 0 ms + + + + true + + + + true + + + + true + + + + true + + + + true + + + + true + false + + + + true + + + + true + everyChunk + + + + true + + + + true + everyChunk + + + + true + + + + true + true + 0 ns + + + + true + 1000 ms + + + + true + 1000 ms + + + + false + beginChunk + + + + false + 1000 ms + + + + false + 1000 ms + + + + false + 60 s + + + + false + + + + false + + + + true + + + + true + beginChunk + + + + false + everyChunk + + + + true + + + + true + beginChunk + + + + true + beginChunk + + + + true + beginChunk + + + + true + 30 s + + + + true + 30 s + + + + true + 30 s + + + + true + 30 s + + + + true + beginChunk + + + + true + 10 s + + + + true + 1000 ms + + + + true + 10 s + + + + true + beginChunk + + + + true + endChunk + + + + true + true + + + + true + 5 s + + + + true + 10 s + + + + true + beginChunk + + + + true + everyChunk + + + + true + everyChunk + + + + true + true + + + + true + true + + + + true + 150/s + true + + + + true + everyChunk + + + + false + false + 0 ms + + + + false + false + 0 ms + + + + false + endChunk + + + + false + endChunk + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + 20 ms + + + + false + false + + + + false + + + + true + beginChunk + + + + false + false + + + + false + false + + + + false + false + + + + false + false + + + + false + false + + + + true + true + + + + true + true + + + + true + 1000 ms + + + + true + + + + true + + + + false + 0 ns + + + + true + + + + true + + + + true + true + 0 ms + + + + true + true + 1 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + true + 0 ms + + + + false + false + + + + true + 0 ns + true + + + + true + 5 s + + + + true + 1 s + true + + + + true + endChunk + + + + false + endChunk + + + + false + endChunk + + + + false + false + forRemoval + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 20 ms + + 20 ms + + 20 ms + + false + + + diff --git a/test/hotspot/gtest/malloctrace/test_site_table.cpp b/test/hotspot/gtest/malloctrace/test_site_table.cpp new file mode 100644 index 00000000000..cd5bf53c68d --- /dev/null +++ b/test/hotspot/gtest/malloctrace/test_site_table.cpp @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" + +#ifdef LINUX + +#include "jvm_io.h" +#include "malloctrace/mallocTrace.hpp" +#include "malloctrace/siteTable.hpp" +#include "memory/allocation.hpp" +#include "runtime/os.hpp" +#include "utilities/debug.hpp" +#include "utilities/ostream.hpp" +#include "unittest.hpp" + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +using sap::SiteTable; +using sap::Stack; + +static void init_random_randomly() { + os::init_random((int)os::elapsed_counter()); +} + +//#define LOG + +static void fill_stack_randomly(sap::Stack* s) { + for (unsigned i = 0; i < Stack::num_frames; i++) { + s->_frames[i] = (address)(intptr_t)os::random(); + } +} + +// Since SiteTable is too large to be put onto the stack of a test function, +// we need to create it dynamically. I don't want to make it a CHeapObj only +// for the sake of these tests though, so I have to use placement new. +static SiteTable* create_site_table() { + void* p = NEW_C_HEAP_ARRAY(SiteTable, 1, mtTest); + return new (p) SiteTable; +} + +static void destroy_site_table(SiteTable* s) { + FREE_C_HEAP_ARRAY(SiteTable, s); +} + +// Helper, create an array of unique stacks, randomly filled; returned array is C-heap allocated +static Stack* create_unique_stack_array(int num) { + Stack* random_stacks = NEW_C_HEAP_ARRAY(Stack, num, mtTest); + for (int i = 0; i < num; i ++) { + fill_stack_randomly(random_stacks + i); + // ensure uniqueness + random_stacks[i]._frames[0] = (address)(intptr_t)i; + } + return random_stacks; +} + +static void test_print_table(const SiteTable* table, int expected_entries) { + stringStream ss; + + table->print_stats(&ss); + if (expected_entries != -1) { + char match[32]; + jio_snprintf(match, sizeof(match), + "num_entries: %u,", expected_entries); + ASSERT_NE(::strstr(ss.base(), match), (char*)NULL); + } + ss.reset(); + + table->print_table(&ss, true); + if (expected_entries != -1) { + if (expected_entries > 0) { + // Note, output buffer may not hold full output + ASSERT_NE(::strstr(ss.base(), "--- 1 ---"), (char*)NULL); + } else { + ASSERT_NE(::strstr(ss.base(), "Table is empty"), (char*)NULL); + } + } +} + +TEST_VM(MallocTrace, site_table_basics) { + + init_random_randomly(); + + SiteTable* table = create_site_table(); + + test_print_table(table, 0); // Test printing empty table. + + const unsigned safe_to_add_without_overflow = SiteTable::max_entries(); + + // Generate a number of random stacks; enough to hit overflow limit from time to time. + const int num_stacks = safe_to_add_without_overflow + 100; + Stack* random_stacks = create_unique_stack_array(num_stacks); + + // Add n guaranteed-to-be-unique call stacks to the table; observe table; do that n times, which should + // increase invoc counters. + uint64_t expected_invocs = 0; + unsigned expected_unique_callsites = 0; + for (int invocs_per_stack = 0; invocs_per_stack < 10; invocs_per_stack++) { + for (unsigned num_callstacks = 0; num_callstacks < safe_to_add_without_overflow; num_callstacks++) { + table->add_site(random_stacks + num_callstacks, 1024); + expected_invocs ++; + if (invocs_per_stack == 0) { + // On the first iteration we expect a new callsite table node to be created for this stack + expected_unique_callsites++; + } + ASSERT_EQ(table->invocations(), expected_invocs); + ASSERT_EQ(table->size(), expected_unique_callsites); // Must be, since all stacks we add are be unique + ASSERT_EQ(table->lost(), (uint64_t)0); // So far we should see no losses + } + } + test_print_table(table, expected_unique_callsites); + DEBUG_ONLY(table->verify();) + + // Now cause table to overflow by adding further unique call stacks. Table should reject these new stacks + // and count them in lost counter + for (int overflow_num = 0; overflow_num < 100; overflow_num++) { + table->add_site(random_stacks + safe_to_add_without_overflow + overflow_num, 1024); + ASSERT_EQ(table->size(), expected_unique_callsites); // Should stay constant, no further adds should be accepted + ASSERT_EQ(table->lost(), (uint64_t)(overflow_num + 1)); // Lost counter should go up + ASSERT_EQ(table->invocations(), expected_invocs + overflow_num + 1); // Invocations counter includes lost + } + + test_print_table(table, expected_unique_callsites); + DEBUG_ONLY(table->verify();) + +#ifdef LOG + //table->print_table(tty, true); + table->print_stats(tty); + tty->cr(); +#endif + + destroy_site_table(table); +} + +TEST_VM(MallocTrace, site_table_random) { + SiteTable* table = create_site_table(); + + init_random_randomly(); + + // Generate a number of random stacks; enough to hit overflow limit from time to time. + const int num_stacks = SiteTable::max_entries() * 1.3; + Stack* random_stacks = create_unique_stack_array(num_stacks); + + for (int i = 0; i < num_stacks; i ++) { + fill_stack_randomly(random_stacks + i); + } + + // Now register these stacks randomly, a lot of times. + for (int i = 0; i < 1000*1000; i ++) { + Stack* stack = random_stacks + (os::random() % num_stacks); + table->add_site(stack, 1024); + ASSERT_EQ(table->invocations(), (uint64_t)i + 1); + } + + // test table printing, but we do not know how many unique stacks we have randomly generated, so don't + // test the exact number of entries + test_print_table(table, -1); + + DEBUG_ONLY(table->verify();) + + FREE_C_HEAP_ARRAY(Stack, random_stacks); + +#ifdef LOG + //table->print_table(tty, true); + table->print_stats(tty); + tty->cr(); +#endif + + destroy_site_table(table); +} + +#endif // HAVE_GLIBC_MALLOC_HOOKS + +#endif // LINUX diff --git a/test/hotspot/gtest/malloctrace/test_tracer.cpp b/test/hotspot/gtest/malloctrace/test_tracer.cpp new file mode 100644 index 00000000000..704d7f07400 --- /dev/null +++ b/test/hotspot/gtest/malloctrace/test_tracer.cpp @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2021 SAP SE. All rights reserved. + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" + +#ifdef LINUX + +#include "malloctrace/mallocTrace.hpp" +#include "memory/allocation.hpp" +#include "runtime/os.hpp" +#include "utilities/debug.hpp" +#include "utilities/ostream.hpp" + +#include "concurrentTestRunner.inline.hpp" +#include "unittest.hpp" +#include + +#ifdef HAVE_GLIBC_MALLOC_HOOKS + +// Since JDK-8289633 we forbid calling raw C-heap allocation functions using Kim's FORBID_C_FUNCTION. +// Callers need to explicitly opt in with ALLOW_C_FUNCTION. +// Since this code calls raw C-heap functions as a matter of course, instead of marking each call site +// with ALLOW_C_FUNCTION(..), I just mark them wholesale. +#if (__GNUC__ >= 10) +PRAGMA_DISABLE_GCC_WARNING("-Wattribute-warning") +#endif + +using sap::MallocTracer; + +static void init_random_randomly() { + os::init_random((int)os::elapsed_counter()); +} + +//#define LOG + +static size_t random_size() { return os::random() % 123; } + +static void test_print_statistics() { + stringStream ss; + + MallocTracer::print_on_error(&ss); // Test print on error + ASSERT_NE(::strstr(ss.base(), "num_entries:"), (char*)NULL); + ss.reset(); + + MallocTracer::print(&ss, false); +} + +struct MyTestRunnable_raw_malloc : public TestRunnable { + void runUnitTest() const { + void* p = ::malloc(random_size()); + if (os::random() % 2) { + p = ::realloc(p, random_size()); + } + ::free(p); + } +}; + +struct MyTestRunnable_raw_memalign : public TestRunnable { + void runUnitTest() const { + void* p = NULL; + // note min alignment for posix_memalign is sizeof(void*) + size_t alignment = 1 << (4 + (os::random() % 4)); // 16 ... 256 + int rc = ::posix_memalign(&p, alignment, random_size()); + assert(rc == 0 && p != NULL && is_aligned(p, alignment), + "bad memalign result %d, " PTR_FORMAT, rc, p2i(p)); + ::free(p); + } +}; + +struct MyTestRunnable_os_malloc : public TestRunnable { + void runUnitTest() const { + void* p = os::malloc(random_size(), mtTest); + if (os::random() % 2) { + p = os::realloc(p, random_size(), mtTest); + } + os::free(p); + } +}; + +struct MyTestRunnable_mixed_all : public TestRunnable { + void runUnitTest() const { + char buf[128]; // truncation ok and expected + stringStream ss(buf, sizeof(buf)); + int chance = os::random() % 100; + if (chance < 20) { + (void) MallocTracer::disable(); + os::naked_short_sleep(1); + (void) MallocTracer::enable(true); + } else if (chance < 25) { + MallocTracer::print(&ss, false); + } else { + void* p = ::malloc(random_size()); + if (os::random() % 2) { + p = ::realloc(p, random_size()); + } + ::free(p); + } + } +}; + +// Mark to switch on tracing and restore the old state +class TraceRestorer { + const bool _restore; +public: + TraceRestorer() : _restore(MallocTracer::enable(true)) {} + ~TraceRestorer() { + if (_restore) { + MallocTracer::disable(); + } + } +}; + +TEST_VM(MallocTrace, tracer_os_malloc) { + init_random_randomly(); + TraceRestorer restorer; + MyTestRunnable_os_malloc my_runnable; + ConcurrentTestRunner testRunner(&my_runnable, 5, 3000); + testRunner.run(); + test_print_statistics(); +#ifdef LOG + MallocTracer::print(tty, false); +#endif +} + +TEST_VM(MallocTrace, tracer_raw_malloc) { + init_random_randomly(); + TraceRestorer restorer; + MyTestRunnable_raw_malloc my_runnable; + ConcurrentTestRunner testRunner(&my_runnable, 5, 3000); + testRunner.run(); + test_print_statistics(); +#ifdef LOG + MallocTracer::print(tty, false); +#endif +} + +TEST_VM(MallocTrace, tracer_raw_memalign) { + init_random_randomly(); + TraceRestorer restorer; + MyTestRunnable_raw_memalign my_runnable; + ConcurrentTestRunner testRunner(&my_runnable, 5, 2000); + testRunner.run(); + test_print_statistics(); +#ifdef LOG + MallocTracer::print(tty, false); +#endif +} + +TEST_VM(MallocTrace, tracer_mixed_all) { + init_random_randomly(); + TraceRestorer restorer; + MyTestRunnable_mixed_all my_runnable; + ConcurrentTestRunner testRunner(&my_runnable, 5, 3000); + testRunner.run(); + test_print_statistics(); +#ifdef LOG + MallocTracer::print(tty, false); +#endif +} + +#endif // HAVE_GLIBC_MALLOC_HOOKS + +#endif // LINUX diff --git a/test/hotspot/gtest/vitals/test_vitals.cpp b/test/hotspot/gtest/vitals/test_vitals.cpp new file mode 100644 index 00000000000..12553af1570 --- /dev/null +++ b/test/hotspot/gtest/vitals/test_vitals.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "runtime/globals.hpp" +#include "utilities/debug.hpp" +#include "utilities/ostream.hpp" +#include "utilities/globalDefinitions.hpp" +#include "unittest.hpp" +#include "vitals/vitals.hpp" + +//#define LOG(s) tty->print_raw(s); +#define LOG(s) + +TEST_VM(vitals, report_with_explicit_default_settings) { + char tmp[64*K]; + stringStream ss(tmp, sizeof(tmp)); + sapmachine_vitals::print_info_t info; + ::memset(&info, 0xBB, sizeof(info)); + sapmachine_vitals::default_settings(&info); + sapmachine_vitals::print_report(&ss, &info); + LOG(tmp); + if (EnableVitals) { + ASSERT_NE(::strstr(tmp, "--jvm--"), (char*)NULL); + } +} + +TEST_VM(vitals, report_with_implicit_default_settings) { + char tmp[64*K]; + stringStream ss(tmp, sizeof(tmp)); + sapmachine_vitals::print_report(&ss, NULL); + LOG(tmp); + if (EnableVitals) { + ASSERT_NE(::strstr(tmp, "--jvm--"), (char*)NULL); + } +} + +TEST_VM(vitals, report_with_nownow) { + char tmp[64*K]; + stringStream ss(tmp, sizeof(tmp)); + sapmachine_vitals::print_info_t info; + ::memset(&info, 0xBB, sizeof(info)); + sapmachine_vitals::default_settings(&info); + info.sample_now = true; + for (int i = 0; i < 100; i ++) { + ss.reset(); + sapmachine_vitals::print_report(&ss, NULL); + if (EnableVitals) { + ASSERT_NE(::strstr(tmp, "--jvm--"), (char*)NULL); + } + } +} diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index 469a410e31d..1bb0b81157a 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -96,6 +96,8 @@ gc/stress/gclocker/TestExcessGCLockerCollections.java 8229120 generic-all # :hotspot_runtime runtime/jni/terminatedThread/TestTerminatedThread.java 8317789 aix-ppc64 +# SapMachine 2022-12-16 Exclude a failing test +runtime/jni/daemonDestroy/TestDaemonDestroy.java windows-all runtime/handshake/HandshakeSuspendExitTest.java 8294313 generic-all runtime/os/TestTracePageSizes.java#no-options 8267460 linux-aarch64 runtime/os/TestTracePageSizes.java#explicit-large-page-size 8267460 linux-aarch64 @@ -148,6 +150,11 @@ serviceability/jvmti/stress/StackTrace/NotSuspended/GetStackTraceNotSuspendedStr ############################################################################# +# SapMachine 2023-11-20 These make trouble in AIX CI +gtest/GTestWrapper.java aix-ppc64 +gtest/MetaspaceGtests.java#balanced-no-ccs aix-ppc64 +gtest/MetaspaceGtests.java#balanced-with-guards aix-ppc64 +gtest/MetaspaceGtests.java#default-debug aix-ppc64 ############################################################################# diff --git a/test/hotspot/jtreg/TEST.groups b/test/hotspot/jtreg/TEST.groups index 101cbe76afd..d443fd00c86 100644 --- a/test/hotspot/jtreg/TEST.groups +++ b/test/hotspot/jtreg/TEST.groups @@ -565,7 +565,16 @@ tier1_serviceability = \ -serviceability/sa/TestJmapCore.java \ -serviceability/sa/TestJmapCoreMetaspace.java +# SapMachine 2019-02-24 : Add tests where SAPMachine has different behavior to tier1, +# or which tests downstream-only features. +tier1_sapmachine = \ + runtime/Vitals \ + runtime/malloctrace + +# SapMachine 2019-02-24 : Add tests where SAPMachine has different behavior to tier1, +# or which tests downstream-only features. tier1 = \ + :tier1_sapmachine \ :tier1_common \ :tier1_compiler \ :tier1_gc \ diff --git a/test/hotspot/jtreg/applications/scimark/Scimark.java b/test/hotspot/jtreg/applications/scimark/Scimark.java index c17fad0ac54..2b0afb6acbf 100644 --- a/test/hotspot/jtreg/applications/scimark/Scimark.java +++ b/test/hotspot/jtreg/applications/scimark/Scimark.java @@ -39,18 +39,23 @@ @Artifact(organization = "gov.nist.math", name = "scimark", revision = "2.0", extension = "zip") public class Scimark { public static void main(String... args) throws Exception { - Map artifacts; - try { - artifacts = ArtifactResolver.resolve(Scimark.class); - } catch (ArtifactResolverException e) { - throw new Error("TESTBUG: Can not resolve artifacts for " - + Scimark.class.getName(), e); + // SapMachine 2018-07-06: Prefer Scimark classpath from system property. + String sciMark2Cp = System.getProperty("SCIMARK_2_CP"); + if (sciMark2Cp == null) { + Map artifacts; + try { + artifacts = ArtifactResolver.resolve(Scimark.class); + } catch (ArtifactResolverException e) { + throw new Error("TESTBUG: Can not resolve artifacts for " + + Scimark.class.getName(), e); + } + sciMark2Cp = artifacts.get("gov.nist.math.scimark-2.0").toString(); } System.setProperty("test.noclasspath", "true"); OutputAnalyzer output = new OutputAnalyzer(ProcessTools.createTestJavaProcessBuilder( - "-cp", artifacts.get("gov.nist.math.scimark-2.0").toString(), + "-cp", sciMark2Cp, "jnt.scimark2.commandline", "-large") .start()); output.shouldHaveExitValue(0); diff --git a/test/hotspot/jtreg/runtime/ErrorHandling/TestSAPSpecificOnOutOfMemoryError.java b/test/hotspot/jtreg/runtime/ErrorHandling/TestSAPSpecificOnOutOfMemoryError.java new file mode 100644 index 00000000000..32b82e53c58 --- /dev/null +++ b/test/hotspot/jtreg/runtime/ErrorHandling/TestSAPSpecificOnOutOfMemoryError.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestSAPSpecificOnOutOfMemoryError + * @summary Test SapMachine/SapJVM specific behavior + * @modules java.base/jdk.internal.misc + * @library /test/lib + * @run driver TestSAPSpecificOnOutOfMemoryError + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; +import java.util.ArrayList; + +public class TestSAPSpecificOnOutOfMemoryError { + + static OutputAnalyzer run_test(String ... vm_args) throws Exception { + ArrayList args = new ArrayList<>(); + for (String s : vm_args) { + args.add(s); + } + args.add("-Xmx128m"); + args.add(TestSAPSpecificOnOutOfMemoryError.class.getName()); + args.add("throwOOME"); + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(args); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + int exitValue = output.getExitValue(); + if (0 == exitValue) { + //expecting a non zero value + throw new Error("Expected to get non zero exit value"); + } + return output; + } + + public static void main(String[] args) throws Exception { + if (args.length == 1) { + // This should guarantee to throw: + // java.lang.OutOfMemoryError: Requested array size exceeds VM limit + Object[] oa = new Object[Integer.MAX_VALUE]; + return; + } + + final String aborting_due = "Aborting due to java.lang.OutOfMemoryError"; + final String terminating_due = "Terminating due to java.lang.OutOfMemoryError"; + final String java_frames = "Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)"; + final String a_fatal_error = "# A fatal error has been detected by the Java Runtime Environment"; + final String no_core = "CreateCoredumpOnCrash turned off, no core file dumped"; + final String yes_core_1 = "Core dump will be written."; // If limit > 0 + final String yes_core_2 = "Core dumps have been disabled."; // if limit == 0. For the purpose of this test this is still okay + final String summary_from_hs_err = "S U M M A R Y"; + + // CrashOnOutOfMemoryError, without cores explicitly enabled: + // - thread stack + // - aborting with hs-err file + // - no core + { + OutputAnalyzer output = run_test("-XX:+CrashOnOutOfMemoryError"); + + output.shouldContain(aborting_due); + output.shouldContain(java_frames); + output.shouldContain(a_fatal_error); + output.shouldContain(no_core); + + output.shouldNotContain(terminating_due); + output.shouldNotContain(yes_core_1); + output.shouldNotContain(yes_core_2); + } + + // ExitVMOnOutOfMemoryError is a SAP specific alias for CrashOnOutOfMemoryError + { + OutputAnalyzer output = run_test("-XX:+ExitVMOnOutOfMemoryError"); + + output.shouldContain(aborting_due); + output.shouldContain(java_frames); + output.shouldContain(a_fatal_error); + output.shouldContain(no_core); + + output.shouldNotContain(terminating_due); + output.shouldNotContain(yes_core_1); + output.shouldNotContain(yes_core_2); + } + + // CrashOnOutOfMemoryError, with cores explicitly enabled: + // - thread stack + // - aborting with hs-err file + // - core is to be written (or, attempted, if ulimit = 0) + { + OutputAnalyzer output = run_test("-XX:+CrashOnOutOfMemoryError", "-XX:+CreateCoredumpOnCrash"); + + output.shouldContain(aborting_due); + output.shouldContain(java_frames); + output.shouldContain(a_fatal_error); + output.shouldMatch("(" + yes_core_1 + "|" + yes_core_2 + ")"); + + output.shouldNotContain(no_core); + output.shouldNotContain(terminating_due); + } + + // ExitOnOutOfMemoryError should: + // - print thread stack + // - terminate the VM + { + OutputAnalyzer output = run_test("-XX:+ExitOnOutOfMemoryError"); + + output.shouldContain(terminating_due); + output.shouldContain(java_frames); + + output.shouldNotContain(aborting_due); + output.shouldNotContain(a_fatal_error); + output.shouldNotContain(yes_core_1); + output.shouldNotContain(yes_core_2); + output.shouldNotContain(no_core); + } + + // Test that giving ErrorFileToStdout in combination with CrashOnOutOfMemoryError will + // print the hs-err file - including the limits, which may be important here - to stdout + { + OutputAnalyzer output = run_test("-XX:+CrashOnOutOfMemoryError", "-XX:+ErrorFileToStdout"); + + output.shouldContain(aborting_due); + output.shouldContain(java_frames); + output.shouldContain(a_fatal_error); + output.shouldContain(no_core); + output.shouldContain(summary_from_hs_err); + + output.shouldNotContain(terminating_due); + output.shouldNotContain(yes_core_1); + output.shouldNotContain(yes_core_2); + } + + // HeapDumpOnOutOfMemoryError should: + // - print thread stack + // - The VM should just run on. OOM will bubble up and end the program. + { + OutputAnalyzer output = run_test("-XX:+HeapDumpOnOutOfMemoryError"); + + output.shouldContain(java_frames); + + output.shouldNotContain(terminating_due); + output.shouldNotContain(aborting_due); + output.shouldNotContain(a_fatal_error); + output.shouldNotContain(yes_core_1); + output.shouldNotContain(yes_core_2); + output.shouldNotContain(no_core); + } + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/CSVParser.java b/test/hotspot/jtreg/runtime/Vitals/CSVParser.java new file mode 100644 index 00000000000..c7a9227b1fb --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/CSVParser.java @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2022, SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +import java.util.ArrayList; +import java.util.Hashtable; +import java.util.List; + +public class CSVParser { + + public final static class CSVHeader { + final List columns = new ArrayList<>(); + Hashtable columnPositions = new Hashtable<>(); + + int size() { + return columns.size(); + } + + String at(int position) { + return columns.get(position); + } + + int findColumn(String name) { + Integer i = columnPositions.get(name); + return (i == null) ? -1 : i; + } + + boolean hasColumn(String name) { + return findColumn(name) != -1; + } + + void addColumn(String name) { + if (columnPositions.containsKey(name)) { + throw new RuntimeException("Already have column " + name); + } + columns.add(name); + columnPositions.put(name, columns.size() - 1); + } + + @Override + public String toString() { + StringBuilder bld = new StringBuilder(); + for (String s : columns) { + bld.append(s); + bld.append(","); + } + return bld.toString(); + } + } + + public final static class CSVDataLine { + ArrayList data = new ArrayList<>(); + + int size() { + return data.size(); + } + + String at(int position) { + return data.get(position); + } + + boolean isEmpty(int position) { + String s = at(position); + return s == null || s.isEmpty() || s.equals("?"); + } + + long numberAt(int position) throws NumberFormatException { + if (isEmpty(position)) { + throw new RuntimeException("no data at position " + position); + } + return Long.parseLong(at(position)); + } + + void addData(String s) { + // If data was surrounded by quotes, remove quotes + if (s.startsWith("\"") && s.endsWith("\"")) { + s = s.substring(1, s.length() - 1); + } + s = s.trim(); + data.add(s); + } + + @Override + public String toString() { + StringBuilder bld = new StringBuilder(); + for (String s : data) { + bld.append(s); + bld.append(","); + } + return bld.toString(); + } + } + + public final static class CSV { + public CSVHeader header; + public CSVDataLine[] lines; + + // Convenience function. Given a column name and a data line number, return value for that column in that line + String getContentOfCell(String columnname, int lineno) { + return lines[lineno].at(header.findColumn(columnname)); + } + + long getContentOfCellAsNumber(String columnname, int lineno) { + return Long.parseLong(getContentOfCell(columnname, lineno)); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("CSV: " + header.size() + " columns, " + lines.length + " data lines.\n"); + builder.append(header); + builder.append("\n"); + for (CSVDataLine line : lines) { + builder.append(line); + builder.append("\n"); + } + return builder.toString(); + } + } + + public static class CSVParseException extends Exception { + public CSVParseException(String message) { + super("CSV parse error " + ": " + message); + } + public CSVParseException(String message, int errorLine) { + super("CSV parse error at line " + errorLine + ": " + message); + } + } + + /** + * Parses the given lines as CSV. The first lines must be the header, all subsequent lines + * valid data. + * @param lines + * @return + */ + public static final CSV parseCSV(String [] lines) throws CSVParseException { + + if (lines.length < 2) { + throw new CSVParseException("Not enough data", -1); + } + + System.out.println("--- CSV parser input: ---"); + for (String s : lines) { + System.out.println(s); + } + System.out.println("--- /CSV parser input: ---"); + + int lineno = 0; + CSVHeader header = new CSVHeader(); + ArrayList datalines = new ArrayList<>(); + + try { + // Parse header line + String[] parts = lines[lineno].split(","); + for (String s : parts) { + header.addColumn(s); + } + + lineno ++; + + // Parse Data + while (lineno < lines.length) { + CSVDataLine dataLine = new CSVDataLine(); + parts = lines[lineno].split(","); + if (parts.length == 1 && parts[0].isEmpty()) { + // We start a new section here. Skip the rest. + break; + } + for (String s : parts) { + dataLine.addData(s); + } + if (dataLine.size() != header.size()) { + // We have more or less data than columns. Print some helpful message to stderr, then abort. + String s = "Line " + lineno + ": expected " + header.size() + " entries, found " + dataLine.size() + "."; + System.err.println(s); + System.err.println("Header: " + header.toString()); + System.err.println("Data: " + dataLine.toString()); + for (int i = 0; i < header.size() || i < dataLine.size(); i ++) { + String col = (i < header.size() ? header.at(i) : ""); + String dat = (i < dataLine.size() ? dataLine.at(i) : ""); + System.err.println("pos: " + i + " column: " + col + " data: " + dat); + } + throw new CSVParseException(s, lineno); + } + datalines.add(dataLine); + lineno ++; + } + + } catch (Exception e) { + System.err.println("--- CSV parse error : " + e.getMessage() + "---"); + e.printStackTrace(); + System.err.println("--- /CSV parse error : " + e.getMessage() + "---"); + throw new CSVParseException(e.getMessage(), lineno); + } + + CSV csv = new CSV(); + csv.header = header; + CSVDataLine[] arr = new CSVDataLine[datalines.size()]; + csv.lines = datalines.toArray(arr); + + System.out.println("---- parsed ----"); + System.out.println(csv); + System.out.println("---- /parsed ----"); + + return csv; + + } + +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestHiMemReport.java b/test/hotspot/jtreg/runtime/Vitals/TestHiMemReport.java new file mode 100644 index 00000000000..362d3dd7afa --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestHiMemReport.java @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestHiMemReport-print + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReport print + */ + +/* + * @test TestHiMemReport-dump + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReport dump + */ + +/* + * @test TestHiMemReport-dump-with-exec-to-reportdir + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReport dump-with-exec-to-reportdir + */ + +/* + * @test TestHiMemReport-dump-with-exec-to-stderr + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReport dump-with-exec-to-stderr + */ + +/* + * @test TestHiMemReport-natural-max + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReport natural-max + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +import java.io.File; +import java.io.IOException; + +public class TestHiMemReport { + + // Match the output file suffix we generate with the HiMemReportFacility + // (xxx_pid______.yyy) + static final String patterFileSuffix = "_pid\\d+_\\d+_\\d+_\\d+_\\d+_\\d+_\\d+"; + + // These tests are very simple. We start the VM with a footprint (rss) higher than + // what we set as HiMemReportMax. Therefore, the HiMem reporter should genereate a + // 100% report right away. + // Testing anything more is much more complicated since the test would have to + // predict, with a certain correctness, rss and swap. + + static final String[] beforeReport = { + "HiMemoryReport: rss\\+swap=.* - alert level increased to 3 \\(>=\\d+%\\).", + "HiMemoryReport: ... seems we passed alert level 1 \\(\\d+%\\) without noticing.", + "HiMemoryReport: ... seems we passed alert level 2 \\(\\d+%\\) without noticing.", + }; + + static final String[] reportHeader = { + "# High Memory Report:", + "# rss\\+swap .* larger than \\d+% of HiMemReportMax.*", + "# Spike number: 1", + }; + + static final String[] reportBody = { + ".*Vitals.*", + "Now:", + "Native Memory Tracking:", + "Total: reserved=.*, committed=.*", + "# END: High Memory Report" + }; + + static void testPrint() throws Exception { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-XX:HiMemReportMax=128m", + "-XX:NativeMemoryTracking=summary", + "-Xmx128m", "-Xms128m", "-XX:+AlwaysPreTouch", + TestHiMemReport.class.getName(), + "sleep", "2" // num seconds to sleep to give the reporter thread time to generate output + ); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + + String[] lines = VitalsUtils.stderrAsLines(output); + int line = VitalsUtils.matchPatterns(lines, 0, beforeReport); + line = VitalsUtils.matchPatterns(lines, line + 1, reportHeader); + line = VitalsUtils.matchPatterns(lines, line + 1, reportBody); + } + + static void testDump() throws Exception { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-XX:HiMemReportMax=128m", + "-XX:NativeMemoryTracking=summary", + "-XX:HiMemReportDir=himemreport-1", + "-Xmx128m", "-Xms128m", "-XX:+AlwaysPreTouch", + TestHiMemReport.class.getName(), + "sleep", "2" // num seconds to sleep to give the reporter thread time to generate output + ); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + + // We expect, on stderr, just the report header + String[] lines = VitalsUtils.stderrAsLines(output); + int line = VitalsUtils.matchPatterns(lines, 0, beforeReport); + line = VitalsUtils.matchPatterns(lines, line + 1, reportHeader); + + // We expect a report in a file inside HiMemReportDir, and a mentioning of it on + // stderr + line = VitalsUtils.matchPatterns(lines, line + 1, + new String[] { "# Printing to .*himemreport-1/sapmachine_himemalert" + patterFileSuffix + ".log", + "# Done\\." } ); + + String reportFile = output.firstMatch("# Printing to (.*himemreport-1/sapmachine_himemalert" + patterFileSuffix + ".log)", 1); + VitalsUtils.assertFileExists(reportFile); + + VitalsUtils.assertFileContentMatches(new File(reportFile), reportBody); + } + + static void testDumpWithExecToReportDir() throws Exception { + String reportDirName = "himemreport-2"; + // Here we not only dump, but we also execute several jcmds. Therefore we take some more seconds to sleep + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-XX:HiMemReportMax=128m", + "-XX:HiMemReportDir=himemreport-2", + "-XX:HiMemReportExec=VM.flags -all;VM.metaspace show-loaders;GC.heap_dump", + "-XX:NativeMemoryTracking=summary", + "-Xmx128m", "-Xms128m", "-XX:+AlwaysPreTouch", + TestHiMemReport.class.getName(), + "sleep", "12" // num seconds to sleep to give the reporter thread time to generate output + ); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + + // We expect, on stderr, just the report header + String[] lines = VitalsUtils.stderrAsLines(output); + int line = VitalsUtils.matchPatterns(lines, 0, beforeReport); + line = VitalsUtils.matchPatterns(lines, line + 1, reportHeader); + + // We expect a report in a file inside HiMemReportDir, and a mentioning of it on + // stderr + line = VitalsUtils.matchPatterns(lines, line + 1, + new String[] { "# Printing to .*himemreport-2/sapmachine_himemalert" + patterFileSuffix + ".log", + "# Done\\." } ); + + String reportFile = output.firstMatch("# Printing to (.*himemreport-2/sapmachine_himemalert" + patterFileSuffix + ".log)", 1); + VitalsUtils.assertFileExists(reportFile); + VitalsUtils.assertFileContentMatches(new File(reportFile), reportBody); + + // We also expect, in the report dir, several more files + String reportDir = output.firstMatch("# Printing to (.*himemreport-2/)sapmachine_himemalert" + patterFileSuffix + ".log", 1); + VitalsUtils.assertFileExists(reportDir); + File reportDirAsFile = new File(reportDir); + + //////// + // HiMemReportExec should have caused three jcmds to fire... + + line = VitalsUtils.matchPatterns(lines, line + 1, + new String[] { "HiMemReport: Successfully executed \"VM.flags -all\" \\(\\d+ ms\\), output redirected to report dir", + "HiMemReport: Successfully executed \"VM.metaspace show-loaders\" \\(\\d+ ms\\), output redirected to report dir", + "HiMemReport: Successfully executed \"GC.heap_dump .*himemreport-2/GC.heap_dump" + patterFileSuffix + ".dump\" \\(\\d+ ms\\), output redirected to report dir" } ); + + // VM.flags: + + // I expect two files, VM.flags_pidXXXX_1_100.err and VM.flags_pidXXXX_1_100.out, in the report dir... + File VMflagsOutFile = VitalsUtils.findFirstMatchingFileInDirectory(reportDirAsFile, "VM.flags" + patterFileSuffix + ".out"); + File VMflagsErrFile = VitalsUtils.findFirstMatchingFileInDirectory(reportDirAsFile, "VM.flags" + patterFileSuffix + ".err"); + + // The err file should be empty + VitalsUtils.assertFileisEmpty(VMflagsErrFile.getAbsolutePath()); + + // The out file should contain a valid flags report, containing, among other things, the flags we passed above + // Note that since we started VM.flags with -all, we have the full flags output + VitalsUtils.assertFileContentMatches(VMflagsOutFile, new String[] { + ".*HiMemReport *= *true.*", + ".*HiMemReportDir *= *himemreport-2.*", + ".*HiMemReportExec *= *VM.flags.*VM.metaspace.*GC.heap_dump.*", + ".*HiMemReportMax *= *134217728.*" + }); + + // "VM.metaspace": + + // I expect two files, VM.metaspace_pidXXXX_1_100.err and VM.metaspace_pidXXXX_1_100.out, in the report dir... + File VMmetaspaceOutFile = VitalsUtils.findFirstMatchingFileInDirectory(reportDirAsFile, "VM.metaspace" + patterFileSuffix + ".out"); + File VMmetaspaceErrFile = VitalsUtils.findFirstMatchingFileInDirectory(reportDirAsFile, "VM.metaspace" + patterFileSuffix + ".err"); + + // The err file should be empty + VitalsUtils.assertFileisEmpty(VMmetaspaceErrFile.getAbsolutePath()); + + // The out file should contain a valid flags report, containing, among other things, the flags we passed above + // Note that since we started VM.flags with -all, we have the full flags output + VitalsUtils.assertFileContentMatches(VMmetaspaceOutFile, new String[] { + "Usage per loader:", + ".*app.*", + ".*bootstrap.*", + "Total Usage.*", + "Virtual space.*", + "Settings.*", + "MaxMetaspaceSize.*" + }); + + // GC.heap_dump: + + // I expect three files, the usual GC.heap_dump_pid_.(out|err) and the heap dump itself as + // GC.heap_dump_pid_.dump. + File heapDumpOutFile = VitalsUtils.findFirstMatchingFileInDirectory(reportDirAsFile, "GC.heap_dump" + patterFileSuffix + ".out"); + File heapDumpErrFile = VitalsUtils.findFirstMatchingFileInDirectory(reportDirAsFile, "GC.heap_dump" + patterFileSuffix + ".err"); + File heapDumpDumpFile = VitalsUtils.findFirstMatchingFileInDirectory(reportDirAsFile, "GC.heap_dump" + patterFileSuffix + ".dump"); + + // The err file should be empty + VitalsUtils.assertFileisEmpty(heapDumpErrFile.getAbsolutePath()); + + // The out file should contain "dumped blabla" + VitalsUtils.assertFileContentMatches(heapDumpOutFile, new String[] { + "Dumping heap to.*", + "Heap dump file created.*" + }); + + // The real dump file should exist and be reasonably sized. + VitalsUtils.assertFileExists(heapDumpDumpFile); + if (heapDumpDumpFile.length() < 1024 * 16) { + throw new RuntimeException("heap dump suspiciously small."); + } + } // end: testDumpWithExecToReportDir + + // Test multiple Execs, with the output of the commands going to stderr + static void testDumpWithExecToStderr() throws Exception { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-XX:HiMemReportMax=128m", + "-XX:HiMemReportExec=VM.flags -all;VM.metaspace show-loaders", + "-XX:NativeMemoryTracking=summary", + "-Xmx128m", "-Xms128m", "-XX:+AlwaysPreTouch", + TestHiMemReport.class.getName(), + "sleep", "8" // num seconds to sleep to give the reporter thread time to generate output + ); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + + // We expect the normal HiMemReport on stderr... + String[] lines = VitalsUtils.stderrAsLines(output); + int line = VitalsUtils.matchPatterns(lines, 0, beforeReport); + line = VitalsUtils.matchPatterns(lines, line + 1, reportHeader); + line = VitalsUtils.matchPatterns(lines, line + 1, reportBody); + + // ... as well as the output of VM.flags + line = VitalsUtils.matchPatterns(lines, line + 1, new String [] { + ".*HiMemReport *= *true.*", + ".*HiMemReportMax *= *134217728.*", + "HiMemReport: Successfully executed \"VM.flags -all\" \\(\\d+ ms\\)" + }); + + // ... as well as the output of VM.metaspace + line = VitalsUtils.matchPatterns(lines, line + 1, new String [] { + "Usage per loader:", + ".*app.*", + ".*bootstrap.*", + "Total Usage.*", + "Virtual space.*", + "Settings.*", + "MaxMetaspaceSize.*", + "HiMemReport: Successfully executed \"VM.metaspace show-loaders\" \\(\\d+ ms\\)" + }); + + } // end: testDumpWithExecToReportDir + + + + /** + * test that HiMemReport can feel out some limit from the environment without being given one explicitely + * (I'm not sure that this always works, but I would like to know if it does not, and if not, in what context) + */ + static void testHasNaturalMax() throws IOException { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-Xlog:vitals", "-Xmx64m", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldHaveExitValue(0); + output.shouldNotMatch("HiMemReport.*limit could not be established"); + } + + public static void main(String[] args) throws Exception { + if (args[0].equals("print")) + testPrint(); + else if (args[0].equals("dump")) + testDump(); + else if (args[0].equals("dump-with-exec-to-reportdir")) + testDumpWithExecToReportDir(); + else if (args[0].equals("dump-with-exec-to-stderr")) + testDumpWithExecToStderr(); + else if (args[0].equals("natural-max")) + testHasNaturalMax(); + else if (args[0].equals("sleep")) { + int numSeconds = Integer.parseInt(args[1]); + Thread.sleep(numSeconds * 1000); + } else + throw new RuntimeException("Invalid test " + args[0]); + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportArgParsing.java b/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportArgParsing.java new file mode 100644 index 00000000000..96b3ba3a8d5 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportArgParsing.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestHiMemReportArgParsing-ValidNonExistingReportDir + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportArgParsing ValidNonExistingReportDir + */ + +/* + * @test TestHiMemReportArgParsing-ValidExistingReportDir + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportArgParsing ValidExistingReportDir + */ + +/* + * @test TestHiMemReportArgParsing-InValidReportDir + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportArgParsing InValidReportDir + */ + +/* + * @test TestHiMemReportArgParsing-HiMemReportOn + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportArgParsing HiMemReportOn + */ + +/* + * @test TestHiMemReportArgParsing-HiMemReportOff + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportArgParsing HiMemReportOff + */ + +/* + * @test TestHiMemReportArgParsing-HiMemReportOffByDefault + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportArgParsing HiMemReportOffByDefault + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +import java.io.File; +import java.io.IOException; + +public class TestHiMemReportArgParsing { + + /** + * test HiMemReportDir with a valid, absolute path to a non-existing directory + */ + static void testValidNonExistingReportDir() throws IOException { + File subdir = VitalsUtils.createSubTestDir("test-outputdir-1", false); + VitalsUtils.fileShouldNotExist(subdir); + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-XX:HiMemReportDir=" + subdir.getAbsolutePath(), "-Xlog:vitals", + "-Xmx64m", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + VitalsUtils.outputStdoutMatchesPatterns(output, new String[] { + ".*[vitals].*HiMemReportDir: Created report directory.*" + subdir.getName() + ".*", + ".*[vitals].*HiMemReport subsystem initialized.*" + }); + VitalsUtils.fileShouldExist(subdir); + } + + /** + * test HiMemReportDir with a valid, absolute path to an existing directory + */ + static void testValidExistingReportDir() throws IOException { + File subdir = VitalsUtils.createSubTestDir("test-outputdir-2", true); + VitalsUtils.fileShouldExist(subdir); + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-XX:HiMemReportDir=" + subdir.getAbsolutePath(), "-Xlog:vitals", + "-Xmx64m", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + VitalsUtils.outputStdoutMatchesPatterns(output, new String[] { + ".*[vitals].*Found existing report directory at.*" + subdir.getName() + ".*", + ".*[vitals].*HiMemReport subsystem initialized.*" + }); + VitalsUtils.fileShouldExist(subdir); + } + + /** + * test HiMemReportDir with an invalid path (containing several layers, which the VM will not create, it will only + * create subdirs of max 1 level). We explicitly omit Xlog:vitals, but still expect a warning + */ + static void testInValidReportDir() throws IOException { + File f = new File("/tmp/gibsnicht/gibsnicht/gibsnicht"); + VitalsUtils.fileShouldNotExist(f); + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-XX:HiMemReportDir=" + f.getAbsolutePath(), "-Xlog:vitals", + "-Xmx64m", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldNotHaveExitValue(0); + VitalsUtils.outputStdoutMatchesPatterns(output, new String[] { + ".*[vitals].*Failed to create report directory.*" + f.getAbsolutePath() + ".*", + "Error occurred during initialization of VM" + }); + VitalsUtils.fileShouldNotExist(f); + } + + /** + * test +HiMemReport without any further options. It should come up with a reasonable limit. + */ + static void testHiMemReportOn() throws IOException { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-Xlog:vitals", + "-Xmx64m", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + VitalsUtils.outputStdoutMatchesPatterns(output, new String[] { + ".*[vitals].*Setting limit to.*", + ".*[vitals].*HiMemReport subsystem initialized.*" + }); + } + + /** + * test that -HiMemReport means off + */ + static void testHiMemReportOff() throws IOException { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:-HiMemReport", "-Xlog:vitals", + "-Xmx64m", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + output.shouldNotContain("HiMemReport subsystem initialized"); + } + + /** + * test that HiMemReport is off by default + */ + static void testHiMemReportOffByDefault() throws IOException { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-Xlog:vitals", + "-Xmx64m", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + output.shouldNotContain("HiMemReport subsystem initialized"); + } + + public static void main(String[] args) throws Exception { + if (args[0].equals("ValidNonExistingReportDir")) + testValidNonExistingReportDir(); + else if (args[0].equals("ValidExistingReportDir")) + testValidExistingReportDir(); + else if (args[0].equals("InValidReportDir")) + testInValidReportDir(); + else if (args[0].equals("HiMemReportOn")) + testHiMemReportOn(); + else if (args[0].equals("HiMemReportOff")) + testHiMemReportOff(); + else if (args[0].equals("HiMemReportOffByDefault")) + testHiMemReportOffByDefault(); + else + throw new RuntimeException("Invalid test " + args[0]); + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportExecParsing.java b/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportExecParsing.java new file mode 100644 index 00000000000..ab448c7f61d --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportExecParsing.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestHiMemReportExecParsing + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportExecParsing 1 + */ + +/* + * @test TestHiMemReportExecParsing + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportExecParsing 2 + */ + +/* + * @test TestHiMemReportExecParsing + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportExecParsing 3 + */ + +/* + * @test TestHiMemReportExecParsing + * @library /test/lib + * @requires os.family == "linux" + * @run driver TestHiMemReportExecParsing 4 + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +import java.io.IOException; + +public class TestHiMemReportExecParsing { + + static void do_test(String execString, boolean shouldSucceed, String... shouldContain) throws IOException { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+HiMemReport", "-Xlog:vitals", "-XX:HiMemReportExec=" + execString, + "-Xmx64m", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + if (shouldSucceed) { + output.shouldHaveExitValue(0); + } else { + output.shouldNotHaveExitValue(0); + } + for (String s : shouldContain) { + output.shouldContain(s); + } + } + + public static void main(String[] args) throws Exception { + int variant = Integer.parseInt(args[0]); + switch (variant) { + case 1: + do_test("VM.metaspace", true, "HiMemReportExec: storing command \"VM.metaspace\""); + break; + case 2: + do_test("VM.info;VM.metaspace;GC.heap_dump", true, + "HiMemReportExec: storing command \"VM.info\"", "HiMemReportExec: storing command \"VM.metaspace\"", + "HiMemReportExec: storing command \"GC.heap_dump\""); + break; + case 3: + do_test("; VM.info;; ; VM.metaspace show-loaders ", true, + "HiMemReportExec: storing command \"VM.info\"", "HiMemReportExec: storing command \"VM.metaspace show-loaders\""); + break; + case 4: + do_test(" hallo ", false, + "HiMemReportExec: Command \"hallo\" invalid"); + break; + } + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportInHSerr.java b/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportInHSerr.java new file mode 100644 index 00000000000..4afcfa4d223 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestHiMemReportInHSerr.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2022, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary Test that HiMemReport overview appears in hs-err file (since this likes to break when merging from upstream) + * @library /test/lib + * @requires vm.debug & os.family == "linux" + * @modules java.base/jdk.internal.misc + * java.management + * @run driver TestHiMemReportInHSerr + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + + +// Test that, when we crash, we have Vitals as expected in the hs-err file +public class TestHiMemReportInHSerr { + + public static void main(String[] args) throws Exception { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+UnlockDiagnosticVMOptions", + "-Xmx100M", + "-XX:-CreateCoredumpOnCrash", + "-XX:ErrorHandlerTest=14", // sigsegv + "-XX:HiMemReportMax=3g", "-XX:+HiMemReport", + "-XX:+ErrorFileToStdout", + "-version"); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldNotHaveExitValue(0); + + output.shouldMatch("# A fatal error has been detected by the Java Runtime Environment:.*"); + + output.shouldMatch("HiMemReport: monitoring rss\\+swap vs HiMemReportMax.*\\(3145728 K\\), all is well, spikes: 0, alerts: 0.*"); + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestVitalsAtExit.java b/test/hotspot/jtreg/runtime/Vitals/TestVitalsAtExit.java new file mode 100644 index 00000000000..9f87aa0c5a2 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestVitalsAtExit.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestVitalsAtExit + * @summary Test verifies that -XX:+PrintVitalsAtExit prints vitals at exit. + * @library /test/lib + * @run driver TestVitalsAtExit print + */ + +/* + * @test TestVitalsAtExit + * @summary Test verifies that -XX:+DumpVitalsAtExit works + * @library /test/lib + * @run driver TestVitalsAtExit dump + */ + +import jdk.test.lib.Asserts; +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +import java.io.File; + +public class TestVitalsAtExit { + + + public static void main(String[] args) throws Exception { + if (args.length == 0) { + try { + Thread.sleep(5000); // we start with interval=1, so give us some secs to gather samples + } catch (InterruptedException err) { + } + return; + } + if (args[0].equals("print")) { + testPrint(); + } else if (args[0].equals("dump")) { + testDump(); + } else { + throw new RuntimeException("invalid argument " + args[0]); + } + } + + static void testPrint() throws Exception { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+EnableVitals", + "-XX:+PrintVitalsAtExit", + "-XX:VitalsSampleInterval=1", + "-XX:MaxMetaspaceSize=16m", + "-Xmx128m", + TestVitalsAtExit.class.getName()); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldHaveExitValue(0); + output.stdoutShouldNotBeEmpty(); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + } + + static void testDump() throws Exception { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+EnableVitals", + "-XX:+DumpVitalsAtExit", + "-XX:VitalsFile=abcd", + "-XX:VitalsSampleInterval=1", + "-XX:MaxMetaspaceSize=16m", + "-Xmx128m", + TestVitalsAtExit.class.getName()); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldHaveExitValue(0); + output.stdoutShouldNotBeEmpty(); + output.shouldContain("Dumping Vitals to abcd.txt"); + output.shouldContain("Dumping Vitals csv to abcd.csv"); + File text_dump = new File("abcd.txt"); + Asserts.assertTrue(text_dump.exists() && text_dump.isFile(), + "Could not find abcd.txt"); + File csv_dump = new File("abcd.csv"); + Asserts.assertTrue(csv_dump.exists() && csv_dump.isFile(), + "Could not find abcd.csv"); + + VitalsTestHelper.fileMatchesVitalsTextMode(text_dump); + VitalsTestHelper.fileMatchesVitalsCSVMode(csv_dump); + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestVitalsInvalidSampleInterval.java b/test/hotspot/jtreg/runtime/Vitals/TestVitalsInvalidSampleInterval.java new file mode 100644 index 00000000000..2033a39524a --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestVitalsInvalidSampleInterval.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestVitalsInvalidSampleInterval + * @summary Test verifies that -XX:-EnableVitals disables vitals + * @library /test/lib + * @run driver TestVitalsInvalidSampleInterval run + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +public class TestVitalsInvalidSampleInterval { + + public static void main(String[] args) throws Exception { + // Invalid Sample interval prints a warning and runs with Vitals disabled + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:VitalsSampleInterval=0", + "-XX:MaxMetaspaceSize=16m", + "-Xmx128m", + "-version"); // Note: explicitly omit Xlog:vitals, since the warning should always appear + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldHaveExitValue(1); + output.shouldContain("Improperly specified VM option 'VitalsSampleInterval=0'"); + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestVitalsOff.java b/test/hotspot/jtreg/runtime/Vitals/TestVitalsOff.java new file mode 100644 index 00000000000..8dc483323b1 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestVitalsOff.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestVitalsOff + * @summary Test verifies that -XX:-EnableVitals disables vitals + * @library /test/lib + * @run driver TestVitalsOff run + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +public class TestVitalsOff { + + public static void main(String[] args) throws Exception { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:-EnableVitals", + "-XX:MaxMetaspaceSize=16m", + "-Xlog:vitals", + "-Xmx128m", + "-version"); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldHaveExitValue(0); + output.shouldNotContain("Initializing Vitals"); + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestVitalsValidSampleInterval.java b/test/hotspot/jtreg/runtime/Vitals/TestVitalsValidSampleInterval.java new file mode 100644 index 00000000000..616a1ed5261 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestVitalsValidSampleInterval.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestVitalsValidSampleInterval + * @summary Test verifies that -XX:-EnableVitals disables vitals + * @library /test/lib + * @run driver TestVitalsValidSampleInterval run + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +import java.io.IOException; + +public class TestVitalsValidSampleInterval { + + static void runTest(int interval) throws IOException { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:VitalsSampleInterval=" + interval, + "-XX:MaxMetaspaceSize=16m", + "-Xlog:vitals=debug", + "-Xmx128m", + "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldHaveExitValue(0); + output.shouldContain("Vitals initialized."); + output.shouldContain("Vitals sample interval: " + interval + " seconds"); + } + + public static void main(String[] args) throws Exception { + runTest(1); + runTest(23); + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/TestVitalsinHSerr.java b/test/hotspot/jtreg/runtime/Vitals/TestVitalsinHSerr.java new file mode 100644 index 00000000000..d0f3ff5abb1 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/TestVitalsinHSerr.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary Test that Vitals report appears in hs-err file + * @library /test/lib + * @requires vm.debug + * @modules java.base/jdk.internal.misc + * java.management + * @run driver TestVitalsinHSerr + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + + +// Test that, when we crash, we have Vitals as expected in the hs-err file +public class TestVitalsinHSerr { + + public static void main(String[] args) throws Exception { + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder( + "-XX:+UnlockDiagnosticVMOptions", + "-Xmx100M", + "-XX:-CreateCoredumpOnCrash", + "-XX:ErrorHandlerTest=14", // sigsegv + "-XX:+ErrorFileToStdout", + "-version"); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldNotHaveExitValue(0); + + output.shouldMatch("# A fatal error has been detected by the Java Runtime Environment:.*"); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + // We want to see the Now value too. Dumping crash reports is about the only place we want + // the sample exactly when we report. + output.shouldContain("Now"); // hs-err file (in this case, dumped to stdout) should contain vitals + + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/ValuesFromProcFS.java b/test/hotspot/jtreg/runtime/Vitals/ValuesFromProcFS.java new file mode 100644 index 00000000000..bd9c3008efb --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/ValuesFromProcFS.java @@ -0,0 +1,78 @@ +import java.io.File; +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +class ValuesFromProcFS { + + // from proc meminfo + public long MemAvail = -1; + public long Committed_AS = -1; + public long SwapTotal = -1; + public long SwapFree = -1; + + private static Pattern patMemAvail = Pattern.compile("MemAvail: *(\\d+) *kB"); + private static Pattern patCommitted_AS = Pattern.compile("Committed_AS: *(\\d+) *kB"); + private static Pattern patSwapTotal = Pattern.compile("SwapTotal: *(\\d+) *kB"); + private static Pattern patSwapFree = Pattern.compile("SwapFree: *(\\d+) *kB"); + + // from proc pid status + public long VmRSS = -1; + public long VmSwap = -1; + public long VmSize = -1; + + private static Pattern patVmRSS = Pattern.compile("VmRSS: *(\\d+) *kB"); + private static Pattern patVmSize = Pattern.compile("VmSize: *(\\d+) *kB"); + private static Pattern patVmSwap = Pattern.compile("VmSwap: *(\\d+) *kB"); + + /** + * Retrieve data from proc fs + * + * @param pid if != -1, return some data for the process too + * @return + * @throws IOException + */ + public static ValuesFromProcFS retrieveForProcess(long pid) throws IOException { + ValuesFromProcFS v = new ValuesFromProcFS(); + + if (pid != -1) { + String lines[] = VitalsUtils.fileAsLines(new File("/proc/" + pid + "/status")); + for (String s : lines) { + Matcher m = patVmRSS.matcher(s); + if (m.matches()) { + v.VmRSS = Long.parseLong(m.group(1)) * 1024; + } + m = patVmSize.matcher(s); + if (m.matches()) { + v.VmSize = Long.parseLong(m.group(1)) * 1024; + } + m = patVmSwap.matcher(s); + if (m.matches()) { + v.VmSwap = Long.parseLong(m.group(1)) * 1024; + } + } + } + + String lines[] = VitalsUtils.fileAsLines(new File("/proc/meminfo")); + for (String s : lines) { + Matcher m = patMemAvail.matcher(s); + if (m.matches()) { + v.MemAvail = Long.parseLong(m.group(1)) * 1024; + } + m = patCommitted_AS.matcher(s); + if (m.matches()) { + v.Committed_AS = Long.parseLong(m.group(1)) * 1024; + } + m = patSwapTotal.matcher(s); + if (m.matches()) { + v.SwapTotal = Long.parseLong(m.group(1)) * 1024; + } + m = patSwapFree.matcher(s); + if (m.matches()) { + v.SwapFree = Long.parseLong(m.group(1)) * 1024; + } + } + + return v; + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/VitalsDCmdStressTest.java b/test/hotspot/jtreg/runtime/Vitals/VitalsDCmdStressTest.java new file mode 100644 index 00000000000..2ba0dc94832 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/VitalsDCmdStressTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2020, SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc + * java.compiler + * java.management + * jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -XX:VitalsSampleInterval=1 VitalsDCmdStressTest + */ + +import jdk.test.lib.dcmd.CommandExecutor; +import jdk.test.lib.dcmd.JMXExecutor; +import jdk.test.lib.process.OutputAnalyzer; +import org.testng.annotations.Test; + +public class VitalsDCmdStressTest { + + final long runtime_secs = 30; + + public void run_once(CommandExecutor executor, boolean silent) { + OutputAnalyzer output = executor.execute("VM.vitals now", silent); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + output.shouldContain("Now"); + } + + public void run(CommandExecutor executor) { + // Let vitals run a while and bombard it with report requests which include "now" sampling + long t1 = System.currentTimeMillis(); + long t2 = t1 + runtime_secs * 1000; + int invocations = 0; + while (System.currentTimeMillis() < t2) { + run_once(executor, true); // run silent to avoid filling logs with garbage output + invocations ++; + } + + // One last time run with silent off + run_once(executor, false); + invocations ++; + + System.out.println("Called " + invocations + " times."); + } + + @Test + public void jmx() { + run(new JMXExecutor()); + // wait two seconds to collect some samples, then repeat with filled tables + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + run(new JMXExecutor()); + } + +} diff --git a/test/hotspot/jtreg/runtime/Vitals/VitalsDCmdTest.java b/test/hotspot/jtreg/runtime/Vitals/VitalsDCmdTest.java new file mode 100644 index 00000000000..33d25ad2229 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/VitalsDCmdTest.java @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2020,2022 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=1 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=2 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=3 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=4 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=5 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=6 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=7 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=8 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=9 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=10 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +/* + * @test + * @summary Test of diagnostic command VM.vitals + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @run testng/othervm -Dsapmachine.vitalstest=11 -XX:VitalsSampleInterval=1 VitalsDCmdTest + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.dcmd.CommandExecutor; +import jdk.test.lib.dcmd.JMXExecutor; +import org.testng.annotations.Test; + +public class VitalsDCmdTest { + + public void run(CommandExecutor executor) { + + try { + + int testnumber = Integer.parseInt(System.getProperties().getProperty("sapmachine.vitalstest")); + + switch (testnumber) { + case 1: + OutputAnalyzer output = executor.execute("VM.vitals"); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + output.shouldMatch("\\d+[gkm]"); // we print by default in "dynamic" scale which should show some values as k or m or g + output.shouldNotContain("Now"); // off by default + break; + case 2: + output = executor.execute("VM.vitals reverse"); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + output.shouldNotContain("Now"); // off by default + break; + case 3: + output = executor.execute("VM.vitals scale=m"); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + output.shouldNotMatch("\\d+[km]"); // A specific scale disables dynamic scaling, and we omit the unit suffix + output.shouldNotContain("Now"); // off by default + break; + case 4: + output = executor.execute("VM.vitals scale=1"); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + output.shouldNotMatch("\\d+[km]"); // A specific scale disables dynamic scaling, and we omit the unit suffix + output.shouldNotContain("Now"); // off by default + break; + case 5: + output = executor.execute("VM.vitals raw"); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + output.shouldNotContain("Now"); // off by default + break; + case 6: + output = executor.execute("VM.vitals now"); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + output.shouldContain("Now"); + break; + case 7: + output = executor.execute("VM.vitals reverse now"); + VitalsTestHelper.outputMatchesVitalsTextMode(output); + output.shouldContain("Now"); + break; + case 8: + output = executor.execute("VM.vitals csv"); + VitalsTestHelper.outputMatchesVitalsCSVMode(output); + output.shouldNotContain("Now"); // off always in csv mode + VitalsTestHelper.parseCSV(output); + break; + case 9: + output = executor.execute("VM.vitals csv reverse"); + VitalsTestHelper.outputMatchesVitalsCSVMode(output); + output.shouldNotContain("Now"); // off always in csv mode + VitalsTestHelper.parseCSV(output); + break; + case 10: { + output = executor.execute("VM.vitals csv reverse raw"); + VitalsTestHelper.outputMatchesVitalsCSVMode(output); + output.shouldNotContain("Now"); // off always in csv mode + CSVParser.CSV csv = VitalsTestHelper.parseCSV(output); + VitalsTestHelper.simpleCSVSanityChecks(csv); // requires raw or scale=1 mode + } + break; + case 11: { + output = executor.execute("VM.vitals csv now reverse scale=1"); + VitalsTestHelper.outputMatchesVitalsCSVMode(output); + // "Now" sample printing always off in csv mode even if explicitly given. + output.shouldNotContain("Now"); + output.shouldContain("\"now\" ignored in csv mode"); + CSVParser.CSV csv = VitalsTestHelper.parseCSV(output); + VitalsTestHelper.simpleCSVSanityChecks(csv); // requires raw or scale=1 mode + } + break; + default: + throw new RuntimeException("unknown test number " + testnumber); + } + + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e.getMessage()); + } + + } + + @Test + public void jmx() { + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + run(new JMXExecutor()); + } + +} diff --git a/test/hotspot/jtreg/runtime/Vitals/VitalsTestHelper.java b/test/hotspot/jtreg/runtime/Vitals/VitalsTestHelper.java new file mode 100644 index 00000000000..315013a7c2f --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/VitalsTestHelper.java @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2022, SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +import jdk.test.lib.Platform; +import jdk.test.lib.process.OutputAnalyzer; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.regex.Pattern; + + +public class VitalsTestHelper { + +// Last 60 minutes: +// ---------------------------------------system---------------------------------------- -------------------------process-------------------------- ----------------------------------------jvm----------------------------------------- +// ------cpu------ ------------cgroup------------- -------rss-------- -cheap-- -cpu- ----io---- --heap--- ----------meta---------- --nmt--- ----jthr----- --cldg-- ----cls----- +// avail comm crt swap si so p t pr pb us sy id st gu lim limsw slim usg usgsw kusg virt all anon file shm swdo usd free us sy of rd wr thr comm used comm used csc csu gctr code mlc map num nd cr st num anon num ld uld +// 2022-05-14 14:52:36 54.4g 21.7g 65 0k 0 0 2 22 2 0 3 0 96 0 0 8.0g 16.0g 118m 118m 2m 5.1g 109m 72m 38m 0k 0k 44m 7m 0 0 4 11k 0k 21 130m 7m 3m 3m 320k 224k 21m 8m 43m 193m 12 1 0 20m 34 31 1330 0 0 +// 2022-05-14 14:52:26 54.4g 21.9g 65 0k 0 0 2 22 4 1 0 0 99 0 0 8.0g 16.0g 118m 118m 2m 5.1g 109m 72m 38m 0k 0k 44m 7m 0 0 4 11k 0k 21 130m 7m 3m 3m 320k 224k 21m 8m 43m 193m 12 1 0 20m 34 31 1330 0 0 +// 2022-05-14 14:52:16 54.4g 21.8g 65 0k 0 0 2 22 3 0 0 0 99 0 0 8.0g 16.0g 118m 118m 2m 5.1g 109m 72m 38m 0k 0k 44m 7m 0 0 4 11k 0k 21 130m 7m 3m 3m 320k 224k 21m 8m 43m 193m 12 1 0 20m 34 31 1330 0 0 +// 2022-05-14 14:52:06 54.4g 21.8g 65 0k 0 0 2 22 1 0 0 0 99 0 0 8.0g 16.0g 118m 118m 2m 5.1g 109m 72m 38m 0k 0k 44m 7m 0 0 4 11k 0k 21 130m 7m 3m 3m 320k 224k 21m 8m 43m 193m 12 1 0 20m 34 31 1330 0 0 +// 2022-05-14 14:51:56 54.4g 21.7g 64 0k 0 0 2 22 2 0 0 0 99 0 0 8.0g 16.0g 118m 118m 2m 5.1g 109m 72m 38m 0k 0k 44m 7m 0 0 4 11k 0k 21 130m 7m 3m 3m 320k 224k 21m 8m 43m 193m 12 1 0 20m 34 31 1330 0 0 +// 2022-05-14 14:51:46 54.4g 21.5g 64 0k 0 0 2 22 1 0 0 0 99 0 0 8.0g 16.0g 118m 118m 2m 5.1g 109m 72m 38m 0k 0k 44m 7m 0 0 4 11k 0k 21 130m 7m 3m 3m 320k 224k 21m 8m 43m 193m 12 1 0 20m 34 31 1330 0 0 +// 2022-05-14 14:51:36 54.4g 21.5g 64 0k 0 0 2 22 1 0 1 0 99 0 0 8.0g 16.0g 118m 118m 2m 5.1g 109m 72m 38m 0k 0k 44m 7m 0 0 4 11k 0k 21 130m 7m 3m 3m 320k 224k 21m 8m 43m 193m 12 1 0 20m 34 31 1330 0 0 + + // Header regex matcher in text mode. These should catch on all platforms, therefore they only check JVM columns, + // and only those columns that are unconditionally available (or should be) + public static final String jvm_header_line0_textmode = ".*---jvm---.*"; + public static final String jvm_header_line1_textmode = ".*-heap-.*-meta-.*-jthr-.*-cldg-.*-cls-.*"; + public static final String jvm_header_line2_textmode = ".*comm.*used.*comm.*used.*gctr.*code.*num.*nd.*cr.*num.*ld.*uld.*"; + + // This is supposed to match a header in csv mode. Here I am rather lenient, because analysing regex mismatches is a + // pain. We later do more strict sanity checks where we check most of the fields anyway. + public static final String jvm_header_line_csvmode = ".*jvm-heap-comm,jvm-heap-used,jvm-meta-comm,jvm-meta-used.*"; + + public static final String timestamp_regex = "\\d{4}+-\\d{2}+-\\d{2}.*\\d{2}:\\d{2}:\\d{2}"; + + // sample line: a timestamp, followed by some numbers + public static final String sample_line_regex_minimal_textmode = timestamp_regex + ".*\\d+.*\\d+.*\\d+.*\\d+.*"; + public static final String sample_line_regex_minimal_csvmode = "\"" + timestamp_regex + "\".*\"\\d+\".*"; + + private static void printLinesWithLineNo(String[] lines) { + int lineno = 0; + for (String s : lines) { + System.err.println(lineno + ": " + s); + lineno++; + } + } + + static final boolean alwaysPrint = true; + + private static boolean findMatchesInStrings(String[] lines, String[] regexes) { + boolean success = false; + Pattern[] pat = new Pattern[regexes.length]; + for (int i = 0; i < regexes.length; i ++) { + pat[i] = Pattern.compile(regexes[i]); + } + int numMatches = 0; + int lineNo = 0; + while (lineNo < lines.length && numMatches < pat.length) { + if (pat[numMatches].matcher(lines[lineNo]).matches()) { + numMatches ++; + } + lineNo ++; + } + success = (numMatches == pat.length); + if (!success) { + System.err.println("Matched " + numMatches + "/" + pat.length + " pattern. First unmatched: " + pat[numMatches]); + } else { + System.err.println("Matched " + numMatches + "/" + pat.length + " pattern. OK!"); + } + if (!success || alwaysPrint) { + System.err.println("Lines: "); + printLinesWithLineNo(lines); + System.err.println("Pattern: "); + printLinesWithLineNo(regexes); + } + return success; + } + + static final String[] expected_output_textmode = new String[] { + jvm_header_line0_textmode, jvm_header_line1_textmode, jvm_header_line2_textmode, sample_line_regex_minimal_textmode + }; + + static final String[] expected_output_csvmode = new String[] { + jvm_header_line_csvmode, sample_line_regex_minimal_csvmode + }; + + static void fileShouldMatch(File f, String[] expected) throws IOException { + Path path = Paths.get(f.getAbsolutePath()); + String[] lines = Files.readAllLines(path).toArray(new String[0]); + if (!findMatchesInStrings(lines, expected)) { + throw new RuntimeException("Expected output not found (see error output)"); + } + } + + public static void fileMatchesVitalsTextMode(File f) throws IOException { + fileShouldMatch(f, expected_output_textmode); + } + + public static void fileMatchesVitalsCSVMode(File f) throws IOException { + fileShouldMatch(f, expected_output_csvmode); + } + + public static void outputMatchesVitalsTextMode(OutputAnalyzer output) { + String[] lines = output.asLines().toArray(new String[0]); + if (!findMatchesInStrings(lines, expected_output_textmode)) { + throw new RuntimeException("Expected output not found (see error output)"); + } + } + + public static void outputMatchesVitalsCSVMode(OutputAnalyzer output) { + String[] lines = output.asLines().toArray(new String[0]); + if (!findMatchesInStrings(lines, expected_output_csvmode)) { + throw new RuntimeException("Expected output not found (see error output)"); + } + } + + // Some more extensive sanity checks are possible in CSV mode + public static CSVParser.CSV parseCSV(OutputAnalyzer output) throws CSVParser.CSVParseException { + String[] lines = output.asLines().toArray(new String[0]); + + // Search for the beginning of the CSV output + int firstline = -1; + int lastline = -1; + Pattern headerLinePattern = Pattern.compile(jvm_header_line_csvmode); + Pattern csvDataLinePattern = Pattern.compile(sample_line_regex_minimal_csvmode); + for (int lineno = 0; lineno < lines.length && firstline == -1 && lastline == -1; lineno ++) { + String line = lines[lineno]; + if (firstline == -1) { + if (headerLinePattern.matcher(line).matches()) { + firstline = lineno; + } + } else { + if (headerLinePattern.matcher(line).matches()) { + throw new CSVParser.CSVParseException("Found header twice", lineno); + } + if (!csvDataLinePattern.matcher(line).matches()) { + lastline = lineno - 1; + break; + } + } + } + if (lastline == -1) { + lastline = lines.length - 1; + } + + if (firstline == -1) { + throw new CSVParser.CSVParseException("Could not find CSV header line"); + } + + String [] csvlines = Arrays.copyOfRange(lines, firstline, lastline + 1); + + CSVParser.CSV csv; + csv = CSVParser.parseCSV(csvlines); + + return csv; + } + + /** + * Does some more extensive tests on a csv raw output. Requires output to be done with scale=1 or raw mode + * @param csv + */ + static public void simpleCSVSanityChecks(CSVParser.CSV csv) throws CSVParser.CSVParseException { + + // The following columns are allowed to be empty (column shown but values missing), e.g. + // for delta columns + // Note: data that are always missing (e.g. because of linux kernel version) should have + // their columns hidden instead, see vitals.cpp) + String colsThatCanBeEmpty = + "|jvm-jthr-cr" // delta column + + "|syst-si|syst-so" // deltas + + "|jvm-cls-ld" // delta + + "|jvm-cls-uld" // delta + ; + + String colsThatCanBeEmpty_WINDOWS = ""; + + String colsThatCanBeEmpty_OSX = ""; + + String colsThatCanBeEmpty_LINUX = + "|syst-avail" // Older kernels < 3.14 miss this value + + "|syst-cpu.*" // CPU values may be omitted in containers; also they are all deltas + + "|syst-cgr.*" // Cgroup values may be omitted in root cgroup + + "|proc-chea-usd|proc-chea-free" // cannot be shown if RSS is > 4g and glibc is too old + + "|proc-io-rd|proc-io-wr" // deltas + + "|proc-cpu-us|proc-cpu-sy" // deltas + ; + + String regexCanBeEmpty = "(x" + colsThatCanBeEmpty; + if (Platform.isLinux()) { + regexCanBeEmpty += colsThatCanBeEmpty_LINUX; + } else if (Platform.isWindows()) { + regexCanBeEmpty += colsThatCanBeEmpty_WINDOWS; + } else if (Platform.isOSX()) { + regexCanBeEmpty += colsThatCanBeEmpty_OSX; + } + regexCanBeEmpty += ")"; + + System.out.println("Columns allowed to be empty: " + regexCanBeEmpty); + Pattern canBeEmptyPattern = Pattern.compile(regexCanBeEmpty); + + for (int lineno = 0; lineno < csv.lines.length; lineno ++) { + CSVParser.CSVDataLine line = csv.lines[lineno]; + // Iterate through all columns and do some basic checks. + // In raw mode, all but the first column are longs. The first column is a time stamp. + for (int i = 1; i < csv.header.size(); i ++) { + String col = csv.header.at(i); + if (line.isEmpty(i)) { + // aka empty + if (!canBeEmptyPattern.matcher(col).matches()) { + throw new CSVParser.CSVParseException("Column " + col + " must not have empty value.", lineno + 1); + } + } else { + long l = 0; + try { + l = line.numberAt(i); + } catch (NumberFormatException e) { + throw new CSVParser.CSVParseException("Column " + col + ": cannot parse value as long (" + l + ")", lineno + 1); + } + long highestReasonableRawValue = 0x00800000_00000000l; + if (l < 0 || l > highestReasonableRawValue) { + throw new CSVParser.CSVParseException("Column " + col + ": Suspiciously high or low value:" + l, lineno + 1); + } + } + } + } + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/VitalsUtils.java b/test/hotspot/jtreg/runtime/Vitals/VitalsUtils.java new file mode 100644 index 00000000000..620a3e41620 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/VitalsUtils.java @@ -0,0 +1,148 @@ +import jdk.test.lib.process.OutputAnalyzer; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class VitalsUtils { + + static String outputDir; + static File outputDirAsFile; + + static { + outputDir = System.getProperty("user.dir", "."); + outputDirAsFile = new File(outputDir); + } + + static File createSubTestDir(String name, boolean create) { + File subdir = new File(outputDir, name); + if (subdir.exists()) { + throw new RuntimeException("test dir already exists: " + subdir); + } + if (create) { + subdir.mkdirs(); + if (!subdir.exists()) { + throw new RuntimeException("Cannot create test dir at " + subdir); + } + } + return subdir; + } + + static void fileShouldExist(File f) { + if (!f.exists()) { + throw new RuntimeException("expected but does not exist: " + f); + } + } + + static void fileShouldNotExist(File f) { + if (f.exists()) { + throw new RuntimeException("expected not to exist, but exists: " + f); + } + } + + static String[] stderrAsLines(OutputAnalyzer output) { + return output.getStderr().split("\\R"); + } + static String[] stdoutAsLines(OutputAnalyzer output) { + return output.getStdout().split("\\R"); + } + + /** + * Look in lines for a number of subsequent matches. Start looking at startAtLine. Return line number of last + * match. Throws RuntimeException if not all regexes where matching. + * @param lines + * @param startAtLine + * @param regexes + * @return + */ + static int matchPatterns(String[] lines, int startAtLine, String[] regexes) { + int nextToMatch = 0; + int nLine = startAtLine; + while (nLine < lines.length) { + if (lines[nLine].matches(regexes[nextToMatch])) { + System.out.println("Matched \"" + regexes[nextToMatch] +"\" at line " + nLine + "(\"" + lines[nLine] + "\")"); + nextToMatch++; + if (nextToMatch == regexes.length) { + break; + } + } + nLine ++; + } + if (nextToMatch < regexes.length) { + throw new RuntimeException("Not all matches found. First missing pattern " + nextToMatch + ":" + regexes[nextToMatch] + + "\nOutput: " + String.join("\n", lines)); + } + return nLine; + } + + static int outputStderrMatchesPatterns(OutputAnalyzer output, String[] regexes) { + return matchPatterns(stderrAsLines(output), 0, regexes); + } + + static int outputStdoutMatchesPatterns(OutputAnalyzer output, String[] regexes) { + return matchPatterns(stdoutAsLines(output), 0, regexes); + } + + static String [] fileAsLines(File f) throws IOException { + Path path = Paths.get(f.getAbsolutePath()); + return Files.readAllLines(path).toArray(new String[0]); + } + + static void assertFileExists(File f) { + if (!f.exists()) { + throw new RuntimeException("File " + f.getAbsolutePath() + " does not exist"); + } else { + System.out.println("File " + f.getAbsolutePath() + " exists - ok!"); + } + } + + static void assertFileExists(String filename) { + File f = new File(filename); + assertFileExists(f); + } + + static void assertFileisEmpty(File f) { + if (f.length() > 0) { + throw new RuntimeException("File " + f.getAbsolutePath() + " expected to be empty, but is not empty."); + } else { + System.out.println("File " + f.getAbsolutePath() + " has zero size - ok!"); + } + } + + static void assertFileisEmpty(String filename) { + File f = new File(filename); + assertFileisEmpty(f); + } + + static void assertFileContentMatches(File f, String[] regexes) throws IOException { + String[] lines = fileAsLines(f); + matchPatterns(lines, 0, regexes); + System.out.println("File " + f.getAbsolutePath() + " matches " + regexes[0] + ", ... etc - ok!"); + } + + static File findFirstMatchingFileInDirectory(File dir, String regex) { + File[] files = dir.listFiles(); + for (File f : files) { + if (f.getName().matches(regex)) { + System.out.println("Found required file " + f.getAbsolutePath() + " - ok!"); + return f; + } + } + throw new RuntimeException("Could not find file matching \"" + regex + "\" inside " + dir.getAbsolutePath()); + } + + // Extract pid of target process (first line of output shall contain ":\n" + private static Pattern patPidInJcmdOutput = Pattern.compile("^(\\d+):\r"); + static long pidFromJcmdOutput(OutputAnalyzer output) { + String s = output.getOutput(); + Matcher m = patPidInJcmdOutput.matcher(s); + if (m.matches()) { + return Long.parseLong(m.group(1)); + } + throw new RuntimeException("Cannot find pid in jcmd output"); + } +} diff --git a/test/hotspot/jtreg/runtime/Vitals/VitalsValuesSanityCheck.java b/test/hotspot/jtreg/runtime/Vitals/VitalsValuesSanityCheck.java new file mode 100644 index 00000000000..947210fec73 --- /dev/null +++ b/test/hotspot/jtreg/runtime/Vitals/VitalsValuesSanityCheck.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2022, SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test Serial + * @summary Test that Vitals memory sizes are within expectations + * @requires os.family == "linux" + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run testng/othervm -XX:+UseSerialGC -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -Xmx64m -Xms64m -XX:NativeMemoryTracking=summary -XX:VitalsSampleInterval=1 VitalsValuesSanityCheck + */ + +/* + * @test G1 + * @summary Test that Vitals memory sizes are within expectations + * @requires os.family == "linux" + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run testng/othervm -XX:+UseG1GC -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -Xmx64m -Xms64m -XX:NativeMemoryTracking=summary -XX:VitalsSampleInterval=1 VitalsValuesSanityCheck + */ + +/* + * @test G1_no_nmt + * @summary Test that Vitals memory sizes are within expectations + * @requires os.family == "linux" + * @library /test/lib + * @modules java.base/jdk.internal.misc java.compiler java.management jdk.internal.jvmstat/sun.jvmstat.monitor + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run testng/othervm -XX:+UseG1GC -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -Xmx64m -Xms64m -XX:NativeMemoryTracking=off -XX:VitalsSampleInterval=1 VitalsValuesSanityCheck + */ + +import jdk.test.lib.Platform; +import jdk.test.lib.dcmd.CommandExecutor; +import jdk.test.lib.dcmd.JMXExecutor; +import jdk.test.lib.process.OutputAnalyzer; +import org.testng.annotations.Test; +import jdk.test.whitebox.WhiteBox; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class VitalsValuesSanityCheck { + + static WhiteBox wb = WhiteBox.getWhiteBox(); + + final long K = 1024; + final long M = 1024 * K; + final long G = 1024 * M; + + // total size of what we will allocate from the cheap in the main function + final long cheapAllocationSize = 32 * M; + // individual block size, small enough to be guaranteed touched and counting toward rss + final long cheapAllocationBlockSize = 1024; + final long numCheapAllocations = cheapAllocationSize / cheapAllocationBlockSize; + + + private static void assertColumnExists(CSVParser.CSV csv, String colname) { + if (!csv.header.hasColumn(colname)) { + throw new RuntimeException("Column " + colname + " not found"); + } + } + + /** + * scan the CSV for the highest value in a column. + * + * @param csv + * @param colname + * @return highest value, or -1 on error (either the column was not found, or no cell in the column contain a valid value) + */ + private static long findHighestValueForColumn(CSVParser.CSV csv, String colname) { + int index = csv.header.findColumn(colname); + if (index == -1) { + throw new RuntimeException("cannot find column " + colname); + } + long max = -1; + int line_containing_max = -1; + int i = 0; + for (i = 0; i < csv.lines.length; i++) { + CSVParser.CSVDataLine line = csv.lines[i]; + if (!line.isEmpty(index)) { + long l2 = line.numberAt(index); + if (l2 > max) { + max = l2; + line_containing_max = i; + } + } + } + if (max == -1) { + System.out.println("Found no valid value " + colname); + } else { + System.out.println("Found highest value for " + colname + " in line " + line_containing_max + ": " + max); + } + return max; + } + + /** + * Check that a certain value is between [min and max). + * @param colname + * @param min + * @param max + */ + private static void checkValueIsBetween(long value, String colname, long min, long max) { + if (value < min) { + throw new RuntimeException(colname + " seems too low (expected at least " + min + ")"); + } else if (value >= max) { + throw new RuntimeException(colname + " seems too high (expected at most " + max + ")"); + } + } + + /** + * Check that a certain value exists (column exists), look for the highest value found, + * and check that it is between [min and max). + * For convenience, we also return that highest value. + * + * @param colname + * @param min + * @param max + * @return the highest value + */ + private static long checkValueIsBetween(CSVParser.CSV csv, String colname, long min, long max) { + long l = findHighestValueForColumn(csv, colname); + checkValueIsBetween(l, colname, min, max); + return l; + } + + public void run(CommandExecutor executor) { + + try { + + final long veryLowButNot0 = K; + // very high but still far away from 0x80000000_00000000 + final long veryVeryHigh = 256 * G; + + // We malloced at least xxx M, and the VM also uses some. + long expected_minimal_cheap_usage_jvm = 2 * M; + long expected_minimal_cheap_usage = cheapAllocationSize + expected_minimal_cheap_usage_jvm; + + // We read the vitals in csv form and get the parsed output. + // Note to self: don't use raw! It does display delta values as accumulating absolutes. + // The heap (64m) should show up as full committed. It is also touched, so it should contribute to RSS unless we swap + // The VM will have allocated 32M C-heap as well, in 1KB chunks. These should show up in NMT, in the cheap columns + // as well as in rss. + OutputAnalyzer output = executor.execute("VM.vitals csv scale=1"); + VitalsTestHelper.outputMatchesVitalsCSVMode(output); + output.shouldNotContain("Now"); // off by default + CSVParser.CSV csv = VitalsTestHelper.parseCSV(output); + + VitalsTestHelper.simpleCSVSanityChecks(csv); + + // Java heap (we specify 64M, but depending on GC this can wobble a bit) + long highest_expected_jvm_heap_comm = M * 68; + long lowest_expected_jvm_heap_comm = M * 60; + long jvm_heap_comm = checkValueIsBetween(csv, "jvm-heap-comm", lowest_expected_jvm_heap_comm, highest_expected_jvm_heap_comm); + // check heap used + long jvm_heap_used = checkValueIsBetween(csv, "jvm-heap-used", veryLowButNot0, jvm_heap_comm); + + // Class+nonclass metaspace + long highest_expected_metaspace_comm = 1 * G; // for this little programm, nothing more + long jvm_meta_comm = checkValueIsBetween(csv, "jvm-meta-comm", veryLowButNot0, highest_expected_metaspace_comm); + long jvm_meta_used = checkValueIsBetween(csv, "jvm-meta-used", veryLowButNot0, jvm_meta_comm); + + // Class space + if (Platform.is64bit()) { + long highest_expected_classspace_comm = highest_expected_metaspace_comm / 2; + long jvm_meta_csc = checkValueIsBetween(csv, "jvm-meta-csc", veryLowButNot0, highest_expected_classspace_comm); + checkValueIsBetween(csv, "jvm-meta-csu", veryLowButNot0, jvm_meta_csc); + } + + // Code cache + long jvm_code = checkValueIsBetween(csv, "jvm-code", veryLowButNot0, 2 * G); + + // How much we know for now the JVM mapped for sure + long jvm_mapped_this_much_at_least = jvm_heap_comm + jvm_code + jvm_meta_comm; + + // How much we think the jvm has touched (RSS) at least. + long jvm_touched_this_much_at_least = (jvm_heap_used + jvm_meta_used) / 2; + + long jvm_highest_expected_cheap_usage = expected_minimal_cheap_usage * 10; + + // NMT (may be off) + long jvm_nmt_mlc = -1; + if (csv.header.hasColumn("jvm-nmt-mlc")) { + // NMT committed maps: + long highest_expected_nmt_mmap = jvm_mapped_this_much_at_least + 2 * G; // very generous + long lowest_expected_nmt_mmap = jvm_mapped_this_much_at_least; + checkValueIsBetween(csv, "jvm-nmt-map", lowest_expected_nmt_mmap, highest_expected_nmt_mmap); + // NMT malloc: + jvm_nmt_mlc = checkValueIsBetween(csv, "jvm-nmt-mlc", expected_minimal_cheap_usage, jvm_highest_expected_cheap_usage); + checkValueIsBetween(csv, "jvm-nmt-ovh", veryLowButNot0, jvm_nmt_mlc); + checkValueIsBetween(csv, "jvm-nmt-gc", veryLowButNot0, jvm_nmt_mlc); + checkValueIsBetween(csv, "jvm-nmt-oth", veryLowButNot0, jvm_nmt_mlc); + + } + + // Number of jvm threads: we expect at least 4-5 in total, with at least one non-deamon thread (the one we run in right now) + long min_expected_java_threads = 5; // probably more + long max_expected_java_threads = 1000; // probably way less, but lets go with this. + long jvm_thr_num = checkValueIsBetween(csv, "jvm-jthr-num", min_expected_java_threads, max_expected_java_threads); + // we expect at least 1 non-deamon thread (me) + checkValueIsBetween(csv, "jvm-jthr-nd", 1, jvm_thr_num); + // How many threads were created in the last second (delta col)? + checkValueIsBetween(csv, "jvm-jthr-cr", 0, 10000); + + // Classloaders: + long min_expected_classloaders = 1; // probably more + long max_expected_classloaders = 100000; // probably way less, not sure how many lambdas are active, dep. on jvm version they show up as cldg entries + long jvm_clg_num = checkValueIsBetween(csv, "jvm-cldg-num", min_expected_classloaders, max_expected_classloaders); + checkValueIsBetween(csv, "jvm-cldg-anon", 0, jvm_clg_num); + + // Classes: + long min_expected_classes_base = 300; // probably more + long max_expected_classes_base = 60000; // probably way less + long jvm_cls_num = checkValueIsBetween(csv, "jvm-cls-num", min_expected_classes_base, max_expected_classes_base); + // cls-ld and cls-uld are delta values! + // Unless loading is insanely slow, I'd expect at least two-digit loads per second (vitals interval) + checkValueIsBetween(csv, "jvm-cls-ld", 10, max_expected_classes_base); + // I don't think we have unloaded yet + checkValueIsBetween(csv, "jvm-cls-uld", 0, max_expected_classes_base); + + // Linux specific platform columns + if (Platform.isLinux()) { + + // Check --- system --- cols on Linux + + long highestExpectedMemoryValue = 512 * G; // I wish... + if (csv.header.hasColumn("syst-avail")) { + checkValueIsBetween(csv, "syst-avail", M, highestExpectedMemoryValue); + } + + long syst_comm = checkValueIsBetween(csv, "syst-comm", M, highestExpectedMemoryValue); + checkValueIsBetween(csv, "syst-crt", 1, 10000); // its a percentage; anything above ~150 is very unlikely unless we aggressivly overcommit + checkValueIsBetween(csv, "syst-swap", 0, highestExpectedMemoryValue); + // si, so: number of pages, delta + checkValueIsBetween(csv, "syst-si", 0, 100000); + checkValueIsBetween(csv, "syst-so", 0, 100000); + + // Number of processes and kernel threads. In containers shows the local processes only. But we should have + // at least one, us, and multiple kernel threads, since we are multithreaded. + long min_expected_processes = 1; + long max_expected_processes = 1000000000; // Anything above a billion would surprise me + checkValueIsBetween(csv, "syst-p", min_expected_processes, max_expected_processes); + + long min_expected_kernel_threads = min_expected_java_threads; + long max_expected_kernel_threads = 1000000000; // same here + long syst_t = checkValueIsBetween(csv, "syst-t", min_expected_kernel_threads, max_expected_kernel_threads); + + // threads running, blocked on disk IO (cannot be larger than number of kernel threads) + checkValueIsBetween(csv, "syst-tr", 0, syst_t); + checkValueIsBetween(csv, "syst-tb", 0, syst_t); + + // Cgroup + // We may not always show this. But if we do, at least the usage numbers should be checked + if (csv.header.hasColumn("syst-cgro-usg")) { + checkValueIsBetween(csv, "syst-cgro-usg", veryLowButNot0, highestExpectedMemoryValue); + } +// Completely disable this test because hunting down erros here is exhausting. Many kernels don't seem to show +// kernel memory values. So this may not show off at all, or show a column without value. +// if (csv.header.hasColumn("syst-cgro-kusg")) { +// // kusg can get surprisingly high, e.g. if ran on a host hosting guest VMs (seen that with virtual box) +// // ... and it can be 0 too for some reason, seen on our loaner ppcle machines at Adopt +// checkValueIsBetween(csv, "syst-cgro-kusg", 0, highestExpectedMemoryValue); +// } + + // Check --- process --- cols on Linux + long proc_virt = checkValueIsBetween(csv, "proc-virt", jvm_mapped_this_much_at_least, 100 * G); // virt size can get crazy + long proc_rss_all = checkValueIsBetween(csv, "proc-rss-all", jvm_touched_this_much_at_least, proc_virt); + if (csv.header.hasColumn("proc-rss-anon")) { + checkValueIsBetween(csv, "proc-rss-anon", jvm_touched_this_much_at_least, proc_rss_all); // most JVM mappings are anon + checkValueIsBetween(csv, "proc-rss-file", 0, proc_rss_all); + checkValueIsBetween(csv, "proc-rss-shm", 0, proc_rss_all); + } + + checkValueIsBetween(csv, "proc-swdo", 0, proc_virt); + + // -cheap-- + if (!Platform.isMusl() && proc_rss_all < 4 * G) { + // we expect to see what NMT sees, plus a bit, since glibc has overhead. If NMT is off, we use + // our initial ballpark number. + long min_c_heap_usage_glibc = (jvm_nmt_mlc == -1 ? jvm_nmt_mlc : expected_minimal_cheap_usage) + K; + long max_c_heap_usage_glibc = min_c_heap_usage_glibc + (2 * G); // glibc has crazy overhead + // but cannot be larger than virt ever was + if (max_c_heap_usage_glibc > proc_virt) { + max_c_heap_usage_glibc = proc_virt - M; // bit less since a lot of stuff is not malloc + } + checkValueIsBetween(csv, "proc-chea-usd", min_c_heap_usage_glibc, max_c_heap_usage_glibc); + checkValueIsBetween(csv, "proc-chea-free", 0, max_c_heap_usage_glibc); + } + + // Counter-check NMT mlc vs rss + // "mlc" can be higher than RSS in release builds, since large malloc blocks may not be fully touched. + // However, in debug builds os::malloc inits all malloced blocks, so - apart from a few raw mallocs here and there - + // we should not see mlc > rss + if (jvm_nmt_mlc != -1) { + if (Platform.isDebugBuild()) { + if (proc_rss_all < jvm_nmt_mlc) { + throw new RuntimeException("NMT mlc higher than RSS?"); + } + } + } + + checkValueIsBetween(csv, "proc-cpu-us", 0, 100000); + checkValueIsBetween(csv, "proc-cpu-sy", 0, 100000); + + // Number of open file descriptors + // (at least one, since we write to stdout. Probably not many more. + checkValueIsBetween(csv, "proc-io-of", 1, 1000); + + // IO read, written (note: deltas, so its "read, written, in one second"). + checkValueIsBetween(csv, "proc-io-rd", 0, 1 * G); + checkValueIsBetween(csv, "proc-io-wr", 0, 10 * G); + + // Number of threads in this process (java + native) + checkValueIsBetween(csv, "proc-thr", jvm_thr_num, jvm_thr_num + 100); + + } // end: linux specific sanity tests + + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e.getMessage()); + } + + } + + @Test + public void jmx() { + for (int i = 0; i < numCheapAllocations; i++) { + long p = wb.NMTMalloc(cheapAllocationBlockSize); + } + try { + // wait some time. We sample with 1sec sample frequency, that should give us more than one sample + // and therefore some of them should show delta values + Thread.sleep(4000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + run(new JMXExecutor()); + } + +} diff --git a/test/hotspot/jtreg/runtime/malloctrace/MallocHooksTest.java b/test/hotspot/jtreg/runtime/malloctrace/MallocHooksTest.java new file mode 100644 index 00000000000..47bd0573c9f --- /dev/null +++ b/test/hotspot/jtreg/runtime/malloctrace/MallocHooksTest.java @@ -0,0 +1,854 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +/** + * @test + * @summary Test functionality of the malloc hooks library. + * @library /test/lib + * + * @run main/othervm/native/timeout=600 MallocHooksTest + */ + +import java.lang.management.ManagementFactory; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; + +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import jdk.test.lib.JDKToolFinder; +import jdk.test.lib.Platform; +import jdk.test.lib.Utils; +import jdk.test.lib.process.ProcessTools; +import jdk.test.lib.process.OutputAnalyzer; + +import static jdk.test.lib.Asserts.*; + +public class MallocHooksTest { + static native void doRandomMemOps(int nrOfOps, int maxLiveAllocations, int seed, + boolean trackLive, long[] sizes, long[] counts); + static native void doRandomAllocsWithFrees(int nrOfOps, int size, int maxStack, int seed); + + private static final String LD_PRELOAD = Platform.isOSX() ? "DYLD_INSERT_LIBRARIES" : "LD_PRELOAD"; + private static final String LIB_SUFFIX = Platform.isOSX() ? ".dylib" : ".so"; + private static final String LIB_DIR = System.getProperty("sun.boot.library.path") + "/"; + private static final String NATIVE_DIR = System.getProperty("test.nativepath") + "/"; + private static final String LIB_MALLOC_HOOKS = LIB_DIR + "libmallochooks" + LIB_SUFFIX; + private static final String LIB_FAKE_MALLOC_HOOKS = NATIVE_DIR + "libfakemallochooks" + LIB_SUFFIX; + + private static void dumpHsErrorFiles() throws Exception { + for (File f: new File(".").listFiles()) { + if (!f.isDirectory() && f.getName().startsWith("hs_err")) { + System.out.println("Found " + f.getName() + ":"); + String output = new String(Files.readAllBytes(f.toPath())); + System.out.println(output); + System.out.println("------- End of " + f.getName()); + // Print the start again, since we might overflow the buffer with + // the whole file. + int startLength = 32768; + + if (output.length() > startLength) { + System.out.println("------- Repeating start of " + f.getName()); + System.out.println(output.substring(0, startLength)); + System.out.println("------- End of start of " + f.getName()); + } + } + } + } + + public static void main(String args[]) throws Exception { + if (!Platform.isLinux() && !Platform.isOSX()) { + return; + } + + while (args.length == 0) { + try { + testNoRecursiveCallsForFallbacks(); + testEnvSanitizing(); + testTracking(false); + testTracking(true); + testDumpPercentage(true); + testDumpPercentage(false); + testUniqueStacks(); + testPartialTracking(false, 2, 0.2); + testPartialTracking(true, 2, 0.2); + testPartialTracking(false, 10, 0.4); + testPartialTracking(true, 10, 0.4); + testFlags(); + testJcmdOptions(); + testEnablingStress(); + return; + } catch (InterruptedException e) { + System.out.println("Retrying because of stale socket"); + // Retry + } finally { + dumpHsErrorFiles(); + } + } + + switch (args[0]) { + case "manyStacks": + System.loadLibrary("testmallochooks"); + doManyStacks(args); + System.out.println("Done"); + + while (true) { + Thread.sleep(1000); + } + + case "checkEnv": + if (args[1].equals(getLdPrelodEnv())) { + return; + } + + throw new Exception("Expected " + LD_PRELOAD + "=\"" + args[1] + "\", but got " + + LD_PRELOAD + "=\"" + getLdPrelodEnv() + "\""); + + case "stress": + System.loadLibrary("testmallochooks"); + doStress(args); + + while (true) { + Thread.sleep(1000); + } + + case "sleep": + System.out.print("*"); + System.out.flush(); + Thread.sleep(1000 * Long.parseLong(args[1])); + return; + + case "enablingStress": + int nrOfEnables = Integer.parseInt(args[2]); + new Thread(() -> { + try { + doEnablingStress(nrOfEnables); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + }, "doEnablingStress").start(); + // fall through intentional to start the work. + + case "wait": + System.loadLibrary("testmallochooks"); + System.out.print("*"); + System.out.flush(); + int nrOfThreads = Integer.parseInt(args[1]); + + for (int i = 0; i < nrOfThreads; ++i) { + new Thread(() -> { + while (true) { + doRandomMemOps(1000000, 32768 / nrOfThreads, 1348763421, + false, new long[8], new long[8]); + } + }, "doRandomMemOps" + i).start(); + } + + return; + + default: + throw new Exception("Unknown command " + args[0]); + } + } + + private static void doManyStacks(String[] args) { + int nrOfOps = Integer.parseInt(args[1]); + int size = Integer.parseInt(args[2]); + int maxStack = Integer.parseInt(args[3]); + int seed = Integer.parseInt(args[4]); + + doRandomAllocsWithFrees(nrOfOps, size, maxStack, seed); + } + + private static void doStress(String[] args) { + int nrOfOps = Integer.parseInt(args[1]); + int maxLiveAllocations = Integer.parseInt(args[2]); + int seed = Integer.parseInt(args[3]); + boolean trackLive = Boolean.parseBoolean(args[4]); + long[] sizes = new long[8]; + long[] counts = new long[8]; + + doRandomMemOps(nrOfOps, maxLiveAllocations, seed, trackLive, sizes, counts); + + for (int i = 0; i < sizes.length; ++i) { + System.out.println(sizes[i] + " " + counts[i]); + } + } + + private static void testNoRecursiveCallsForFallbacks() throws Exception { + ProcessBuilder pb = ProcessTools.createNativeTestProcessBuilder("testmallochooks"); + pb.environment().put(LD_PRELOAD, LIB_MALLOC_HOOKS); + new OutputAnalyzer(pb.start()).shouldHaveExitValue(0); + } + + private static void testEnvSanitizing() throws Exception { + ProcessBuilder pb = checkEnvProc(""); + pb.environment().put(LD_PRELOAD, LIB_MALLOC_HOOKS); + new OutputAnalyzer(pb.start()).shouldHaveExitValue(0); + pb = checkEnvProc(LIB_FAKE_MALLOC_HOOKS); + pb.environment().put(LD_PRELOAD, LIB_MALLOC_HOOKS + ":" + LIB_FAKE_MALLOC_HOOKS); + new OutputAnalyzer(pb.start()).shouldHaveExitValue(0); + if (!Platform.isMusl()) { + pb = checkEnvProc(LIB_FAKE_MALLOC_HOOKS); + pb.environment().put(LD_PRELOAD, LIB_FAKE_MALLOC_HOOKS + ":" + LIB_MALLOC_HOOKS); + new OutputAnalyzer(pb.start()).shouldHaveExitValue(0); + } + pb = checkEnvProcWithHooks(""); + new OutputAnalyzer(pb.start()).shouldHaveExitValue(0); + pb = checkEnvProcWithHooks(""); + pb.environment().put(LD_PRELOAD, LIB_MALLOC_HOOKS); + new OutputAnalyzer(pb.start()).shouldHaveExitValue(0); + pb = checkEnvProcWithHooks(LIB_FAKE_MALLOC_HOOKS); + pb.environment().put(LD_PRELOAD, LIB_MALLOC_HOOKS + ":" + LIB_FAKE_MALLOC_HOOKS); + new OutputAnalyzer(pb.start()).shouldHaveExitValue(0); + if (!Platform.isMusl()) { + pb = checkEnvProcWithHooks(LIB_FAKE_MALLOC_HOOKS); + pb.environment().put(LD_PRELOAD, LIB_FAKE_MALLOC_HOOKS + ":" + LIB_MALLOC_HOOKS); + new OutputAnalyzer(pb.start()).shouldHaveExitValue(0); + } + } + + private static long getPid(Process p) { + return p.pid(); + } + + private static OutputAnalyzer callJcmd(Process p, String... args) throws Exception { + ProcessBuilder pb = new ProcessBuilder(); + String[] realArgs = new String[args.length + 2]; + System.arraycopy(args, 0, realArgs, 2, args.length); + realArgs[0] = JDKToolFinder.getJDKTool("jcmd"); + realArgs[1] = Long.toString(getPid(p)); + pb.command(realArgs); + OutputAnalyzer oa = new OutputAnalyzer(pb.start()); + System.out.println("Output of jcmd " + String.join(" ", args)); + System.out.println(oa.getOutput()); + System.out.println("---------------------"); + oa.shouldHaveExitValue(0); + return oa; + } + + private static void checkIsAttachable(Process p) throws Exception { + // There should not already be a socket for this VM. If it exists it doesn't + // comes from the VM itself, but was there + if (new File("/tmp/.java_pid" + getPid(p)).exists()) { + p.destroy(); + Thread.sleep(1000); // Don't retry too fast. + throw new InterruptedException(); + } + } + + private static Process runEnablingStress(int nrOfThreads, int nrOfEnables, String... opts) throws Exception { + String[] args = new String[opts.length + 5]; + System.arraycopy(opts, 0, args, 0, opts.length); + args[opts.length + 0] = "-Djava.library.path=" + System.getProperty("java.library.path"); + args[opts.length + 1] = MallocHooksTest.class.getName(); + args[opts.length + 2] = "enablingStress"; + args[opts.length + 3] = "" + nrOfThreads; + args[opts.length + 4] = "" + nrOfEnables; + Process p = ProcessTools.createLimitedTestJavaProcessBuilder(args).start(); + checkIsAttachable(p); + // Make sure java is running when we return + p.getInputStream().read(); + return p; + } + + private static Process runWait(String... opts) throws Exception { + String[] args = new String[opts.length + 4]; + System.arraycopy(opts, 0, args, 0, opts.length); + args[opts.length + 0] = "-Djava.library.path=" + System.getProperty("java.library.path"); + args[opts.length + 1] = MallocHooksTest.class.getName(); + args[opts.length + 2] = "wait"; + args[opts.length + 3] = "1"; + Process p = ProcessTools.createLimitedTestJavaProcessBuilder(args).start(); + checkIsAttachable(p); + // Make sure java is running when we return + p.getInputStream().read(); + return p; + } + + private static void doEnablingStress(int nrOfEnables) throws Exception { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + ObjectName name = new ObjectName("com.sun.management:type=DiagnosticCommand"); + String[] signature = new String[] {String[].class.getName()}; + + for (int i = 0; i < nrOfEnables; ++i) { + String[] args; + + if ((i & 1) == 0) { + args = new String[] {"-force"}; + } else { + args = new String[] {"-force", "-track-free"}; + } + + server.invoke(name, "malloctraceEnable", new Object[] {args}, signature); + } + + System.exit(0); + } + + private static void testEnablingStress() throws Exception { + Process p = runEnablingStress(4, 20000, "-XX:+UseMallocHooks", "-XX:+MallocTraceAtStartup"); + OutputAnalyzer oa = new OutputAnalyzer(p); + System.out.println(oa.getOutput()); + oa.shouldHaveExitValue(0); + } + + private static void testJcmdOptions() throws Exception { + // Check we cannot enable if already enabled. + Process p = runWait("-XX:+UseMallocHooks", "-XX:+MallocTraceAtStartup"); + OutputAnalyzer oa = callJcmd(p, "MallocTrace.enable"); + oa.shouldContain("Malloc statistic is already enabled"); + oa = callJcmd(p, "MallocTrace.enable", "-force"); + oa.shouldNotContain("Malloc statistic is already enabled"); + oa.shouldContain("Malloc statistic enabled"); + oa.shouldContain("Disabled already running trace first"); + oa.shouldContain("Tracking all allocated memory"); + oa = callJcmd(p, "MallocTrace.disable"); + oa.shouldContain("Malloc statistic disabled"); + oa = callJcmd(p, "MallocTrace.disable"); + oa.shouldNotContain("Malloc statistic disabled"); + oa.shouldContain("Malloc statistic is already disabled"); + oa = callJcmd(p, "MallocTrace.enable", "-detailed-stats"); + oa.shouldContain("Malloc statistic enabled"); + oa.shouldContain("Collecting detailed statistics"); + oa = callJcmd(p, "MallocTrace.dump", "-internal-stats"); + oa.shouldMatch("Sampled [0-9,.]+ stacks, took [0-9,.]+ ns per stack on average"); + oa.shouldMatch("Sampling took [0-9,.]+ seconds in total"); + oa.shouldContain("Tracked allocations"); + oa.shouldContain("Untracked allocations"); + oa.shouldMatch("Untracked frees[ ]*:[ ]*0"); + oa.shouldContain("Statistic for stack maps"); + oa.shouldNotContain("Statistic for alloc maps"); + oa = callJcmd(p, "MallocTrace.enable", "-force", "-use-backtrace", "-only-nth=4"); + oa.shouldContain("Malloc statistic enabled"); // We cannot assume this machine has backtrace available. + oa = callJcmd(p, "MallocTrace.enable", "-force", "-track-free"); + oa.shouldContain("Tracking live memory"); + oa = callJcmd(p, "MallocTrace.dump", "-dump-file=stdout"); + oa.shouldContain("Dumping done in"); + oa.shouldNotContain("Total unique stacks"); + try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()))) { + while (!br.readLine().startsWith("Total unique stacks")) { + } + } + p.destroy(); + p = runWait("-XX:+UseMallocHooks"); + oa = callJcmd(p, "MallocTrace.enable"); + oa = callJcmd(p, "MallocTrace.dump", "-dump-file=stderr"); + oa.shouldContain("Dumping done in"); + oa.shouldNotContain("Total unique stacks"); + try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()))) { + while (!br.readLine().startsWith("Total unique stacks")) { + } + } + p.destroy(); + p = runWait("-XX:+UseMallocHooks"); + oa = callJcmd(p, "MallocTrace.enable"); + oa = callJcmd(p, "MallocTrace.dump", "-dump-file=malloctrace.txt"); + oa.shouldContain("Dumping done in"); + oa.shouldNotContain("Total unique stacks"); + try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream("malloctrace.txt")))) { + while (!br.readLine().startsWith("Total unique stacks")) { + } + } + p.destroy(); + p = runWait("-XX:+UseMallocHooks"); + oa = callJcmd(p, "MallocTrace.enable", "-force", "-stack-depth=2"); + oa = callJcmd(p, "MallocTrace.dump", "-dump-file=stderr"); + oa.shouldContain("Dumping done in"); + try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()))) { + MallocTraceResult result = new MallocTraceResult(br); + int maxDepth = 0; + long lastSize = Long.MAX_VALUE; + + for (int i = 0; i < result.nrOfStacks(); ++i) { + Stack stack = result.getStack(i); + maxDepth = Math.max(maxDepth, stack.getStackDepth()); + assertLTE(stack.getBytes(), lastSize); + lastSize = stack.getBytes(); + } + + assertLTE(maxDepth, 2); + } + p.destroy(); + p = runWait("-XX:+UseMallocHooks"); + oa = callJcmd(p, "MallocTrace.enable", "-force", "-stack-depth=8"); + oa = callJcmd(p, "MallocTrace.dump", "-dump-file=stderr", "-sort-by-count"); + oa.shouldContain("Dumping done in"); + try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()))) { + MallocTraceResult result = new MallocTraceResult(br); + int maxDepth = 0; + long lastCount = Long.MAX_VALUE; + + for (int i = 0; i < result.nrOfStacks(); ++i) { + Stack stack = result.getStack(i); + maxDepth = Math.max(maxDepth, stack.getStackDepth()); + assertLTE(stack.getCount(), lastCount); + lastCount = stack.getCount(); + } + + assertLTE(maxDepth, 8); + } + p.destroy(); + p = runWait("-XX:+UseMallocHooks"); + oa = callJcmd(p, "MallocTrace.enable", "-force", "-stack-depth=8"); + oa = callJcmd(p, "MallocTrace.dump", "-dump-file=stderr", "-filter=Java_MallocHooksTest_doRandomMemOps"); + oa.shouldContain("Dumping done in"); + try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()))) { + MallocTraceResult result = new MallocTraceResult(br); + + for (int i = 0; i < result.nrOfStacks(); ++i) { + Stack stack = result.getStack(i); + boolean found = false; + + for (int j = 0; j < stack.getStackDepth(); ++j) { + if (stack.getFunction(j).contains("Java_MallocHooksTest_doRandomMemOps")) { + found = true; + break; + } + } + + assertTrue(found); + } + } + p.destroy(); + p = runWait("-XX:+UseMallocHooks"); + oa = callJcmd(p, "MallocTrace.enable", "-force", "-stack-depth=8"); + oa = callJcmd(p, "MallocTrace.dump", "-dump-file=stderr", "-filter=should_not_be_found"); + oa.shouldContain("Dumping done in"); + try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()))) { + while (true) { + String line = br.readLine(); + + if (line.startsWith("Printed 0 stacks")) { + break; + } + + if (line.startsWith("Stack ")) { + throw new Exception(line); + } + } + } + p.destroy(); + } + + private static void testPartialTracking(boolean trackLive, int nth, double diff) throws Exception { + ProcessBuilder pb = runStress(1024 * 1024 * 10, 65536, 172369977, trackLive, + "-Djava.library.path=" + System.getProperty("java.library.path"), + "-XX:MallocTraceStackDepth=2", + "-XX:+MallocTraceDetailedStats", + "-XX:MallocTraceOnlyNth=" + nth); + Process p = pb.start(); + checkIsAttachable(p); + MallocTraceExpectedStatistic expected = new MallocTraceExpectedStatistic(p); + OutputAnalyzer oa = callJcmd(p, "MallocTrace.dump", "-max-entries=10", "-sort-by-count", + "-filter=Java_MallocHooksTest_doRandomMemOps", + "-internal-stats"); + p.destroy(); + MallocTraceResult actual = MallocTraceResult.fromString(oa.getOutput()); + System.out.println(expected); + System.out.println(actual); + + double real_nth = Double.parseDouble(oa.getOutput().split("about every ")[1].split(" ")[0]); + assertGTE(real_nth, (1 - diff) * nth); + assertLTE(real_nth, (1 + diff) * nth); + + expected.check(actual, nth, diff); + + if (trackLive) { + oa.shouldContain("Contains the currently allocated memory since enabling"); + } else { + oa.shouldContain("Contains every allocation done since enabling"); + } + oa.shouldContain("libtestmallochooks."); + oa.shouldContain("Total allocated bytes"); + oa.shouldContain("Total printed count"); + } + + private static void testTracking(boolean trackLive) throws Exception { + ProcessBuilder pb = runStress(1024 * 1024 * 10, 65536, 172369973, trackLive, + "-Djava.library.path=" + System.getProperty("java.library.path"), + "-XX:MallocTraceStackDepth=2"); + Process p = pb.start(); + checkIsAttachable(p); + MallocTraceExpectedStatistic expected = new MallocTraceExpectedStatistic(p); + OutputAnalyzer oa = callJcmd(p, "MallocTrace.dump", "-max-entries=10", "-sort-by-count", + "-filter=Java_MallocHooksTest_doRandomMemOps"); + p.destroy(); + MallocTraceResult actual = MallocTraceResult.fromString(oa.getOutput()); + System.out.println(expected); + System.out.println(actual); + + expected.check(actual); + + if (trackLive) { + oa.shouldContain("Contains the currently allocated memory since enabling"); + } else { + oa.shouldContain("Contains every allocation done since enabling"); + } + oa.shouldContain("libtestmallochooks."); + oa.shouldContain("Total allocated bytes"); + oa.shouldContain("Total printed count"); + } + + private static OutputAnalyzer runSleep(int sleep, String... opts) throws Exception { + String[] args = new String[opts.length + 4]; + System.arraycopy(opts, 0, args, 0, opts.length); + args[opts.length + 0] = "-XX:+MallocTraceAtStartup"; + args[opts.length + 1] = MallocHooksTest.class.getName(); + args[opts.length + 2] = "sleep"; + args[opts.length + 3] = Integer.toString(sleep); + Process p = ProcessTools.createLimitedTestJavaProcessBuilder(args).start(); + checkIsAttachable(p); + + return new OutputAnalyzer(p); + } + + private static void testBasicOutput(OutputAnalyzer oa) throws Exception { + oa.shouldHaveExitValue(0); + oa.shouldContain("Total printed bytes:"); + } + + private static void testFlags() throws Exception { + OutputAnalyzer oa = null; + try { + oa = runSleep(1); + oa.shouldHaveExitValue(1); + oa.shouldContain("Could not find preloaded libmallochooks"); + oa.shouldContain(LD_PRELOAD + "="); + oa = runSleep(10, "-XX:+UseMallocHooks", "-XX:MallocTraceDumpDelay=1s", + "-XX:MallocTraceDumpCount=1"); + testBasicOutput(oa); + oa.shouldContain("Contains the currently allocated memory since enabling"); + oa.shouldContain("Stacks were collected via"); + oa = runSleep(10, "-XX:+UseMallocHooks", "-XX:MallocTraceDumpDelay=1s", + "-XX:MallocTraceDumpCount=1", "-XX:-MallocTraceTrackFree", + "-XX:-MallocTraceUseBacktrace"); + testBasicOutput(oa); + oa.shouldContain("Contains every allocation done since enabling"); + oa.shouldContain("Stacks were collected via the fallback mechanism."); + } catch (Exception e) { + System.out.println(oa.getOutput()); + throw e; + } + } + + private static ProcessBuilder runManyStacks(int nrOfOps, int size, int maxStack, int seed, + String... opts) { + String[] args = new String[opts.length + 9]; + System.arraycopy(opts, 0, args, 0, opts.length); + args[opts.length + 0] = "-XX:+UseMallocHooks"; + args[opts.length + 1] = "-XX:+MallocTraceAtStartup"; + args[opts.length + 2] = "-XX:-MallocTraceTrackFree"; + args[opts.length + 3] = MallocHooksTest.class.getName(); + args[opts.length + 4] = "manyStacks"; + args[opts.length + 5] = Integer.toString(nrOfOps); + args[opts.length + 6] = Integer.toString(size); + args[opts.length + 7] = Integer.toString(maxStack); + args[opts.length + 8] = Integer.toString(seed); + return ProcessTools.createLimitedTestJavaProcessBuilder(args); + } + + private static void testDumpPercentage(boolean bySize) throws Exception { + Process p = runManyStacks(1024 * 1024 * 10, 1024 * 16, 5, 172369973, + "-Djava.library.path=" + System.getProperty("java.library.path"), + "-XX:MallocTraceStackDepth=12").start(); + checkIsAttachable(p); + p.getInputStream().read(); + OutputAnalyzer oa = bySize ? callJcmd(p, "MallocTrace.dump", "-percentage=90") : + callJcmd(p, "MallocTrace.dump", "-sort-by-count", "-percentage=90"); + oa.shouldHaveExitValue(0); + p.destroy(); + oa.stdoutShouldMatch("Total printed " + (bySize ? "bytes" : "count") + ": .*[(].*90[.][0-9]+ %[)]"); + } + + private static void testUniqueStacks() throws Exception { + Process p = runManyStacks(1024 * 1024 * 10, 1024 * 16, 12, 172369975, + "-Djava.library.path=" + System.getProperty("java.library.path"), + "-XX:MallocTraceStackDepth=14").start(); + checkIsAttachable(p); + p.getInputStream().read(); + OutputAnalyzer oa = callJcmd(p, "MallocTrace.dump", "-percentage=100"); + oa.shouldHaveExitValue(0); + p.destroy(); + + MallocTraceResult result = MallocTraceResult.fromString(oa.getOutput()); + HashSet seenStacks = new HashSet<>(); + + for (int i = 0; i < result.nrOfStacks(); ++i) { + if (!seenStacks.add(result.getStack(i))) { + throw new Exception("Duplicate stack"); + } + } + } + + private static ProcessBuilder runStress(int nrOfOps, int maxLiveAllocations, int seed, + boolean trackLive, String... opts) { + String[] args = new String[opts.length + 9]; + System.arraycopy(opts, 0, args, 0, opts.length); + args[opts.length + 0] = "-XX:+UseMallocHooks"; + args[opts.length + 1] = "-XX:+MallocTraceAtStartup"; + args[opts.length + 2] = "-XX:" + (trackLive ? "+" : "-") + "MallocTraceTrackFree"; + args[opts.length + 3] = MallocHooksTest.class.getName(); + args[opts.length + 4] = "stress"; + args[opts.length + 5] = Integer.toString(nrOfOps); + args[opts.length + 6] = Integer.toString(maxLiveAllocations); + args[opts.length + 7] = Integer.toString(seed); + args[opts.length + 8] = Boolean.toString(trackLive); + return ProcessTools.createLimitedTestJavaProcessBuilder(args); + } + + private static ProcessBuilder checkEnvProc(String env) { + String[] args = {MallocHooksTest.class.getName(), "checkEnv", env}; + return ProcessTools.createLimitedTestJavaProcessBuilder(args); + } + + private static ProcessBuilder checkEnvProcWithHooks(String env) { + String[] args = {"-XX:+UseMallocHooks", MallocHooksTest.class.getName(),"checkEnv", env}; + return ProcessTools.createLimitedTestJavaProcessBuilder(args); + } + + private static String getLdPrelodEnv() { + String env = System.getenv(LD_PRELOAD); + + return env == null ? "" : env; + } + + private static String absPath(String file) { + return new File(file).getAbsolutePath().toString(); + } +} + +class Stack { + private String[] funcs; + private int index; + private int maxIndex; + private long bytes; + private long count; + + public Stack(BufferedReader reader, int index, int maxIndex, long bytes, long count) throws Exception { + this.index = index; + this.maxIndex = maxIndex; + this.bytes = bytes; + this.count = count; + + ArrayList lines = new ArrayList<>(); + + while (true) { + reader.mark(10); + + if (reader.read() != ' ') { + reader.reset(); + break; + } + + lines.add(reader.readLine()); + } + + funcs = new String[lines.size()]; + + for (int i = 0; i < funcs.length; ++i) { + String line = lines.get(i); + int pos = line.indexOf(']'); + + if (pos > 0) { + funcs[i] = line.substring(pos + 1).trim(); + } else { + funcs[i] = ""; + } + } + } + + public int getStackIndex() { + return index; + } + + public int getMaxStackIndex() { + return maxIndex; + } + + public long getBytes() { + return bytes; + } + + public long getCount() { + return count; + } + + public int getStackDepth() { + return funcs.length; + } + + public String getFunction(int index) { + return funcs[index]; + } + + public String toString() { + StringBuilder result = new StringBuilder(); + + result.append("Stack " + index + " of " + maxIndex + ": " + bytes + " bytes, " + + count + " allocations" + System.lineSeparator()); + + for (String f: funcs) { + result.append(" " + f + System.lineSeparator()); + } + + return result.toString(); + } + + public int hashCode() { + return Arrays.hashCode(funcs); + } + + public boolean equals(Object other) { + if (other instanceof Stack) { + return Arrays.equals(funcs, ((Stack) other).funcs); + } + + return false; + } +} + +class MallocTraceResult { + private ArrayList stacks = new ArrayList<>(); + + public MallocTraceResult(BufferedReader reader) throws Exception { + String line; + + while (true) { + line = reader.readLine(); + + if (line.startsWith("Stack ")) { + break; + } + } + + while (true) { + String[] parts = line.split(":|(bytes,)"); + long bytes = Long.parseLong(parts[1].trim().split(" ")[0].replaceAll(",", "")); + long count = Long.parseLong(parts[2].trim().split(" ")[0].replaceAll(",", "")); + int index = Integer.parseInt(parts[0].trim().split(" ")[1].replaceAll(",", "")); + int maxIndex = Integer.parseInt(parts[0].trim().split(" ")[3].replaceAll(",", "")); + stacks.add(new Stack(reader, index, maxIndex, bytes, count)); + line = reader.readLine(); + + if (!line.startsWith("Stack ")) { + break; + } + } + } + + public static MallocTraceResult fromString(String output) throws Exception { + try (BufferedReader br = new BufferedReader(new StringReader(output))) { + return new MallocTraceResult(br); + } + } + + public int nrOfStacks() { + return stacks.size(); + } + + public Stack getStack(int index) { + return stacks.get(index); + } + + public String toString() { + StringBuilder result = new StringBuilder(); + + for (Stack s: stacks) { + result.append(s.toString()); + } + + return result.toString(); + } +} + +class MallocTraceExpectedStatistic { + private static String[] funcs = new String[] {"malloc", "calloc", "realloc", "posix_memalign", + "memalign", "aligned_alloc", "valloc", "pvalloc"}; + private long[] bytes = new long[funcs.length]; + private long[] counts = new long[funcs.length]; + + public MallocTraceExpectedStatistic(Process p) throws Exception { + BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream())); + + for (int i = 0; i < bytes.length; ++i) { + String line = br.readLine(); + System.out.println(i + ": " + line); + String[] parts = line.split(" "); + bytes[i] = Long.parseLong(parts[0].replaceAll(",", "")); + counts[i] = Long.parseLong(parts[1].replaceAll(",", "")); + } + } + + public void check(MallocTraceResult actual) throws Exception { + check(actual, 1, 0.0); + } + + public void check(MallocTraceResult actual, int nth, double diff) throws Exception { + for (int i = 0; i < funcs.length; ++i) { + if (counts[i] == 0) { + continue; // Not supported by platform + } + + boolean found = false; + + for (int j = 0; j < actual.nrOfStacks(); ++j) { + Stack stack = actual.getStack(j); + + if (stack.getFunction(0).startsWith(funcs[i] + (Platform.isOSX() ? "_interpose " : " "))) { + assertFalse(found, "Found entry for " + funcs[i] + " more than once"); + long expected_bytes = bytes[i] / nth; + long expected_count = counts[i] / nth; + // Check we are in the range of +/- 20 percent. + assertLTE(stack.getBytes(), (long) (expected_bytes * (1 + diff))); + assertGTE(stack.getBytes(), (long) (expected_bytes * (1 - diff))); + assertLTE(stack.getCount(), (long) (expected_count * (1 + diff))); + assertGTE(stack.getCount(), (long) (expected_count * (1 - diff))); + found = true; + } + } + + assertTrue(found, "Didn't found entry for " + funcs[i]); + } + } + + public String toString() { + StringBuilder result = new StringBuilder("Expacted results:" + System.lineSeparator()); + + for (int i = 0; i < funcs.length; ++i) { + result.append(funcs[i] + ": " + bytes[i] + " bytes, " + counts[i] + " counts" + System.lineSeparator()); + } + + return result.toString(); + } +} + diff --git a/test/hotspot/jtreg/runtime/malloctrace/MallocTraceDcmdTest.java b/test/hotspot/jtreg/runtime/malloctrace/MallocTraceDcmdTest.java new file mode 100644 index 00000000000..ef99371a256 --- /dev/null +++ b/test/hotspot/jtreg/runtime/malloctrace/MallocTraceDcmdTest.java @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, SAP and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +import jdk.test.lib.JDKToolFinder; +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; +import jdk.test.whitebox.WhiteBox; +import jtreg.SkippedException; + +import java.util.ArrayList; +import java.util.Random; + +// For now 64bit only, 32bit stack capturing still does not work that well + +/* + * @test + * @summary Test the System.malloctrace command + * @library /test/lib + * @requires vm.bits == "64" + * @requires os.family == "linux" + * @modules java.base/jdk.internal.misc + * java.management + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm/timeout=400 + * -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:-EnableMallocTrace + * MallocTraceDcmdTest + */ + +public class MallocTraceDcmdTest { + + private static class MallocStresser { + int numThreads = 5; + Thread threads[]; + static WhiteBox whiteBox = WhiteBox.getWhiteBox(); + + public MallocStresser(int numThreads) { + this.numThreads = numThreads; + } + + boolean stop = false; + + class StresserThread extends Thread { + public void run() { + Random rand = new Random(); + while (!stop) { + // We misuse NMT's whitebox functions for mallocing + // We also could use Unsafe.allocateMemory, but dealing with + // deprecation is annoying. + long p[] = new long[10]; + for (int i = 0; i < p.length; i++) { + p[i] = whiteBox.NMTMalloc(rand.nextInt(128)); + } + for (int i = 0; i < p.length; i++) { + whiteBox.NMTFree(p[i]); + } + } + } + } + + public void start() { + threads = new Thread[numThreads]; + for (int i = 0; i < numThreads; i ++) { + threads[i] = new StresserThread(); + threads[i].start(); + } + } + + public void stop() throws InterruptedException { + stop = true; + for (int i = 0; i < numThreads; i++) { + threads[i].join(); + } + } + } + + static boolean currentState = false; + static int numSwitchedOn = 0; // How often we switched tracing on + static int numReseted = 0; // How often we reseted + + static OutputAnalyzer testCommand(String... options) throws Exception { + OutputAnalyzer output; + // Grab my own PID + String pid = Long.toString(ProcessTools.getProcessId()); + String jcmdPath = JDKToolFinder.getJDKTool("jcmd"); + ArrayList command = new ArrayList<>(); + command.add(jcmdPath); + command.add(pid); + command.add("System.malloctrace"); + for (String option: options) { + if (option != null) { + command.add(option); + } + } + System.out.println("--- " + command); + ProcessBuilder pb = new ProcessBuilder(command); + output = new OutputAnalyzer(pb.start()); + output.shouldHaveExitValue(0); + return output; + } + + // System.malloctrace on + private static void testOn() throws Exception { + OutputAnalyzer output = testCommand("on"); + output.shouldContain("Tracing active"); + numSwitchedOn ++; + currentState = true; + } + + // System.malloctrace off + private static void testOff() throws Exception { + OutputAnalyzer output = testCommand("off"); + output.shouldContain("Tracing inactive"); + currentState = false; + } + + // System.malloctrace reset + private static void testReset() throws Exception { + OutputAnalyzer output = testCommand("reset"); + output.shouldContain("Tracing table reset"); + numReseted ++; + } + + // System.malloctrace print + private static void testPrint(boolean all) throws Exception { + OutputAnalyzer output = testCommand("print", all ? "all" : null); + if (currentState) { + output.shouldContain("WB_NMTMalloc"); + output.shouldContain("Malloc trace on"); + } else { + output.shouldContain("Malloc trace off"); + } + } + + // aka, Alpine + private static boolean NotAGlibcSystem() throws Exception { + OutputAnalyzer output = testCommand("print"); + return output.getStdout().contains("Not a glibc system"); + } + + public static void main(String args[]) throws Throwable { + + if (NotAGlibcSystem()) { + throw new SkippedException("Not a glibc system, skipping test"); + } + + if (!MallocTraceTestHelpers.GlibcSupportsMallocHooks()) { + throw new SkippedException("Glibc has no malloc hooks. Skipping test."); + } + + MallocStresser stresser = new MallocStresser(3); + stresser.start(); + Thread.sleep(1000); + testPrint(false); + Thread.sleep(500); + testOn(); + Thread.sleep(500); + testPrint(false); + testPrint(true); + testReset(); + testPrint(true); + testOn(); + Thread.sleep(500); + testOff(); + testOff(); + Thread.sleep(500); + testPrint(false); + testReset(); + testPrint(false); + testOff(); + } +} diff --git a/test/hotspot/jtreg/runtime/malloctrace/MallocTraceTest.java b/test/hotspot/jtreg/runtime/malloctrace/MallocTraceTest.java new file mode 100644 index 00000000000..12f71cc4508 --- /dev/null +++ b/test/hotspot/jtreg/runtime/malloctrace/MallocTraceTest.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021, 2023 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; +import jtreg.SkippedException; + +// SapMachine 2021-08-01: malloctrace +// For now 64bit only, 32bit stack capturing still does not work that well + + +/* + * @test MallocTraceTest + * @requires os.family == "linux" + * @requires vm.bits == "64" + * @library /test/lib + * @run driver MallocTraceTest on + */ + +/* + * @test MallocTraceTest + * @requires os.family == "linux" + * @requires vm.bits == "64" + * @library /test/lib + * @run driver MallocTraceTest off + */ + +public class MallocTraceTest { + + public static void main(String... args) throws Throwable { + + if (!MallocTraceTestHelpers.GlibcSupportsMallocHooks()) { + throw new SkippedException("Glibc has no malloc hooks. Skipping test."); + } + + if (args[0].equals("off") || args[0].equals("on")) { + boolean active = args[0].equals("on"); + String option = active ? "-XX:+EnableMallocTrace" : "-XX:-EnableMallocTrace"; + ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder("-XX:+UnlockDiagnosticVMOptions", + option, "-XX:+PrintMallocTraceAtExit", "-version"); + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.reportDiagnosticSummary(); + output.shouldHaveExitValue(0); + + // Ignore output for Alpine + if (output.getStderr().contains("Not a glibc system")) { + throw new SkippedException("Not a glibc system, skipping test"); + } + + if (active) { + String stdout = output.getStdout(); + // Checking for the correct frames is a whack-the-mole game since we cannot be sure how frames + // are inlined. Therefore we cannot just use "shouldContain". + output.stdoutShouldMatch("(os::malloc|my_malloc_hook)"); + output.shouldContain("Malloc trace on."); + } else { + output.shouldContain("Malloc trace off."); + } + } else { + throw new RuntimeException("Wrong or missing args."); + } + } +} diff --git a/test/hotspot/jtreg/runtime/malloctrace/MallocTraceTestHelpers.java b/test/hotspot/jtreg/runtime/malloctrace/MallocTraceTestHelpers.java new file mode 100644 index 00000000000..715d7d13d0b --- /dev/null +++ b/test/hotspot/jtreg/runtime/malloctrace/MallocTraceTestHelpers.java @@ -0,0 +1,27 @@ +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +public class MallocTraceTestHelpers { + + // Returns in the form: + // 0x0000MMmm + public static int GlibVersion() throws Throwable { + OutputAnalyzer output = ProcessTools.executeProcess("/usr/bin/ldd", "--version"); + output.reportDiagnosticSummary(); + String version = output.firstMatch("^ldd.* ([\\d+\\.]+)$", 1); + System.out.println("Glibc version is " + version); + if (version == null) { + return 0; // unknown, assume very low + } + String[] parts = version.split("\\."); + int major = Integer.parseInt(parts[0]); + int minor = Integer.parseInt(parts[1]); + int as_numeric = minor | (major << 8); + return as_numeric; + } + + public static boolean GlibcSupportsMallocHooks() throws Throwable { + return GlibVersion() < 0x220; // 2.32 + } + +} diff --git a/test/hotspot/jtreg/runtime/malloctrace/exetestmallochooks.c b/test/hotspot/jtreg/runtime/malloctrace/exetestmallochooks.c new file mode 100644 index 00000000000..00d96e48cfa --- /dev/null +++ b/test/hotspot/jtreg/runtime/malloctrace/exetestmallochooks.c @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#if defined(LINUX) || defined(__APPLE__) +#include +#include +#include +#include + +#include + +#include "mallochooks.h" + + +static void write_string(char const* str) { + size_t left = strlen(str); + char const* pos = str; + + while (left > 0) { + ssize_t result = write(1, pos, left); + + if (result <= 0) { + break; + } + + pos += result; + left -= result; + } +} + +static void check(bool condition, char const* msg) { + if (!condition) { + write_string("Check failed: "); + write_string(msg); + write_string("\n"); + exit(1); + } +} + +static real_malloc_funcs_t* funcs; + +static bool no_hooks_should_be_called; + +static void* test_malloc_hook(size_t size, void* caller) { + check(!no_hooks_should_be_called, "Called malloc hook when should not"); + no_hooks_should_be_called = true; + + void* result = funcs->malloc(size); + no_hooks_should_be_called = false; + + return result; +} + +static void* test_calloc_hook(size_t elems, size_t size, void* caller) { + check(!no_hooks_should_be_called, "Called calloc hook when should not"); + no_hooks_should_be_called = true; + + void* result = funcs->calloc(elems, size); + no_hooks_should_be_called = false; + + return result; +} + +static void* test_realloc_hook(void* ptr, size_t size, void* caller) { + check(!no_hooks_should_be_called, "Called realloc hook when should not"); + no_hooks_should_be_called = true; + + void* result = funcs->realloc(ptr, size); + no_hooks_should_be_called = false; + + return result; +} + +static void test_free_hook(void* ptr, void* caller) { + check(!no_hooks_should_be_called, "Called free hook when should not"); + no_hooks_should_be_called = true; + + funcs->free(ptr); + no_hooks_should_be_called = false; +} + +static int test_posix_memalign_hook(void** ptr, size_t align, size_t size, void* caller) { + check(!no_hooks_should_be_called, "Called posix_memalign hook when should not"); + no_hooks_should_be_called = true; + + int result = funcs->posix_memalign(ptr, align, size); + no_hooks_should_be_called = false; + + return result; +} + +static void* test_memalign_hook(size_t align, size_t size, void* caller) { + check(!no_hooks_should_be_called, "Called memalign hook when should not"); + no_hooks_should_be_called = true; + + void* result = funcs->memalign(align, size); + no_hooks_should_be_called = false; + + return result; +} + +static void* test_aligned_alloc_hook(size_t align, size_t size, void* caller) { + check(!no_hooks_should_be_called, "Called aligned_alloc hook when should not"); + no_hooks_should_be_called = true; + + void* result = funcs->aligned_alloc(align, size); + no_hooks_should_be_called = false; + + return result; +} + +static void* test_valloc_hook(size_t size, void* caller) { + check(!no_hooks_should_be_called, "Called valloc hook when should not"); + no_hooks_should_be_called = true; + + void* result = funcs->valloc(size); + no_hooks_should_be_called = false; + + return result; +} + +static void* test_pvalloc_hook(size_t size, void* caller) { + check(!no_hooks_should_be_called, "Called pvalloc hook when should not"); + no_hooks_should_be_called = true; + + void* result = funcs->pvalloc(size); + no_hooks_should_be_called = false; + + return result; +} + +static void test_no_recursive_calls() { + register_hooks_t* register_hooks = (register_hooks_t*) dlsym((void*) RTLD_DEFAULT, REGISTER_HOOKS_NAME); + check(register_hooks != NULL, "Could not get register function"); + get_real_malloc_funcs_t* get_real_malloc_funcs = (get_real_malloc_funcs_t*) + dlsym((void*) RTLD_DEFAULT, GET_REAL_MALLOC_FUNCS_NAME); + check(get_real_malloc_funcs != NULL, "Could not get get_real_funcs function"); + + registered_hooks_t test_hooks = { + test_malloc_hook, + test_calloc_hook, + test_realloc_hook, + test_free_hook, + test_posix_memalign_hook, + test_memalign_hook, + test_aligned_alloc_hook, + test_valloc_hook, + test_pvalloc_hook + }; + + funcs = get_real_malloc_funcs(); + register_hooks(&test_hooks); + + // Check that all the real functions do not trigger the hooks. + void* ptr; + + write_string("Testing malloc\n"); + funcs->malloc(0); + funcs->malloc(1); + + write_string("Testing calloc\n"); + funcs->calloc(0, 12); + funcs->calloc(12, 0); + funcs->calloc(12, 12); + + write_string("Testing realloc\n"); + funcs->realloc(NULL, 0); + funcs->realloc(NULL, 12); + funcs->realloc(funcs->malloc(12), 0); + funcs->realloc(funcs->malloc(12), 12); + + write_string("Testing free\n"); + funcs->free(NULL); + funcs->free(funcs->malloc(12)); + + write_string("Testing posix_memalign\n"); + funcs->posix_memalign(&ptr, 1024, 0); + funcs->posix_memalign(&ptr, 1024, 12); + + // MacOSX has no memalign and aligned_alloc. +#if !defined(__APPLE__) + write_string("Testing memalign\n"); + funcs->memalign(1024, 0); + funcs->memalign(1024, 12); + + write_string("Testing aligned_alloc\n"); + funcs->aligned_alloc(1024, 0); + funcs->aligned_alloc(1024, 12); +#endif + + // Musl has no valloc function. +#if defined(__GLIBC__) || defined(__APPLE__) + write_string("Testing valloc\n"); + funcs->valloc(0); + funcs->valloc(12); +#endif + + // Musl and MacOSX have no pvalloc function. +#if defined(__GLIBC__) + write_string("Testing pvalloc\n"); + funcs->pvalloc(0); + funcs->pvalloc(12); +#endif + + write_string("Testing hooks finished \n"); + register_hooks(NULL); +} + +int main(int argc, char** argv) { + test_no_recursive_calls(); + return 0; +} + +#else // defined(LINUX) || defined(__APPLE__) + +int main(int argc, char** argv) { + return 0; +} + +#endif // defined(LINUX) || defined(__APPLE__) + diff --git a/test/hotspot/jtreg/runtime/malloctrace/libfakemallochooks.c b/test/hotspot/jtreg/runtime/malloctrace/libfakemallochooks.c new file mode 100644 index 00000000000..abbac3cf624 --- /dev/null +++ b/test/hotspot/jtreg/runtime/malloctrace/libfakemallochooks.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +/* This is just a dummy library. */ diff --git a/test/hotspot/jtreg/runtime/malloctrace/libtestmallochooks.c b/test/hotspot/jtreg/runtime/malloctrace/libtestmallochooks.c new file mode 100644 index 00000000000..6e411619757 --- /dev/null +++ b/test/hotspot/jtreg/runtime/malloctrace/libtestmallochooks.c @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#if defined(LINUX) || defined(__APPLE__) + +#include +#include +#include +#include +#include + +#if defined(__APPLE__) +#define NO_OPT_ATTR __attribute__((optnone)) +#elif defined(LINUX) +#include +#define NO_OPT_ATTR __attribute__((optimize(0),noinline)) +#else +#error "Should not be compiled" +#endif + +#include "jni.h" + +#include "mallochooks.h" + +#define MAX_ALLOC 8192 +#define MAX_CALLOC 64 + +#define SAFE_PRIME_64 7570865665517081723ull +#define SAFE_PRIME 1000000007 + +int next_rand(int last, int base) { + return (int) ((((unsigned long long) last) * base) % SAFE_PRIME); +} + +#define MALLOC 0 +#define CALLOC 1 +#define REALLOC 2 +#define POSIX_MEMALIGN 3 +#define MEMALIGN 4 +#define ALIGNED_ALLOC 5 +#define VALLOC 6 +#define PVALLOC 7 + +#define TRACK(what, size) \ +if (roots[idx] != NULL) { \ + if (trackLive) { \ + sizes[(what)] += funcs->malloc_size(roots[idx]); \ + } else { \ + sizes[(what)] += (size); \ + } \ + counts[(what)] += 1; \ + source[(idx)] = (what); \ +} else { \ + source[(idx)] = -1; \ +} + +static void do_alloc_with_stack_impl(int size, int type); +static void do_alloc_with_stack2(int size, int type, int stack); + +static void NO_OPT_ATTR do_alloc_with_stack1(int size, int type, int stack) { + int new_stack = stack / 2; + + if (new_stack == 0) { + do_alloc_with_stack_impl(size, type); + } else if (new_stack & 1) { + do_alloc_with_stack1(size, type, new_stack); + } else { + do_alloc_with_stack2(size, type, new_stack); + } +} + +static void NO_OPT_ATTR do_alloc_with_stack2(int size, int type, int stack) { + int new_stack = stack / 2; + + if (new_stack == 0) { + do_alloc_with_stack_impl(size, type); + } else if (new_stack & 1) { + do_alloc_with_stack1(size, type, new_stack); + } else { + do_alloc_with_stack2(size, type, new_stack); + } +} + +static void do_alloc_with_stack_impl(int size, int type) { + void* mem = NULL; + + switch (type & 7) { + case 0: + mem = malloc(size); + break; + case 1: + mem = calloc(1, size); + break; + case 2: + mem = realloc(NULL, size); + break; + case 3: + if (posix_memalign(&mem, 128, size) != 0) { + mem = NULL; + } + break; +#if !defined(__APPLE__) + case 4: + mem = memalign(128, size); + break; +#endif +#if !defined(__APPLE__) + case 5: + mem = aligned_alloc(128, size); + break; +#endif +#if defined(__GLIBC__) || defined(__APPLE__) + case 6: + mem = valloc(size); + break; +#endif +#if defined(__GLIBC__) + case 7: + mem = pvalloc(size); + break; +#endif + default: + mem = malloc(size); + } + + free(mem); +} + +JNIEXPORT void JNICALL +Java_MallocHooksTest_doRandomAllocsWithFrees(JNIEnv *env, jclass cls, jint nrOfOps, jint size, + jint maxStack, jint seed) { + int i; + int rand = 1; + int stack_rand = 1; + + for (i = 0; i < nrOfOps; ++i) { + rand = next_rand(rand, seed); + stack_rand = next_rand(rand, seed); + + if (stack_rand & 1) { + do_alloc_with_stack1(size, rand, stack_rand & ((1 << maxStack) - 1)); + } else { + do_alloc_with_stack2(size, rand, stack_rand & ((1 << maxStack) - 1)); + } + + rand = stack_rand; + } +} + +JNIEXPORT void JNICALL +Java_MallocHooksTest_doRandomMemOps(JNIEnv *env, jclass cls, jint nrOfOps, jint maxLiveAllocations, jint seed, + jboolean trackLive, jlongArray resultSizes, jlongArray resultCounts) { + get_real_malloc_funcs_t* get_real_malloc_funcs = (get_real_malloc_funcs_t*) + dlsym((void*) RTLD_DEFAULT, GET_REAL_MALLOC_FUNCS_NAME); + real_malloc_funcs_t* funcs = get_real_malloc_funcs(); + + void** roots = funcs->calloc(maxLiveAllocations, sizeof(void*)); + signed char* source = (signed char*) funcs->calloc(maxLiveAllocations, sizeof(char)); + + int i; + int rand = 1; + jlong sizes[] = {0, 0, 0, 0, 0, 0, 0, 0}; + jlong counts[] = {0, 0, 0, 0, 0, 0, 0, 0}; + + for (i = 0; i < nrOfOps; ++i) { + rand = next_rand(rand, seed); + int idx = rand % maxLiveAllocations; + + if (roots[idx] == NULL) { + rand = next_rand(rand, seed); + int what = rand & 31; + rand = next_rand(rand, seed); + int malloc_size = rand & (MAX_ALLOC - 1); + int calloc_size = rand & (MAX_CALLOC - 1); + + if (what < 11) { + roots[idx] = malloc(malloc_size + 1); + TRACK(MALLOC, malloc_size + 1); + } else if (what < 22) { + rand = next_rand(rand, seed); + int calloc_count = rand & (MAX_CALLOC - 1); + roots[idx] = calloc(calloc_count + 1, calloc_size + 1); + TRACK(CALLOC, (calloc_count + 1) * (calloc_size + 1)); + } else if (what < 24) { + void* mem; + int result = posix_memalign(&mem, 64, malloc_size + 1); + roots[idx] = result != 0 ? NULL : mem; + TRACK(POSIX_MEMALIGN, result != 0 ? 0 : funcs->malloc_size(mem)); + } else if (what < 26) { +#if !defined(__APPLE__) + roots[idx] = memalign(64, malloc_size + 1); + TRACK(MEMALIGN, roots[idx] == NULL ? 0 : funcs->malloc_size(roots[idx])); +#endif + } else if (what < 28) { +#if !defined(__APPLE__) + roots[idx] = aligned_alloc(64, malloc_size + 1); + TRACK(ALIGNED_ALLOC, roots[idx] == NULL ? 0 : funcs->malloc_size(roots[idx])); +#endif + } else if (what < 30) { +#if defined(__GLIBC__) || defined(__APPLE__) + roots[idx] = valloc(malloc_size + 1); + TRACK(VALLOC, roots[idx] == NULL ? 0 : funcs->malloc_size(roots[idx])); +#endif + } else { +#if defined(__GLIBC__) + roots[idx] = pvalloc(malloc_size + 1); + TRACK(PVALLOC, roots[idx] == NULL ? 0 : funcs->malloc_size(roots[idx])); +#endif + } + } else { + rand = next_rand(rand, seed); + + if ((rand & 3) != 0) { + if (trackLive) { + sizes[source[idx]] -= funcs->malloc_size(roots[idx]); + counts[source[idx]] -= 1; + } + free(roots[idx]); + roots[idx] = NULL; + source[idx] = -1; + } else { + rand = next_rand(rand, seed); + size_t old_size = funcs->malloc_size(roots[idx]); + int malloc_size = rand & (MAX_ALLOC - 1); + roots[idx] = realloc(roots[idx], malloc_size + 1); + if (roots[idx] != NULL) { + if (trackLive) { + sizes[source[idx]] -= old_size; + counts[source[idx]] -= 1; + } + } + if (trackLive) { + TRACK(REALLOC, malloc_size + 1); + } else if (old_size < (size_t) (malloc_size + 1)) { + TRACK(REALLOC, malloc_size + 1 - old_size); + } + } + } + } + + /* Free at least some the memory. We cannot do this when tracking live, obviously. */ + if (!trackLive) { + for (i = 0; i < maxLiveAllocations; ++i) { + free(roots[i]); + } + } + + funcs->free(roots); + funcs->free(source); + + (*env)->SetLongArrayRegion(env, resultSizes, 0, 8, sizes); + (*env)->SetLongArrayRegion(env, resultCounts, 0, 8, counts); +} + +#endif // defined(LINUX) || defined(__APPLE__) + diff --git a/test/hotspot/jtreg/serviceability/HeapDump/HeapDumpJcmdPresetPathTest.java b/test/hotspot/jtreg/serviceability/HeapDump/HeapDumpJcmdPresetPathTest.java new file mode 100644 index 00000000000..e23a5ebf1f1 --- /dev/null +++ b/test/hotspot/jtreg/serviceability/HeapDump/HeapDumpJcmdPresetPathTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @summary Test jcmd GC.heap_dump without path option; path is set via HeapDumpPath + * @library /test/lib + * @run main/othervm -XX:HeapDumpPath=testjcmd.hprof HeapDumpJcmdPresetPathTest + */ + +import java.io.File; + +import jdk.test.lib.Asserts; +import jdk.test.lib.dcmd.PidJcmdExecutor; +import jdk.test.lib.process.OutputAnalyzer; + +public class HeapDumpJcmdPresetPathTest { + + public static void main(String[] args) throws Exception { + PidJcmdExecutor executor = new PidJcmdExecutor(); + OutputAnalyzer output = executor.execute("GC.heap_dump"); + output.shouldContain("Dumping heap to testjcmd.hprof"); + output.shouldContain("Heap dump file created"); + + Asserts.assertTrue(new File("testjcmd.hprof").exists()); + } +} diff --git a/test/jaxp/ProblemList-SapMachine.txt b/test/jaxp/ProblemList-SapMachine.txt new file mode 100644 index 00000000000..5c1f6da0deb --- /dev/null +++ b/test/jaxp/ProblemList-SapMachine.txt @@ -0,0 +1,43 @@ +############################################################################### +# +# This is the additional jtreg exclude list for SapMachine jaxp tests. +# +# List of tests that should not be run by test/Makefile, for various reasons: +# 1. Does not run with jtreg -samevm mode +# 2. Causes problems in jtreg -samevm mode for jtreg or tests that follow it +# 3. The test is too slow or consumes too many system resources +# 4. The test fails when run on any official build systems +# +# Tests marked @ignore are not run by test/Makefile, but harmless to be listed. +# +# List items are testnames followed by labels, all MUST BE commented +# as to why they are here and use a label: +# generic-all Problems on all platforms +# generic-ARCH Where ARCH is one of: sparc, sparcv9, x64, i586, ppc64, +# ppc64le, s390x etc +# OSNAME-all Where OSNAME is one of: solaris, linux, windows, macosx, aix +# OSNAME-ARCH Specific on to one OSNAME and ARCH, e.g. solaris-amd64 +# OSNAME-REV Specific on to one OSNAME and REV, e.g. solaris-5.8 +# +# More than one label is allowed but must be on the same line comma seperated, +# without spaces! +# If there are several lines, the last one is used. +# +# SAP/SapMachine usage notes: +# +# This exclude list is a vehicle only for temporary exclusions of tests +# or exclusions that are caused by infrastrucure specifics. +# +# Our first goal is to fix test issues upstream or at least open upstream +# bugs and get the test excluded via the upstream exclusion list. +# +# This list is refreshed periodically from an SAP-internal version, +# removing comments which reveal internal URLs, names or hostnames. +# +# It might contain additional test exclusions, specific to the SapMachine build +# and test infrastructure. That section is found at the end of the file. +# +############################################################################### + +############################################################################### +# Tests known to be failing in SapMachine due to SapMachine specific setup. diff --git a/test/jdk/TEST.groups b/test/jdk/TEST.groups index fa868699aab..6cf9fcd1ed7 100644 --- a/test/jdk/TEST.groups +++ b/test/jdk/TEST.groups @@ -37,7 +37,9 @@ jdk_all = \ # # When adding tests to tier1, make sure they end up in one of the tier1_partX groups +# SapMachine 2019-01-24: Add tests where SAPMachine has different behavior to tier1 tier1 = \ + :tier1_sapmachine \ :tier1_part1 \ :tier1_part2 \ :tier1_part3 @@ -58,6 +60,14 @@ tier1_part3 = \ jdk/classfile \ sun/nio/cs/ISO8859x.java +# SapMachine 2019-01-24: Add tests where SAPMachine has different behavior to tier1 +tier1_sapmachine = \ + java/net/Socket/ExceptionText.java \ + java/util/jar/Manifest/IncludeInExceptionsTest.java \ + jdk/security/JavaDotSecurity/TestJDKIncludeInExceptions.java \ + sun/net/www/http/KeepAliveCache/TestConnectionIDFeature.java \ + sun/security/lib/cacerts/VerifyCACerts.java + # When adding tests to tier2, make sure they end up in one of the tier2_partX groups tier2 = \ :tier2_part1 \ diff --git a/test/jdk/com/sun/jdi/FileSocketTransportTest.java b/test/jdk/com/sun/jdi/FileSocketTransportTest.java new file mode 100644 index 00000000000..d5505d4db9e --- /dev/null +++ b/test/jdk/com/sun/jdi/FileSocketTransportTest.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2018, 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @summary Test basic functionality of the file socket debug transport. + * + * @author Ralf Schmelter + * + * @library /test/lib + * @run main/othervm FileSocketTransportTest + */ + +import static jdk.test.lib.Asserts.assertEquals; +import static jdk.test.lib.Asserts.assertTrue; +import static jdk.test.lib.Asserts.assertFalse; + +import java.io.File; +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +import jdk.test.lib.Platform; +import jdk.test.lib.Utils; +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +public class FileSocketTransportTest { + + private static int handshake(String path, byte[] handshake, byte[] reply) throws IOException { + UnixDomainSocketAddress addr = UnixDomainSocketAddress.of(path); + SocketChannel channel = SocketChannel.open(StandardProtocolFamily.UNIX); + channel.connect(addr); + ByteBuffer out = ByteBuffer.wrap(handshake); + while (out.hasRemaining()) { + channel.write(out); + } + ByteBuffer in = ByteBuffer.wrap(reply); + int result = channel.read(in); + channel.close(); + + return result; + } + + private static void dumpHsErrorFiles() throws Exception { + for (File f: new File(".").listFiles()) { + if (!f.isDirectory() && f.getName().startsWith("hs_err")) { + System.out.println("Found " + f.getName() + ":"); + String output = new String(Files.readAllBytes(f.toPath())); + System.out.println(output); + System.out.println("------- End of " + f.getName()); + // Print the start again, since we might overflow the buffer with + // the whole file. + int startLength = 32768; + + if (output.length() > startLength) { + System.out.println("------- Repeating start of " + f.getName()); + System.out.println(output.substring(0, startLength)); + System.out.println("------- End of start of " + f.getName()); + } + } + } + } + + public static void main(String[] args) throws Throwable { + if (args.length == 1 && "--sleep".equals(args[0])) { + Thread.sleep(30000); + System.exit(1); + } + + String socketName = "test.socket"; + + List opts = new ArrayList<>(); + opts.add("-agentlib:jdwp=transport=dt_filesocket,address=" + + socketName + ",server=y,suspend=n"); + opts.add(FileSocketTransportTest.class.getName()); + opts.add("--sleep"); + + // First check if we get the expected errors. + ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder( + opts.toArray(new String[0])); + Process proc = pb.start(); + new Thread(() -> { + try { + OutputAnalyzer output = new OutputAnalyzer(proc); + System.out.println("Output of debuggee:"); + System.out.println(">>>>> START <<<<<"); + System.out.println(output.getOutput()); + System.out.println(">>>>> END <<<<<"); + } catch (IOException e) { + e.printStackTrace(); + } + }).start(); + + // Debug 3 times. + try { + byte[] handshake = "JDWP-Handshake".getBytes("UTF-8"); + byte[] received = new byte[handshake.length]; + int read; + + for (int i = 0; i < 3; ++i) { + System.out.println("Run " + i); + // Wait a bit to let the debugging be set up properly. + Thread.sleep(3000); + checkSocketPresent(socketName); + read = handshake(socketName, handshake, received); + checkSocketDeleted(socketName); + assertEquals(new String(handshake, "UTF-8"), + new String(received, "UTF-8")); + assertEquals(read, received.length); + } + } finally { + dumpHsErrorFiles(); + Thread.sleep(2000); + checkSocketPresent(socketName); + proc.destroy(); + Thread.sleep(2000); + checkSocketDeleted(socketName); + } + } + + private static void checkSocketPresent(String name) { + if (!Platform.isWindows()) { + assertTrue(new File(name).exists(), "Socket " + name + " missing"); + } + } + + private static void checkSocketDeleted(String name) { + if (!Platform.isWindows()) { + assertFalse(new File(name).exists(), "Socket " + name + " exists"); + } + } +} diff --git a/test/jdk/java/net/Socket/ExceptionText.java b/test/jdk/java/net/Socket/ExceptionText.java index 7f565e9777c..d49aed71ded 100644 --- a/test/jdk/java/net/Socket/ExceptionText.java +++ b/test/jdk/java/net/Socket/ExceptionText.java @@ -21,6 +21,9 @@ * questions. */ + +// SapMachine 2018-11-23: SapMachine has set jdk.includeInExceptions to hostInfo,jar +// by default. Therefore expect according output! /* * @test * @library /test/lib @@ -29,7 +32,7 @@ * @summary Add configurable option for enhanced socket IOException messages * @run main/othervm * ExceptionText - * WITHOUT_Enhanced_Text + * expectEnhancedText * @run main/othervm * -Djdk.includeInExceptions= * ExceptionText diff --git a/test/jdk/java/util/jar/Manifest/IncludeInExceptionsTest.java b/test/jdk/java/util/jar/Manifest/IncludeInExceptionsTest.java index 1e272015087..642bd3621bc 100644 --- a/test/jdk/java/util/jar/Manifest/IncludeInExceptionsTest.java +++ b/test/jdk/java/util/jar/Manifest/IncludeInExceptionsTest.java @@ -34,10 +34,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; /** + * SapMachine 2019-01-18: In SapMachine "jdk.includeInExceptions" contains the + * value "jar" by default. So we simply switch the properties for the test VMs. + * * @test * @bug 8216362 - * @run main/othervm -Djdk.includeInExceptions=jar IncludeInExceptionsTest yes - * @run main/othervm IncludeInExceptionsTest + * @run main/othervm IncludeInExceptionsTest yes + * @run main/othervm -Djdk.includeInExceptions= IncludeInExceptionsTest * @summary Verify that the property jdk.net.includeInExceptions works as expected * when an error occurs while reading an invalid Manifest file. */ diff --git a/test/jdk/jdk/jfr/api/recording/settings/TestGetConfigurations.java b/test/jdk/jdk/jfr/api/recording/settings/TestGetConfigurations.java index 465a07ab021..67f8714408d 100644 --- a/test/jdk/jdk/jfr/api/recording/settings/TestGetConfigurations.java +++ b/test/jdk/jdk/jfr/api/recording/settings/TestGetConfigurations.java @@ -48,10 +48,22 @@ public class TestGetConfigurations { private static final String PROFILE_CONFIG_DESCRIPTION = "Low overhead configuration for profiling, typically around 2 % overhead."; private static final String PROFILE_CONFIG_PROVIDER = "Oracle"; + //SapMachine 2023-02-20 Add the two additional configs and checks + private static final String GC_CONFIG_NAME = "gc"; + private static final String GC_CONFIG_LABEL = "gc"; + private static final String GC_CONFIG_DESCRIPTION = "Configuration for GC related events. No stack traces. Small recording size."; + private static final String GC_CONFIG_PROVIDER = "SAP SE"; + + private static final String GC_DETAILS_CONFIG_NAME = "gc_details"; + private static final String GC_DETAILS_CONFIG_LABEL = "gc_details"; + private static final String GC_DETAILS_CONFIG_DESCRIPTION = "Configuration for all GC related events. Higher impact caused by heap inspection initiated GCs to get heap statistics. Large recording size."; + private static final String GC_DETAILS_CONFIG_PROVIDER = "SAP SE"; + public static void main(String[] args) throws Throwable { List predefinedConfigs = Configuration.getConfigurations(); Asserts.assertNotNull(predefinedConfigs, "List of predefined configs is null"); - Asserts.assertEquals(predefinedConfigs.size(), 2, "Expected exactly two predefined configurations"); + //SapMachine 2023-02-20 Add the two additional configs and checks + Asserts.assertEquals(predefinedConfigs.size(), 4, "Expected exactly four predefined configurations"); Configuration defaultConfig = findConfigByName(predefinedConfigs, DEFAULT_CONFIG_NAME); Asserts.assertNotNull(defaultConfig, "Config '" + DEFAULT_CONFIG_NAME + "' not found"); @@ -64,6 +76,19 @@ public static void main(String[] args) throws Throwable { Asserts.assertEquals(profileConfig.getLabel(), PROFILE_CONFIG_LABEL); Asserts.assertEquals(profileConfig.getDescription(), PROFILE_CONFIG_DESCRIPTION); Asserts.assertEquals(profileConfig.getProvider(), PROFILE_CONFIG_PROVIDER); + + //SapMachine 2023-02-20 Add the two additional configs and checks + Configuration gcConfig = findConfigByName(predefinedConfigs, GC_CONFIG_NAME); + Asserts.assertNotNull(gcConfig, "Config '" + GC_CONFIG_NAME + "' not found"); + Asserts.assertEquals(gcConfig.getLabel(), GC_CONFIG_LABEL); + Asserts.assertEquals(gcConfig.getDescription(), GC_CONFIG_DESCRIPTION); + Asserts.assertEquals(gcConfig.getProvider(), GC_CONFIG_PROVIDER); + + Configuration gcDetailsConfig = findConfigByName(predefinedConfigs, GC_DETAILS_CONFIG_NAME); + Asserts.assertNotNull(gcDetailsConfig, "Config '" + GC_DETAILS_CONFIG_NAME + "' not found"); + Asserts.assertEquals(gcDetailsConfig.getLabel(), GC_DETAILS_CONFIG_LABEL); + Asserts.assertEquals(gcDetailsConfig.getDescription(), GC_DETAILS_CONFIG_DESCRIPTION); + Asserts.assertEquals(gcDetailsConfig.getProvider(), GC_DETAILS_CONFIG_PROVIDER); } private static Configuration findConfigByName(List configs, String name) { diff --git a/test/jdk/jdk/security/JavaDotSecurity/TestJDKIncludeInExceptions.java b/test/jdk/jdk/security/JavaDotSecurity/TestJDKIncludeInExceptions.java index 353309c1654..355b10ea53a 100644 --- a/test/jdk/jdk/security/JavaDotSecurity/TestJDKIncludeInExceptions.java +++ b/test/jdk/jdk/security/JavaDotSecurity/TestJDKIncludeInExceptions.java @@ -34,11 +34,14 @@ */ public class TestJDKIncludeInExceptions { + // SapMachine 2018-11-23: SapMachine has a different default for jdk.includeInExceptions + private static final String EXPECTED = "hostInfo,jar"; + public static void main(String args[]) throws Exception { String incInExc = Security.getProperty("jdk.includeInExceptions"); - if (incInExc != null) { + if (!EXPECTED.equals(incInExc)) { throw new RuntimeException("Test failed: default value of " + - "jdk.includeInExceptions security property is not null: " + + "jdk.includeInExceptions security property is not " + EXPECTED + ": " + incInExc); } } diff --git a/test/jdk/sun/net/www/http/KeepAliveCache/TestConnectionIDFeature.java b/test/jdk/sun/net/www/http/KeepAliveCache/TestConnectionIDFeature.java new file mode 100644 index 00000000000..6edf5097dad --- /dev/null +++ b/test/jdk/sun/net/www/http/KeepAliveCache/TestConnectionIDFeature.java @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2024 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary Test the SapMachine specific KeepAliveCache connectionID feature + * @library /test/lib + * @modules java.base/sun.net.www.http + * @run main/othervm -Dcom.sap.jvm.UseHttpKeepAliveCacheKeyExtension=true TestConnectionIDFeature + */ + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.URL; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Supplier; + +import jdk.test.lib.net.URIBuilder; + +import sun.net.www.http.KeepAliveCache; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +/* + * At first, the client opens 5 connections which get cached. + * Then in a second round of requests each thread should use the connection which + * is requested through the value of the KeepAliveCache.connectionID field. + */ +public class TestConnectionIDFeature { + static final byte[] PAYLOAD = "hello".getBytes(); + static final int CLIENT_CONNECTIONS = 6; + + static final ExecutorService serverExecutor = Executors.newSingleThreadExecutor(); + static final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); + static HttpServer server; + + static ArrayDeque connectionIds = new ArrayDeque<>(); + static Map clientPorts = new ConcurrentHashMap<>(); + static Map clientAsserts = new ConcurrentHashMap<>(); + static CountDownLatch clientSync = new CountDownLatch(CLIENT_CONNECTIONS); + static List> clientFutures = new ArrayList<>(CLIENT_CONNECTIONS); + + static class TestHttpHandler implements HttpHandler { + public void handle(HttpExchange trans) { + String connectionId = trans.getRequestURI().getPath().substring(1); + int port = trans.getRemoteAddress().getPort(); + if (clientPorts.containsKey(connectionId)) { + int expectedPort = clientPorts.get(connectionId); + if (expectedPort == port) { + System.out.println("Server handler for connectionId " + connectionId + ": Incoming connection seemingly reuses old connection (from port " + expectedPort + ")"); + } else { + String msg = "Server handler for connectionId " + connectionId + ": Incoming connection from different port (" + port + " instead of " + expectedPort + ")"; + System.out.println(msg); + clientAsserts.put(connectionId, msg); + } + } else { + System.out.println("Server handler for connectionId " + connectionId + ": Adding " + connectionId + "->" + port); + clientPorts.put(connectionId, port); + } + try { + trans.sendResponseHeaders(200, PAYLOAD.length); + try (OutputStream os = trans.getResponseBody()) { + os.write(PAYLOAD); + } + } catch (IOException e) { + clientAsserts.put(connectionId, e.getMessage()); + throw new RuntimeException(e); + } + } + } + + static abstract class Request implements Supplier { + String connectionId; + + Request(String connectionId) { + this.connectionId = connectionId; + } + } + + static class InitialRequest extends Request { + InitialRequest(String connectionId) { + super(connectionId); + } + + @Override + public String get() { + System.out.println("Running initial request for key: " + connectionId); + KeepAliveCache.connectionID.set(connectionId); + try { + URL url = URIBuilder.newBuilder() + .scheme("http") + .host(InetAddress.getLocalHost()) + .port(server.getAddress().getPort()) + .path("/" + connectionId) + .toURL(); + + try (InputStream is = url.openConnection(Proxy.NO_PROXY).getInputStream()) { + clientSync.countDown(); + clientSync.await(); + byte[] ba = new byte[PAYLOAD.length]; + is.read(ba); + } + System.out.println("Initial request for key " + connectionId + " done."); + return connectionId; + } catch (Exception e) { + throw new RuntimeException("Error in request thread for key " + connectionId + ".", e); + } + } + } + + static class SecondRequest extends Request { + SecondRequest(String connectionId) { + super(connectionId); + } + + @Override + public String get() { + System.out.println("Running second request for key: " + connectionId); + KeepAliveCache.connectionID.set(connectionId); + try { + URL url = URIBuilder.newBuilder() + .scheme("http") + .host(InetAddress.getLocalHost()) + .port(server.getAddress().getPort()) + .path("/" + connectionId) + .toURL(); + try (InputStream is = url.openConnection(Proxy.NO_PROXY).getInputStream()) { + byte[] ba = new byte[PAYLOAD.length]; + is.read(ba); + } + System.out.println("Second request for key " + connectionId + " done."); + return connectionId; + } catch (Exception e) { + throw new RuntimeException("Error in request thread for key " + connectionId + "."); + } + } + } + + public static void initialize() { + // start server + try { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLocalHost(), 0), 10, "/", new TestConnectionIDFeature.TestHttpHandler()); + } catch (IOException e) { + throw new RuntimeException("Could not create server", e); + } + server.setExecutor(serverExecutor); + server.start(); + + // initialize thread keys + for (int i = 0; i < CLIENT_CONNECTIONS; i++) { + connectionIds.push(Integer.toString(i)); + } + } + + public static void runRequests() { + // run initial set of requests in parallel to make sure that as many connections as the value of + // CLIENT_THREADS are open. This is achieved by waiting for a joined synchronization latch while + // the connections are still open. + while (connectionIds.peek() != null) { + clientFutures.add(CompletableFuture.supplyAsync(new InitialRequest(connectionIds.pop()), executor)); + } + for (var future : clientFutures) { + connectionIds.push(future.join()); + } + + // run second batch of requests where we expect that connections be reused + clientFutures.clear(); + while (connectionIds.peek() != null) { + clientFutures.add(CompletableFuture.supplyAsync(new InitialRequest(connectionIds.pop()), executor)); + } + for (var future : clientFutures) { + connectionIds.push(future.join()); + } + + // now check for any failures + while (connectionIds.peek() != null) { + String assertMsg = clientAsserts.get(connectionIds.pop()); + if (assertMsg != null) { + throw new RuntimeException(assertMsg); + } + } + } + + public static void shutdown() { + server.stop(0); + serverExecutor.shutdown(); + executor.shutdown(); + } + + public static void main(String[] args) { + initialize(); + try { + runRequests(); + } finally { + shutdown(); + } + } +} diff --git a/test/jdk/sun/security/lib/cacerts/VerifyCACerts.java b/test/jdk/sun/security/lib/cacerts/VerifyCACerts.java index 83427f1a33a..70af74e7ed8 100644 --- a/test/jdk/sun/security/lib/cacerts/VerifyCACerts.java +++ b/test/jdk/sun/security/lib/cacerts/VerifyCACerts.java @@ -48,12 +48,14 @@ public class VerifyCACerts { + File.separator + "security" + File.separator + "cacerts"; // The numbers of certs now. - private static final int COUNT = 110; + // SapMachine 2021-09-23: Additional certificate for SAP + private static final int COUNT = 111; // SHA-256 of cacerts, can be generated with // shasum -a 256 cacerts | sed -e 's/../&:/g' | tr '[:lower:]' '[:upper:]' | cut -c1-95 + // SapMachine 2021-09-23: Additional certificate for SAP private static final String CHECKSUM - = "BD:80:65:81:68:E5:6C:51:64:ED:B9:08:53:9F:BB:2F:D9:6C:5D:D4:06:D4:16:59:39:10:8E:F8:24:81:8B:78"; + = "B3:DF:BD:E2:E9:8C:D3:53:2A:ED:28:2A:EB:24:BF:4E:CE:4A:1F:71:57:87:50:23:42:28:28:7D:94:3D:CD:43"; // Hex formatter to upper case with ":" delimiter private static final HexFormat HEX = HexFormat.ofDelimiter(":").withUpperCase(); @@ -124,6 +126,9 @@ public class VerifyCACerts { "B4:78:B8:12:25:0D:F8:78:63:5C:2A:A7:EC:7D:15:5E:AA:62:5E:E8:29:16:E2:CD:29:43:61:88:6C:D1:FB:D4"); put("geotrustuniversalca [jdk]", "A0:45:9B:9F:63:B2:25:59:F5:FA:5D:4C:6D:B3:F9:F7:2F:F1:93:42:03:35:78:F0:73:BF:1D:1B:46:CB:B9:12"); + // SapMachine 2021-09-23: Additional certificate for SAP + put("sapglobalrootca [jdk]", + "56:53:9C:1E:7B:5E:D5:58:2B:79:68:00:61:CB:F2:14:86:A8:50:22:6B:2F:CF:30:B5:B1:52:A7:20:E1:34:DE"); put("thawteprimaryrootca [jdk]", "8D:72:2F:81:A9:C1:13:C0:79:1D:F1:36:A2:96:6D:B2:6C:95:0A:97:1D:B4:6B:41:99:F4:EA:54:B7:8B:FB:9F"); put("thawteprimaryrootcag2 [jdk]",