Merge to tag jdk-21+30

src/hotspot/share/classfile/vmClassMacros.hpp

@@ -31,7 +31,7 @@
 #define VM_CLASS_ID(kname) vmClassID::_VM_CLASS_ENUM(kname)
 
 // VM_CLASSES_DO iterates the classes that are directly referenced
-// by the VM, suhch as java.lang.Object and java.lang.String. These
+// by the VM, such as java.lang.Object and java.lang.String. These
 // classes are resolved at VM bootstrap, before any Java code is executed,
 // so no class loader is able to provide a different definition.
 //

src/hotspot/share/classfile/vmSymbols.hpp

@@ -760,7 +760,7 @@
  template(decodeAndThrowThrowable_name, "decodeAndThrowThrowable") \
  template(encodeAnnotations_name, "encodeAnnotations") \
  template(encodeAnnotations_signature, "([BLjava/lang/Class;Ljdk/internal/reflect/ConstantPool;Z[Ljava/lang/Class;)[B")\
- template(decodeAndThrowThrowable_signature, "(JZ)V")  \
+ template(decodeAndThrowThrowable_signature, "(IJZ)V") \
  template(classRedefinedCount_name, "classRedefinedCount") \
  template(classLoader_name, "classLoader") \
  template(componentType_name, "componentType") \

src/hotspot/share/gc/z/zGlobals.hpp

@@ -38,6 +38,9 @@ const size_t ZGranuleSize = (size_t)1 << ZGranuleSizeShift
 // Virtual memory to physical memory ratio
 const size_t ZVirtualToPhysicalRatio = 16; // 16:1
 
+// Max virtual memory ranges
+const size_t ZMaxVirtualReservations = 100; // Each reservation at least 1% of total
+
 // Page size shifts
 const size_t ZPageSizeSmallShift = ZGranuleSizeShift;
 extern size_t ZPageSizeMediumShift;

src/hotspot/share/gc/z/zMemory.cpp

@@ -81,6 +81,10 @@ ZMemoryManager::ZMemoryManager()
  : _freelist(),
  _callbacks() {}
 
+bool ZMemoryManager::free_is_contiguous() const {
+ return _freelist.size() == 1;
+}
+
 void ZMemoryManager::register_callbacks(const Callbacks& callbacks) {
  _callbacks = callbacks;
 }

src/hotspot/share/gc/z/zMemory.hpp

@@ -81,6 +81,8 @@ class ZMemoryManager {
 public:
  ZMemoryManager();
 
+ bool free_is_contiguous() const;
+
  void register_callbacks(const Callbacks& callbacks);
 
  zoffset peek_low_address() const;

src/hotspot/share/gc/z/zNMT.cpp

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zNMT.hpp"
+#include "gc/z/zVirtualMemory.hpp"
+#include "memory/allocation.hpp"
+#include "services/memTracker.hpp"
+#include "utilities/nativeCallStack.hpp"
+
+ZNMT::Reservation ZNMT::_reservations[ZMaxVirtualReservations] = {};
+size_t ZNMT::_num_reservations = 0;
+
+size_t ZNMT::reservation_index(zoffset offset, size_t* offset_in_reservation) {
+ assert(_num_reservations > 0, "at least one reservation must exist");
+
+ size_t index = 0;
+ *offset_in_reservation = untype(offset);
+ for (; index < _num_reservations; ++index) {
+ const size_t reservation_size = _reservations[index]._size;
+ if (*offset_in_reservation < reservation_size) {
+ break;
+ }
+ *offset_in_reservation -= reservation_size;
+ }
+
+ assert(index != _num_reservations, "failed to find reservation index");
+ return index;
+}
+
+void ZNMT::process_fake_mapping(zoffset offset, size_t size, bool commit) {
+ // In order to satisfy NTM's requirement of an 1:1 mapping between committed
+ // and reserved addresses, a fake mapping from the offset into the reservation
+ // is used.
+ //
+ // These mappings from
+ // [offset, offset + size) -> {[virtual address range], ...}
+ // are stable after the heap has been reserved. No commits proceed any
+ // reservations. Committing and uncommitting the same [offset, offset + size)
+ // range will result in same virtual memory ranges.
+
+ size_t left_to_process = size;
+ size_t offset_in_reservation;
+ for (size_t i = reservation_index(offset, &offset_in_reservation); i < _num_reservations; ++i) {
+ const zaddress_unsafe reservation_start = _reservations[i]._start;
+ const size_t reservation_size = _reservations[i]._size;
+ const size_t sub_range_size = MIN2(left_to_process, reservation_size - offset_in_reservation);
+ const uintptr_t sub_range_addr = untype(reservation_start) + offset_in_reservation;
+
+ // commit / uncommit memory
+ if (commit) {
+ MemTracker::record_virtual_memory_commit((void*)sub_range_addr, sub_range_size, CALLER_PC);
+ } else {
+ if (MemTracker::enabled()) {
+ Tracker tracker(Tracker::uncommit);
+ tracker.record((address)sub_range_addr, sub_range_size);
+ }
+ }
+
+ left_to_process -= sub_range_size;
+ if (left_to_process == 0) {
+ // Processed all nmt registrations
+ return;
+ }
+
+ offset_in_reservation = 0;
+ }
+
+ assert(left_to_process == 0, "everything was not commited");
+}
+
+void ZNMT::reserve(zaddress_unsafe start, size_t size) {
+ assert(_num_reservations < ZMaxVirtualReservations, "too many reservations");
+ // Keep track of the reservations made in order to create fake mappings
+ // between the reserved and commited memory.
+ // See details in ZNMT::process_fake_mapping
+ _reservations[_num_reservations++] = {start, size};
+
+ MemTracker::record_virtual_memory_reserve((void*)untype(start), size, CALLER_PC, mtJavaHeap);
+}
+
+void ZNMT::commit(zoffset offset, size_t size) {
+ // NMT expects a 1-to-1 mapping between virtual and physical memory.
+ // ZGC can temporarily have multiple virtual addresses pointing to
+ // the same physical memory.
+ //
+ // When this function is called we don't know where in the virtual memory
+ // this physical memory will be mapped. So we fake the virtual memory
+ // address by mapping the physical offset into offsets in the reserved
+ // memory space.
+ process_fake_mapping(offset, size, true);
+}
+
+void ZNMT::uncommit(zoffset offset, size_t size) {
+ // We fake the virtual memory address by mapping the physical offset
+ // into offsets in the reserved memory space.
+ // See comment in ZNMT::commit
+ process_fake_mapping(offset, size, false);
+}

src/hotspot/share/gc/z/zNMT.hpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZNMT_HPP
+#define SHARE_GC_Z_ZNMT_HPP
+
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zMemory.hpp"
+#include "gc/z/zVirtualMemory.hpp"
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/nativeCallStack.hpp"
+
+class ZNMT : public AllStatic {
+private:
+ struct Reservation {
+ zaddress_unsafe _start;
+ size_t _size;
+ };
+ static Reservation _reservations[ZMaxVirtualReservations];
+ static size_t _num_reservations;
+
+ static size_t reservation_index(zoffset offset, size_t* offset_in_reservation);
+ static void process_fake_mapping(zoffset offset, size_t size, bool commit);
+
+public:
+ static void reserve(zaddress_unsafe start, size_t size);
+ static void commit(zoffset offset, size_t size);
+ static void uncommit(zoffset offset, size_t size);
+};
+
+#endif // SHARE_GC_Z_ZNMT_HPP

src/hotspot/share/gc/z/zPhysicalMemory.cpp

@@ -27,14 +27,14 @@
 #include "gc/z/zArray.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zLargePages.inline.hpp"
+#include "gc/z/zNMT.hpp"
 #include "gc/z/zNUMA.inline.hpp"
 #include "gc/z/zPhysicalMemory.inline.hpp"
 #include "logging/log.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/globals_extension.hpp"
 #include "runtime/init.hpp"
 #include "runtime/os.hpp"
-#include "services/memTracker.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
@@ -275,26 +275,6 @@ void ZPhysicalMemoryManager::try_enable_uncommit(size_t min_capacity, size_t max
  log_info_p(gc, init)("Uncommit Delay: " UINTX_FORMAT "s", ZUncommitDelay);
 }
 
-void ZPhysicalMemoryManager::nmt_commit(zoffset offset, size_t size) const {
- // NMT expects a 1-to-1 mapping between virtual and physical memory.
- // ZGC can temporarily have multiple virtual addresses pointing to
- // the same physical memory.
- //
- // When this function is called we don't know where in the virtual memory
- // this physical memory will be mapped. So we fake that the virtual memory
- // address is the heap base + the given offset.
- const uintptr_t addr = ZAddressHeapBase + untype(offset);
- MemTracker::record_virtual_memory_commit((void*)addr, size, CALLER_PC);
-}
-
-void ZPhysicalMemoryManager::nmt_uncommit(zoffset offset, size_t size) const {
- if (MemTracker::enabled()) {
- const uintptr_t addr = ZAddressHeapBase + untype(offset);
- Tracker tracker(Tracker::uncommit);
- tracker.record((address)addr, size);
- }
-}
-
 void ZPhysicalMemoryManager::alloc(ZPhysicalMemory& pmem, size_t size) {
  assert(is_aligned(size, ZGranuleSize), "Invalid size");
 
@@ -329,7 +309,7 @@ bool ZPhysicalMemoryManager::commit(ZPhysicalMemory& pmem) {
  const size_t committed = _backing.commit(segment.start(), segment.size());
 
  // Register with NMT
- nmt_commit(segment.start(), committed);
+ ZNMT::commit(segment.start(), committed);
 
  // Register committed segment
  if (!pmem.commit_segment(i, committed)) {
@@ -355,7 +335,7 @@ bool ZPhysicalMemoryManager::uncommit(ZPhysicalMemory& pmem) {
  const size_t uncommitted = _backing.uncommit(segment.start(), segment.size());
 
  // Unregister with NMT
- nmt_uncommit(segment.start(), uncommitted);
+ ZNMT::uncommit(segment.start(), uncommitted);
 
  // Deregister uncommitted segment
  if (!pmem.uncommit_segment(i, uncommitted)) {

src/hotspot/share/gc/z/zPhysicalMemory.hpp

@@ -84,9 +84,6 @@ class ZPhysicalMemoryManager {
  ZPhysicalMemoryBacking _backing;
  ZMemoryManager _manager;
 
- void nmt_commit(zoffset offset, size_t size) const;
- void nmt_uncommit(zoffset offset, size_t size) const;
-
  void pretouch_view(zaddress addr, size_t size) const;
  void map_view(zaddress_unsafe addr, const ZPhysicalMemory& pmem) const;
  void unmap_view(zaddress_unsafe addr, size_t size) const;