Merge upstream-jdk

src/hotspot/cpu/riscv/assembler_riscv.hpp

@@ -3070,42 +3070,63 @@ enum Nf {
 #undef INSN
 
 // Cache Management Operations
-#define INSN(NAME, funct)                                                                    \
-  void NAME(Register Rs1) {                                                                  \
-    unsigned insn = 0;                                                                       \
-    patch((address)&insn, 6,  0, 0b0001111);                                                 \
-    patch((address)&insn, 14, 12, 0b010);                                                    \
-    patch_reg((address)&insn, 15, Rs1);                                                      \
-    patch((address)&insn, 31, 20, funct);                                                    \
-    emit(insn);                                                                              \
+// These instruction may be turned off for user space.
+ private:
+  enum CBO_FUNCT : unsigned int {
+    CBO_INVAL = 0b0000000000000,
+    CBO_CLEAN = 0b0000000000001,
+    CBO_FLUSH = 0b0000000000010,
+    CBO_ZERO  = 0b0000000000100
+  };
+
+  template <CBO_FUNCT FUNCT>
+  void cbo_base(Register Rs1) {
+    assert((UseZicbom && FUNCT != CBO_ZERO) || UseZicboz, "sanity");
+    unsigned insn = 0;
+    patch((address)&insn, 6,  0, 0b0001111);
+    patch((address)&insn, 14, 12, 0b010);
+    patch_reg((address)&insn, 15, Rs1);
+    patch((address)&insn, 31, 20, FUNCT);
+    emit(insn);
   }
 
-  INSN(cbo_inval, 0b0000000000000);
-  INSN(cbo_clean, 0b0000000000001);
-  INSN(cbo_flush, 0b0000000000010);
-  INSN(cbo_zero,  0b0000000000100);
+  // This instruction have some security implication.
+  // At this time it's not likely to be enabled for user mode.
+  void cbo_inval(Register Rs1) { cbo_base<CBO_INVAL>(Rs1); }
+ public:
+  // Zicbom
+  void cbo_clean(Register Rs1) { cbo_base<CBO_CLEAN>(Rs1); }
+  void cbo_flush(Register Rs1) { cbo_base<CBO_FLUSH>(Rs1); }
+  // Zicboz
+  void cbo_zero(Register Rs1)  { cbo_base<CBO_ZERO>(Rs1); }
 
-#undef INSN
+ private:
+  enum PREFETCH_FUNCT : unsigned int {
+    PREFETCH_I = 0b0000000000000,
+    PREFETCH_R = 0b0000000000001,
+    PREFETCH_W = 0b0000000000011
+  };
 
-#define INSN(NAME, funct)                                                                    \
-  void NAME(Register Rs1, int32_t offset) {                                                  \
-    guarantee((offset & 0x1f) == 0, "offset lowest 5 bits must be zero");                    \
-    int32_t upperOffset = offset >> 5;                                                       \
-    unsigned insn = 0;                                                                       \
-    patch((address)&insn, 6,  0, 0b0010011);                                                 \
-    patch((address)&insn, 14, 12, 0b110);                                                    \
-    patch_reg((address)&insn, 15, Rs1);                                                      \
-    patch((address)&insn, 24, 20, funct);                                                    \
-    upperOffset &= 0x7f;                                                                     \
-    patch((address)&insn, 31, 25, upperOffset);                                              \
-    emit(insn);                                                                              \
+  template <PREFETCH_FUNCT FUNCT>
+  void prefetch_base(Register Rs1, int32_t offset) {
+    assert_cond(UseZicbop);
+    guarantee((offset & 0x1f) == 0, "offset lowest 5 bits must be zero");
+    int32_t upperOffset = offset >> 5;
+    unsigned insn = 0;
+    patch((address)&insn, 6,  0, 0b0010011);
+    patch((address)&insn, 14, 12, 0b110);
+    patch_reg((address)&insn, 15, Rs1);
+    patch((address)&insn, 24, 20, FUNCT);
+    upperOffset &= 0x7f;
+    patch((address)&insn, 31, 25, upperOffset);
+    emit(insn);
   }
 
-  INSN(prefetch_i, 0b0000000000000);
-  INSN(prefetch_r, 0b0000000000001);
-  INSN(prefetch_w, 0b0000000000011);
-
-#undef INSN
+ public:
+  // Zicbop
+  void prefetch_i(Register Rs1, int32_t offset) { prefetch_base<PREFETCH_I>(Rs1, offset); }
+  void prefetch_r(Register Rs1, int32_t offset) { prefetch_base<PREFETCH_R>(Rs1, offset); }
+  void prefetch_w(Register Rs1, int32_t offset) { prefetch_base<PREFETCH_W>(Rs1, offset); }
 
 // --------------  Zicond Instruction Definitions  --------------
 // Zicond conditional operations extension

src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@@ -1333,22 +1333,11 @@ void MacroAssembler::cmov_gtu(Register cmp1, Register cmp2, Register dst, Regist
 
 #undef INSN
 
-
-#define INSN(NAME, CSR)                       \
-  void MacroAssembler::NAME(Register Rd) {    \
-    csrr(Rd, CSR);                            \
-  }
-
-  INSN(rdinstret,  CSR_INSTRET);
-  INSN(rdcycle,    CSR_CYCLE);
-  INSN(rdtime,     CSR_TIME);
-  INSN(frcsr,      CSR_FCSR);
-  INSN(frrm,       CSR_FRM);
-  INSN(frflags,    CSR_FFLAGS);
-
-#undef INSN
-
 void MacroAssembler::csrr(Register Rd, unsigned csr) {
+  // These three are specified in zicntr and are unused.
+  // Before adding use-cases add the appropriate hwprobe and flag.
+  assert(csr != CSR_INSTRET && csr != CSR_CYCLE && csr != CSR_TIME,
+         "Not intended for use without enabling zicntr.");
   csrrs(Rd, csr, x0);
 }
 

src/hotspot/cpu/riscv/macroAssembler_riscv.hpp

@@ -626,29 +626,30 @@ class MacroAssembler: public Assembler {
   }
 
   // Control and status pseudo instructions
-  void rdinstret(Register Rd);                  // read instruction-retired counter
-  void rdcycle(Register Rd);                    // read cycle counter
-  void rdtime(Register Rd);                     // read time
   void csrr(Register Rd, unsigned csr);         // read csr
   void csrw(unsigned csr, Register Rs);         // write csr
   void csrs(unsigned csr, Register Rs);         // set bits in csr
   void csrc(unsigned csr, Register Rs);         // clear bits in csr
   void csrwi(unsigned csr, unsigned imm);
   void csrsi(unsigned csr, unsigned imm);
   void csrci(unsigned csr, unsigned imm);
-  void frcsr(Register Rd);                      // read float-point csr
-  void fscsr(Register Rd, Register Rs);         // swap float-point csr
-  void fscsr(Register Rs);                      // write float-point csr
-  void frrm(Register Rd);                       // read float-point rounding mode
-  void fsrm(Register Rd, Register Rs);          // swap float-point rounding mode
-  void fsrm(Register Rs);                       // write float-point rounding mode
+  void frcsr(Register Rd) { csrr(Rd, CSR_FCSR); }; // read float-point csr
+  void fscsr(Register Rd, Register Rs);            // swap float-point csr
+  void fscsr(Register Rs);                         // write float-point csr
+  void frrm(Register Rd) { csrr(Rd, CSR_FRM); };   // read float-point rounding mode
+  void fsrm(Register Rd, Register Rs);             // swap float-point rounding mode
+  void fsrm(Register Rs);                          // write float-point rounding mode
   void fsrmi(Register Rd, unsigned imm);
   void fsrmi(unsigned imm);
-  void frflags(Register Rd);                    // read float-point exception flags
-  void fsflags(Register Rd, Register Rs);       // swap float-point exception flags
-  void fsflags(Register Rs);                    // write float-point exception flags
+  void frflags(Register Rd) { csrr(Rd, CSR_FFLAGS); }; // read float-point exception flags
+  void fsflags(Register Rd, Register Rs);              // swap float-point exception flags
+  void fsflags(Register Rs);                           // write float-point exception flags
   void fsflagsi(Register Rd, unsigned imm);
   void fsflagsi(unsigned imm);
+  // Requires Zicntr
+  void rdinstret(Register Rd) { csrr(Rd, CSR_INSTRET); }; // read instruction-retired counter
+  void rdcycle(Register Rd)   { csrr(Rd, CSR_CYCLE); };   // read cycle counter
+  void rdtime(Register Rd)    { csrr(Rd, CSR_TIME); };    // read time
 
   // Restore cpu control state after JNI call
   void restore_cpu_control_state_after_jni(Register tmp);

src/hotspot/cpu/riscv/vm_version_riscv.hpp

@@ -158,6 +158,7 @@ class VM_Version : public Abstract_VM_Version {
   decl(ext_Zcb         , "Zcb"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZcb))         \
   decl(ext_Zfh         , "Zfh"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfh))         \
   decl(ext_Zicsr       , "Zicsr"       , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)              \
+  decl(ext_Zicntr      , "Zicntr"      , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)              \
   decl(ext_Zifencei    , "Zifencei"    , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)              \
   decl(ext_Zic64b      , "Zic64b"      , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b))      \
   decl(ext_Ztso        , "Ztso"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso))        \

src/hotspot/os/posix/perfMemory_posix.cpp

@@ -1086,7 +1086,7 @@ static char* mmap_create_shared(size_t size) {
 static void unmap_shared(char* addr, size_t bytes) {
   int res;
   if (MemTracker::enabled()) {
-    ThreadCritical tc;
+    MemTracker::NmtVirtualMemoryLocker nvml;
     res = ::munmap(addr, bytes);
     if (res == 0) {
       MemTracker::record_virtual_memory_release((address)addr, bytes);

src/hotspot/os/windows/os_windows.cpp

@@ -3639,10 +3639,7 @@ bool os::pd_release_memory(char* addr, size_t bytes) {
     // Handle mapping error. We assert in debug, unconditionally print a warning in release.
     if (err != nullptr) {
       log_warning(os)("bad release: [" PTR_FORMAT "-" PTR_FORMAT "): %s", p2i(start), p2i(end), err);
-#ifdef ASSERT
-      os::print_memory_mappings((char*)start, bytes, tty);
       assert(false, "bad release: [" PTR_FORMAT "-" PTR_FORMAT "): %s", p2i(start), p2i(end), err);
-#endif
       return false;
     }
     // Free this range

src/hotspot/os/windows/perfMemory_windows.cpp

@@ -1803,7 +1803,7 @@ void PerfMemory::detach(char* addr, size_t bytes) {
 
   if (MemTracker::enabled()) {
     // it does not go through os api, the operation has to record from here
-    ThreadCritical tc;
+    MemTracker::NmtVirtualMemoryLocker nvml;
     remove_file_mapping(addr);
     MemTracker::record_virtual_memory_release((address)addr, bytes);
   } else {

src/hotspot/share/nmt/memBaseline.cpp

@@ -141,7 +141,7 @@ void MemBaseline::baseline_summary() {
   MallocMemorySummary::snapshot(&_malloc_memory_snapshot);
   VirtualMemorySummary::snapshot(&_virtual_memory_snapshot);
   {
-    MemoryFileTracker::Instance::Locker lock;
+    MemTracker::NmtVirtualMemoryLocker nvml;
     MemoryFileTracker::Instance::summary_snapshot(&_virtual_memory_snapshot);
   }
 

src/hotspot/share/nmt/memReporter.cpp

@@ -28,6 +28,7 @@
 #include "nmt/mallocTracker.hpp"
 #include "nmt/memTag.hpp"
 #include "nmt/memReporter.hpp"
+#include "nmt/memTracker.hpp"
 #include "nmt/memoryFileTracker.hpp"
 #include "nmt/threadStackTracker.hpp"
 #include "nmt/virtualMemoryTracker.hpp"
@@ -465,7 +466,7 @@ void MemDetailReporter::report_virtual_memory_region(const ReservedMemoryRegion*
 void MemDetailReporter::report_memory_file_allocations() {
   stringStream st;
   {
-    MemoryFileTracker::Instance::Locker lock;
+    MemTracker::NmtVirtualMemoryLocker nvml;
     MemoryFileTracker::Instance::print_all_reports_on(&st, scale());
   }
   output()->print_raw(st.freeze());

src/hotspot/share/nmt/memTracker.cpp

@@ -52,6 +52,8 @@ NMT_TrackingLevel MemTracker::_tracking_level = NMT_unknown;
 
 MemBaseline MemTracker::_baseline;
 
+bool MemTracker::NmtVirtualMemoryLocker::_safe_to_use;
+
 void MemTracker::initialize() {
   bool rc = true;
   assert(_tracking_level == NMT_unknown, "only call once");

src/hotspot/share/nmt/memTracker.hpp

@@ -31,7 +31,6 @@
 #include "nmt/threadStackTracker.hpp"
 #include "nmt/virtualMemoryTracker.hpp"
 #include "runtime/mutexLocker.hpp"
-#include "runtime/threadCritical.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/nativeCallStack.hpp"
 
@@ -62,6 +61,12 @@ class MemTracker : AllStatic {
     return _tracking_level != NMT_unknown;
   }
 
+  // This may be called on a detached thread during VM init, so we should check that first.
+  static inline void assert_locked() {
+    assert(!NmtVirtualMemoryLocker::is_safe_to_use() || NmtVirtualMemory_lock->owned_by_self(),
+           "should have acquired NmtVirtualMemory_lock");
+  }
+
   static inline NMT_TrackingLevel tracking_level() {
     return _tracking_level;
   }
@@ -125,7 +130,7 @@ class MemTracker : AllStatic {
     assert_post_init();
     if (!enabled()) return;
     if (addr != nullptr) {
-      ThreadCritical tc;
+      NmtVirtualMemoryLocker nvml;
       VirtualMemoryTracker::add_reserved_region((address)addr, size, stack, mem_tag);
     }
   }
@@ -151,7 +156,7 @@ class MemTracker : AllStatic {
     assert_post_init();
     if (!enabled()) return;
     if (addr != nullptr) {
-      ThreadCritical tc;
+      NmtVirtualMemoryLocker nvml;
       VirtualMemoryTracker::add_reserved_region((address)addr, size, stack, mem_tag);
       VirtualMemoryTracker::add_committed_region((address)addr, size, stack);
     }
@@ -162,23 +167,23 @@ class MemTracker : AllStatic {
     assert_post_init();
     if (!enabled()) return;
     if (addr != nullptr) {
-      ThreadCritical tc;
+      NmtVirtualMemoryLocker nvml;
       VirtualMemoryTracker::add_committed_region((address)addr, size, stack);
     }
   }
 
   static inline MemoryFileTracker::MemoryFile* register_file(const char* descriptive_name) {
     assert_post_init();
     if (!enabled()) return nullptr;
-    MemoryFileTracker::Instance::Locker lock;
+    NmtVirtualMemoryLocker nvml;
     return MemoryFileTracker::Instance::make_file(descriptive_name);
   }
 
   static inline void remove_file(MemoryFileTracker::MemoryFile* file) {
     assert_post_init();
     if (!enabled()) return;
     assert(file != nullptr, "must be");
-    MemoryFileTracker::Instance::Locker lock;
+    NmtVirtualMemoryLocker nvml;
     MemoryFileTracker::Instance::free_file(file);
   }
 
@@ -187,7 +192,7 @@ class MemTracker : AllStatic {
     assert_post_init();
     if (!enabled()) return;
     assert(file != nullptr, "must be");
-    MemoryFileTracker::Instance::Locker lock;
+    NmtVirtualMemoryLocker nvml;
     MemoryFileTracker::Instance::allocate_memory(file, offset, size, stack, mem_tag);
   }
 
@@ -196,7 +201,7 @@ class MemTracker : AllStatic {
     assert_post_init();
     if (!enabled()) return;
     assert(file != nullptr, "must be");
-    MemoryFileTracker::Instance::Locker lock;
+    NmtVirtualMemoryLocker nvml;
     MemoryFileTracker::Instance::free_memory(file, offset, size);
   }
 
@@ -210,7 +215,7 @@ class MemTracker : AllStatic {
     assert_post_init();
     if (!enabled()) return;
     if (addr != nullptr) {
-      ThreadCritical tc;
+      NmtVirtualMemoryLocker nvml;
       VirtualMemoryTracker::split_reserved_region((address)addr, size, split, mem_tag, split_tag);
     }
   }
@@ -219,7 +224,7 @@ class MemTracker : AllStatic {
     assert_post_init();
     if (!enabled()) return;
     if (addr != nullptr) {
-      ThreadCritical tc;
+      NmtVirtualMemoryLocker nvml;
       VirtualMemoryTracker::set_reserved_region_type((address)addr, mem_tag);
     }
   }
@@ -269,6 +274,39 @@ class MemTracker : AllStatic {
   // and return true; false if not found.
   static bool print_containing_region(const void* p, outputStream* out);
 
+  /*
+   * NmtVirtualMemoryLocker is similar to MutexLocker but can be used during VM init before mutexes are ready or
+   * current thread has been assigned. Performs no action during VM init.
+   *
+   * Unlike malloc, NMT requires locking for virtual memory operations. This is because it must synchronize the usage
+   * of global data structures used for modelling the effect of virtual memory operations.
+   * It is important that locking is used such that the actual OS memory operations (mmap) are done atomically with the
+   * corresponding NMT accounting (updating the internal model). Currently, this is not the case in all situations
+   * (see JDK-8341491), but this should be changed in the future.
+   *
+   * An issue with using Mutex is that NMT is used early during VM initialization before mutexes are initialized
+   * and current thread is attached. Mutexes do not work under those conditions, so we must use a flag to avoid
+   * attempting to lock until initialization is finished. Lack of synchronization here should not be a problem since it
+   * is single threaded at that point in time anyway.
+   */
+  class NmtVirtualMemoryLocker: StackObj {
+    // Returns true if it is safe to start using this locker.
+    static bool _safe_to_use;
+    ConditionalMutexLocker _cml;
+
+  public:
+    NmtVirtualMemoryLocker(): _cml(NmtVirtualMemory_lock, _safe_to_use, Mutex::_no_safepoint_check_flag){}
+
+    static inline bool is_safe_to_use()  {
+      return _safe_to_use;
+    }
+
+    // Set in Threads::create_vm once threads and mutexes have been initialized.
+    static inline void set_safe_to_use()  {
+      _safe_to_use = true;
+    }
+  };
+
  private:
   static void report(bool summary_only, outputStream* output, size_t scale);
 
@@ -277,8 +315,6 @@ class MemTracker : AllStatic {
   static NMT_TrackingLevel   _tracking_level;
   // Stored baseline
   static MemBaseline      _baseline;
-  // Query lock
-  static Mutex*           _query_lock;
 };
 
 #endif // SHARE_NMT_MEMTRACKER_HPP