Merge pull request #1452 from SAP/pr-jdk-21+32

Merge to tag jdk-21+32

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -2944,6 +2944,23 @@ class StubGenerator: public StubCodeGenerator {
  return start;
  }
 
+ // Big-endian 128-bit + 64-bit -> 128-bit addition.
+ // Inputs: 128-bits. in is preserved.
+ // The least-significant 64-bit word is in the upper dword of each vector.
+ // inc (the 64-bit increment) is preserved. Its lower dword must be zero.
+ // Output: result
+ void be_add_128_64(FloatRegister result, FloatRegister in,
+ FloatRegister inc, FloatRegister tmp) {
+ assert_different_registers(result, tmp, inc);
+
+ __ addv(result, __ T2D, in, inc); // Add inc to the least-significant dword of
+ // input
+ __ cm(__ HI, tmp, __ T2D, inc, result);// Check for result overflowing
+ __ ext(tmp, __ T16B, tmp, tmp, 0x08); // Swap LSD of comparison result to MSD and
+ // MSD == 0 (must be!) to LSD
+ __ subv(result, __ T2D, result, tmp); // Subtract -1 from MSD if there was an overflow
+ }
+
  // CTR AES crypt.
  // Arguments:
  //
@@ -3053,13 +3070,16 @@ class StubGenerator: public StubCodeGenerator {
  // Setup the counter
  __ movi(v4, __ T4S, 0);
  __ movi(v5, __ T4S, 1);
- __ ins(v4, __ S, v5, 3, 3); // v4 contains { 0, 0, 0, 1 }
+ __ ins(v4, __ S, v5, 2, 2); // v4 contains { 0, 1 }
 
- __ ld1(v0, __ T16B, counter); // Load the counter into v0
- __ rev32(v16, __ T16B, v0);
- __ addv(v16, __ T4S, v16, v4);
- __ rev32(v16, __ T16B, v16);
- __ st1(v16, __ T16B, counter); // Save the incremented counter back
+ // 128-bit big-endian increment
+ __ ld1(v0, __ T16B, counter);
+ __ rev64(v16, __ T16B, v0);
+ be_add_128_64(v16, v16, v4, /*tmp*/v5);
+ __ rev64(v16, __ T16B, v16);
+ __ st1(v16, __ T16B, counter);
+ // Previous counter value is in v0
+ // v4 contains { 0, 1 }
 
  {
  // We have fewer than bulk_width blocks of data left. Encrypt
@@ -3091,9 +3111,9 @@ class StubGenerator: public StubCodeGenerator {
 
  // Increment the counter, store it back
  __ orr(v0, __ T16B, v16, v16);
- __ rev32(v16, __ T16B, v16);
- __ addv(v16, __ T4S, v16, v4);
- __ rev32(v16, __ T16B, v16);
+ __ rev64(v16, __ T16B, v16);
+ be_add_128_64(v16, v16, v4, /*tmp*/v5);
+ __ rev64(v16, __ T16B, v16);
  __ st1(v16, __ T16B, counter); // Save the incremented counter back
 
  __ b(inner_loop);
@@ -3141,7 +3161,7 @@ class StubGenerator: public StubCodeGenerator {
  // Keys should already be loaded into the correct registers
 
  __ ld1(v0, __ T16B, counter); // v0 contains the first counter
- __ rev32(v16, __ T16B, v0); // v16 contains byte-reversed counter
+ __ rev64(v16, __ T16B, v0); // v16 contains byte-reversed counter
 
  // AES/CTR loop
  {
@@ -3151,12 +3171,12 @@ class StubGenerator: public StubCodeGenerator {
  // Setup the counters
  __ movi(v8, __ T4S, 0);
  __ movi(v9, __ T4S, 1);
- __ ins(v8, __ S, v9, 3, 3); // v8 contains { 0, 0, 0, 1 }
+ __ ins(v8, __ S, v9, 2, 2); // v8 contains { 0, 1 }
 
  for (int i = 0; i < bulk_width; i++) {
  FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
- __ rev32(v0_ofs, __ T16B, v16);
- __ addv(v16, __ T4S, v16, v8);
+ __ rev64(v0_ofs, __ T16B, v16);
+ be_add_128_64(v16, v16, v8, /*tmp*/v9);
  }
 
  __ ld1(v8, v9, v10, v11, __ T16B, __ post(in, 4 * 16));
@@ -3186,7 +3206,7 @@ class StubGenerator: public StubCodeGenerator {
  }
 
  // Save the counter back where it goes
- __ rev32(v16, __ T16B, v16);
+ __ rev64(v16, __ T16B, v16);
  __ st1(v16, __ T16B, counter);
 
  __ pop(saved_regs, sp);
@@ -7221,7 +7241,6 @@ class StubGenerator: public StubCodeGenerator {
  // The handle is dereferenced through a load barrier.
  static void jfr_epilogue(MacroAssembler* _masm) {
  __ reset_last_Java_frame(true);
- __ resolve_global_jobject(r0, rscratch1, rscratch2);
  }
 
  // For c2: c_rarg0 is junk, call to runtime to write a checkpoint.
@@ -7250,6 +7269,7 @@ class StubGenerator: public StubCodeGenerator {
  jfr_prologue(the_pc, _masm, rthread);
  __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), 1);
  jfr_epilogue(_masm);
+ __ resolve_global_jobject(r0, rscratch1, rscratch2);
  __ leave();
  __ ret(lr);
 
@@ -7263,6 +7283,44 @@ class StubGenerator: public StubCodeGenerator {
  return stub;
  }
 
+ // For c2: call to return a leased buffer.
+ static RuntimeStub* generate_jfr_return_lease() {
+ enum layout {
+ rbp_off,
+ rbpH_off,
+ return_off,
+ return_off2,
+ framesize // inclusive of return address
+ };
+
+ int insts_size = 1024;
+ int locs_size = 64;
+ CodeBuffer code("jfr_return_lease", insts_size, locs_size);
+ OopMapSet* oop_maps = new OopMapSet();
+ MacroAssembler* masm = new MacroAssembler(&code);
+ MacroAssembler* _masm = masm;
+
+ address start = __ pc();
+ __ enter();
+ int frame_complete = __ pc() - start;
+ address the_pc = __ pc();
+ jfr_prologue(the_pc, _masm, rthread);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), 1);
+ jfr_epilogue(_masm);
+
+ __ leave();
+ __ ret(lr);
+
+ OopMap* map = new OopMap(framesize, 1); // rfp
+ oop_maps->add_gc_map(the_pc - start, map);
+
+ RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size)
+ RuntimeStub::new_runtime_stub("jfr_return_lease", &code, frame_complete,
+ (framesize >> (LogBytesPerWord - LogBytesPerInt)),
+ oop_maps, false);
+ return stub;
+ }
+
 #endif // INCLUDE_JFR
 
  // Continuation point for throwing of implicit exceptions that are
@@ -8261,10 +8319,18 @@ class StubGenerator: public StubCodeGenerator {
  StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
  StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
 
- JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();)
- JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();)
+ JFR_ONLY(generate_jfr_stubs();)
  }
 
+#if INCLUDE_JFR
+ void generate_jfr_stubs() {
+ StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();
+ StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();
+ StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease();
+ StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point();
+ }
+#endif // INCLUDE_JFR
+
  void generate_final_stubs() {
  // support for verify_oop (must happen after universe_init)
  if (VerifyOops) {

src/hotspot/cpu/arm/stubGenerator_arm.cpp

@@ -3072,6 +3072,46 @@ class StubGenerator: public StubCodeGenerator {
  return stub;
  }
 
+ // For c2: call to return a leased buffer.
+ static RuntimeStub* generate_jfr_return_lease() {
+ enum layout {
+ r1_off,
+ r2_off,
+ return_off,
+ framesize // inclusive of return address
+ };
+
+ CodeBuffer code("jfr_return_lease", 512, 64);
+ MacroAssembler* masm = new MacroAssembler(&code);
+
+ address start = __ pc();
+ __ raw_push(R1, R2, LR);
+ address the_pc = __ pc();
+
+ int frame_complete = the_pc - start;
+
+ __ set_last_Java_frame(SP, FP, true, Rtemp);
+ __ mov(c_rarg0, Rthread);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), c_rarg0);
+ __ reset_last_Java_frame(Rtemp);
+
+ __ raw_pop(R1, R2, LR);
+ __ ret();
+
+ OopMapSet* oop_maps = new OopMapSet();
+ OopMap* map = new OopMap(framesize, 1);
+ oop_maps->add_gc_map(frame_complete, map);
+
+ RuntimeStub* stub =
+ RuntimeStub::new_runtime_stub(code.name(),
+ &code,
+ frame_complete,
+ (framesize >> (LogBytesPerWord - LogBytesPerInt)),
+ oop_maps,
+ false);
+ return stub;
+ }
+
 #endif // INCLUDE_JFR
 
  //---------------------------------------------------------------------------
@@ -3116,10 +3156,18 @@ class StubGenerator: public StubCodeGenerator {
  StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
  StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
 
- JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();)
- JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();)
+ JFR_ONLY(generate_jfr_stubs();)
  }
 
+#if INCLUDE_JFR
+ void generate_jfr_stubs() {
+ StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();
+ StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();
+ StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease();
+ StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point();
+ }
+#endif // INCLUDE_JFR
+
  void generate_final_stubs() {
  // Generates all stubs and initializes the entry points
 

src/hotspot/cpu/ppc/stubGenerator_ppc.cpp

@@ -4680,6 +4680,41 @@ class StubGenerator: public StubCodeGenerator {
  return stub;
  }
 
+ // For c2: call to return a leased buffer.
+ RuntimeStub* generate_jfr_return_lease() {
+ CodeBuffer code("jfr_return_lease", 512, 64);
+ MacroAssembler* _masm = new MacroAssembler(&code);
+
+ Register tmp1 = R10_ARG8;
+ Register tmp2 = R9_ARG7;
+
+ int framesize = frame::native_abi_reg_args_size / VMRegImpl::stack_slot_size;
+ address start = __ pc();
+ __ mflr(tmp1);
+ __ std(tmp1, _abi0(lr), R1_SP); // save return pc
+ __ push_frame_reg_args(0, tmp1);
+ int frame_complete = __ pc() - start;
+ __ set_last_Java_frame(R1_SP, noreg);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), R16_thread);
+ address calls_return_pc = __ last_calls_return_pc();
+ __ reset_last_Java_frame();
+ __ pop_frame();
+ __ ld(tmp1, _abi0(lr), R1_SP);
+ __ mtlr(tmp1);
+ __ blr();
+
+ OopMapSet* oop_maps = new OopMapSet();
+ OopMap* map = new OopMap(framesize, 0);
+ oop_maps->add_gc_map(calls_return_pc - start, map);
+
+ RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size)
+ RuntimeStub::new_runtime_stub(code.name(),
+ &code, frame_complete,
+ (framesize >> (LogBytesPerWord - LogBytesPerInt)),
+ oop_maps, false);
+ return stub;
+ }
+
 #endif // INCLUDE_JFR
 
 
@@ -4728,10 +4763,18 @@ class StubGenerator: public StubCodeGenerator {
  StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
  StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
 
- JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();)
- JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();)
+ JFR_ONLY(generate_jfr_stubs();)
  }
 
+#if INCLUDE_JFR
+ void generate_jfr_stubs() {
+ StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();
+ StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();
+ StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease();
+ StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point();
+ }
+#endif // INCLUDE_JFR
+
  void generate_final_stubs() {
  // Generates all stubs and initializes the entry points
 

src/hotspot/cpu/riscv/icache_riscv.cpp

@@ -1,6 +1,7 @@
 /*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2023, Rivos Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +26,8 @@
 
 #include "precompiled.hpp"
 #include "asm/macroAssembler.hpp"
+#include "riscv_flush_icache.hpp"
+#include "runtime/java.hpp"
 #include "runtime/icache.hpp"
 
 #define __ _masm->
@@ -33,16 +36,22 @@ static int icache_flush(address addr, int lines, int magic) {
  // To make a store to instruction memory visible to all RISC-V harts,
  // the writing hart has to execute a data FENCE before requesting that
  // all remote RISC-V harts execute a FENCE.I.
- //
- // No sush assurance is defined at the interface level of the builtin
- // method, and so we should make sure it works.
+
+ // We need to make sure stores happens before the I/D cache synchronization.
  __asm__ volatile("fence rw, rw" : : : "memory");
 
- __builtin___clear_cache(addr, addr + (lines << ICache::log2_line_size));
+ RiscvFlushIcache::flush((uintptr_t)addr, ((uintptr_t)lines) << ICache::log2_line_size);
+
  return magic;
 }
 
 void ICacheStubGenerator::generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub) {
+ // Only riscv_flush_icache is supported as I-cache synchronization.
+ // We must make sure the VM can execute such without error.
+ if (!RiscvFlushIcache::test()) {
+ vm_exit_during_initialization("Unable to synchronize I-cache");
+ }
+
  address start = (address)icache_flush;
  *flush_icache_stub = (ICache::flush_icache_stub_t)start;