0069-RISCV-Codegen-for-atomic-fences-loads-and-stores.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@lowrisc.org>
Subject: [RISCV] Codegen for atomic fences, loads, and stores

---
 lib/Target/RISCV/RISCVISelLowering.cpp  |  17 ++
 lib/Target/RISCV/RISCVISelLowering.h    |   8 +
 lib/Target/RISCV/RISCVInstrInfo.td      |  20 +++
 lib/Target/RISCV/RISCVTargetMachine.cpp |   7 +
 test/CodeGen/RISCV/atomics.ll           | 270 ++++++++++++++++++++++++++++++++
 5 files changed, 322 insertions(+)
 create mode 100644 test/CodeGen/RISCV/atomics.ll

diff --git a/lib/Target/RISCV/RISCVISelLowering.cpp b/lib/Target/RISCV/RISCVISelLowering.cpp
index e024744c88c..a3349ef49fc 100644
--- a/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1200,3 +1200,20 @@ RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
     return 2;
   return getNumRegisters(Context, VT);
 }
+
+Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
+                                                   Instruction *Inst,
+                                                   AtomicOrdering Ord) const {
+  if (Ord == AtomicOrdering::Release ||
+      Ord == AtomicOrdering::SequentiallyConsistent)
+    return Builder.CreateFence(Ord);
+  return nullptr;
+}
+
+Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
+                                                    Instruction *Inst,
+                                                    AtomicOrdering Ord) const {
+  if (Inst->hasAtomicLoad() && isAcquireOrStronger(Ord))
+    return Builder.CreateFence(AtomicOrdering::Acquire);
+  return nullptr;
+}
diff --git a/lib/Target/RISCV/RISCVISelLowering.h b/lib/Target/RISCV/RISCVISelLowering.h
index 3e49c2db7f3..30403f45045 100644
--- a/lib/Target/RISCV/RISCVISelLowering.h
+++ b/lib/Target/RISCV/RISCVISelLowering.h
@@ -57,6 +57,14 @@ public:
   unsigned getNumRegistersForCallingConv(LLVMContext &Context,
                                          EVT VT) const override;
 
+  bool shouldInsertFencesForAtomic(const Instruction *I) const override {
+    return true;
+  }
+  Instruction *emitLeadingFence(IRBuilder<> &Builder, Instruction *Inst,
+                                AtomicOrdering Ord) const override;
+  Instruction *emitTrailingFence(IRBuilder<> &Builder, Instruction *Inst,
+                                 AtomicOrdering Ord) const override;
+
 private:
   void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
                         const SmallVectorImpl<ISD::InputArg> &Ins,
diff --git a/lib/Target/RISCV/RISCVInstrInfo.td b/lib/Target/RISCV/RISCVInstrInfo.td
index cdeef08640c..983732db2c8 100644
--- a/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/lib/Target/RISCV/RISCVInstrInfo.td
@@ -652,6 +652,26 @@ defm : StPat<truncstorei8, SB>;
 defm : StPat<truncstorei16, SH>;
 defm : StPat<store, SW>, Requires<[IsRV32]>;
 
+/// Atomics
+
+// fence acquire -> fence r, rw
+def : Pat<(atomic_fence (i32 4), (imm)), (FENCE 2, 3)>;
+// fence release -> fence rw, r
+def : Pat<(atomic_fence (i32 5), (imm)), (FENCE 3, 1)>;
+// fence acq_rel -> fence rw, rw (but ideally would be fence.tso)
+// TODO: convert to fence.tso (when defined) for fence r, rw + fence rw, r
+def : Pat<(atomic_fence (i32 6), (imm)), (FENCE 3, 3)>;
+// fence seq_cst -> fence rw, rw
+def : Pat<(atomic_fence (i32 7), (imm)), (FENCE 3, 3)>;
+
+defm : LdPat<atomic_load_8, LB>;
+defm : LdPat<atomic_load_16, LH>;
+defm : LdPat<atomic_load_32, LW>;
+
+defm : StPat<atomic_store_8, SB>;
+defm : StPat<atomic_store_16, SH>;
+defm : StPat<atomic_store_32, SW>;
+
 /// Other pseudo-instructions
 
 // Pessimistically assume the stack pointer will be clobbered
diff --git a/lib/Target/RISCV/RISCVTargetMachine.cpp b/lib/Target/RISCV/RISCVTargetMachine.cpp
index e75fb3b701c..7cf90fa66e2 100644
--- a/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -74,6 +74,7 @@ public:
     return getTM<RISCVTargetMachine>();
   }
 
+  void addIRPasses() override;
   bool addInstSelector() override;
   void addPreEmitPass() override;
 };
@@ -83,6 +84,12 @@ TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
   return new RISCVPassConfig(*this, PM);
 }
 
+void RISCVPassConfig::addIRPasses() {
+  addPass(createAtomicExpandPass());
+
+  TargetPassConfig::addIRPasses();
+}
+
 bool RISCVPassConfig::addInstSelector() {
   addPass(createRISCVISelDag(getRISCVTargetMachine()));
 
diff --git a/test/CodeGen/RISCV/atomics.ll b/test/CodeGen/RISCV/atomics.ll
new file mode 100644
index 00000000000..bb81aa9d0eb
--- /dev/null
+++ b/test/CodeGen/RISCV/atomics.ll
@@ -0,0 +1,270 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+
+define void @fence_acquire() nounwind {
+; RV32I-LABEL: fence_acquire:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence r, rw
+; RV32I-NEXT:    ret
+  fence acquire
+  ret void
+}
+
+define void @fence_release() nounwind {
+; RV32I-LABEL: fence_release:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, w
+; RV32I-NEXT:    ret
+  fence release
+  ret void
+}
+
+define void @fence_acq_rel() nounwind {
+; RV32I-LABEL: fence_acq_rel:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, rw
+; RV32I-NEXT:    ret
+  fence acq_rel
+  ret void
+}
+
+define void @fence_seq_cst() nounwind {
+; RV32I-LABEL: fence_seq_cst:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, rw
+; RV32I-NEXT:    ret
+  fence seq_cst
+  ret void
+}
+
+define i8 @atomic_load_i8_unordered(i8 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i8_unordered:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    ret
+  %1 = load atomic i8, i8* %a unordered, align 1
+  ret i8 %1
+}
+
+define i8 @atomic_load_i8_monotonic(i8 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i8_monotonic:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    ret
+  %1 = load atomic i8, i8* %a monotonic, align 1
+  ret i8 %1
+}
+
+define i8 @atomic_load_i8_acquire(i8 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i8_acquire:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    fence r, rw
+; RV32I-NEXT:    ret
+  %1 = load atomic i8, i8* %a acquire, align 1
+  ret i8 %1
+}
+
+define i8 @atomic_load_i8_seq_cst(i8 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i8_seq_cst:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, rw
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    fence r, rw
+; RV32I-NEXT:    ret
+  %1 = load atomic i8, i8* %a seq_cst, align 1
+  ret i8 %1
+}
+
+define i16 @atomic_load_i16_unordered(i16 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i16_unordered:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lh a0, 0(a0)
+; RV32I-NEXT:    ret
+  %1 = load atomic i16, i16* %a unordered, align 2
+  ret i16 %1
+}
+
+define i16 @atomic_load_i16_monotonic(i16 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i16_monotonic:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lh a0, 0(a0)
+; RV32I-NEXT:    ret
+  %1 = load atomic i16, i16* %a monotonic, align 2
+  ret i16 %1
+}
+
+define i16 @atomic_load_i16_acquire(i16 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i16_acquire:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lh a0, 0(a0)
+; RV32I-NEXT:    fence r, rw
+; RV32I-NEXT:    ret
+  %1 = load atomic i16, i16* %a acquire, align 2
+  ret i16 %1
+}
+
+define i16 @atomic_load_i16_seq_cst(i16 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i16_seq_cst:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, rw
+; RV32I-NEXT:    lh a0, 0(a0)
+; RV32I-NEXT:    fence r, rw
+; RV32I-NEXT:    ret
+  %1 = load atomic i16, i16* %a seq_cst, align 2
+  ret i16 %1
+}
+
+define i32 @atomic_load_i32_unordered(i32 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i32_unordered:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    ret
+  %1 = load atomic i32, i32* %a unordered, align 4
+  ret i32 %1
+}
+
+define i32 @atomic_load_i32_monotonic(i32 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i32_monotonic:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    ret
+  %1 = load atomic i32, i32* %a monotonic, align 4
+  ret i32 %1
+}
+
+define i32 @atomic_load_i32_acquire(i32 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i32_acquire:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    fence r, rw
+; RV32I-NEXT:    ret
+  %1 = load atomic i32, i32* %a acquire, align 4
+  ret i32 %1
+}
+
+define i32 @atomic_load_i32_seq_cst(i32 *%a) nounwind {
+; RV32I-LABEL: atomic_load_i32_seq_cst:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, rw
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    fence r, rw
+; RV32I-NEXT:    ret
+  %1 = load atomic i32, i32* %a seq_cst, align 4
+  ret i32 %1
+}
+
+define void @atomic_store_i8_unordered(i8 *%a, i8 %b) nounwind {
+; RV32I-LABEL: atomic_store_i8_unordered:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sb a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i8 %b, i8* %a unordered, align 1
+  ret void
+}
+
+define void @atomic_store_i8_monotonic(i8 *%a, i8 %b) nounwind {
+; RV32I-LABEL: atomic_store_i8_monotonic:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sb a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i8 %b, i8* %a monotonic, align 1
+  ret void
+}
+
+define void @atomic_store_i8_release(i8 *%a, i8 %b) nounwind {
+; RV32I-LABEL: atomic_store_i8_release:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, w
+; RV32I-NEXT:    sb a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i8 %b, i8* %a release, align 1
+  ret void
+}
+
+define void @atomic_store_i8_seq_cst(i8 *%a, i8 %b) nounwind {
+; RV32I-LABEL: atomic_store_i8_seq_cst:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, rw
+; RV32I-NEXT:    sb a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i8 %b, i8* %a seq_cst, align 1
+  ret void
+}
+
+define void @atomic_store_i16_unordered(i16 *%a, i16 %b) nounwind {
+; RV32I-LABEL: atomic_store_i16_unordered:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sh a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i16 %b, i16* %a unordered, align 2
+  ret void
+}
+
+define void @atomic_store_i16_monotonic(i16 *%a, i16 %b) nounwind {
+; RV32I-LABEL: atomic_store_i16_monotonic:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sh a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i16 %b, i16* %a monotonic, align 2
+  ret void
+}
+
+define void @atomic_store_i16_release(i16 *%a, i16 %b) nounwind {
+; RV32I-LABEL: atomic_store_i16_release:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, w
+; RV32I-NEXT:    sh a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i16 %b, i16* %a release, align 2
+  ret void
+}
+
+define void @atomic_store_i16_seq_cst(i16 *%a, i16 %b) nounwind {
+; RV32I-LABEL: atomic_store_i16_seq_cst:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, rw
+; RV32I-NEXT:    sh a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i16 %b, i16* %a seq_cst, align 2
+  ret void
+}
+
+define void @atomic_store_i32_unordered(i32 *%a, i32 %b) nounwind {
+; RV32I-LABEL: atomic_store_i32_unordered:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sw a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i32 %b, i32* %a unordered, align 4
+  ret void
+}
+
+define void @atomic_store_i32_monotonic(i32 *%a, i32 %b) nounwind {
+; RV32I-LABEL: atomic_store_i32_monotonic:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    sw a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i32 %b, i32* %a monotonic, align 4
+  ret void
+}
+
+define void @atomic_store_i32_release(i32 *%a, i32 %b) nounwind {
+; RV32I-LABEL: atomic_store_i32_release:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, w
+; RV32I-NEXT:    sw a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i32 %b, i32* %a release, align 4
+  ret void
+}
+
+define void @atomic_store_i32_seq_cst(i32 *%a, i32 %b) nounwind {
+; RV32I-LABEL: atomic_store_i32_seq_cst:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    fence rw, rw
+; RV32I-NEXT:    sw a0, 0(a1)
+; RV32I-NEXT:    ret
+  store atomic i32 %b, i32* %a seq_cst, align 4
+  ret void
+}
-- 
2.16.2