From edab0150db749add93db712b57665139490d9e5b Mon Sep 17 00:00:00 2001
From: Pu Wang <nicolas.weeks@gmail.com>
Date: Wed, 13 Jul 2022 23:22:35 +0800
Subject: [PATCH] RQ e2e fatal NAK

---
 rocev2/src/rdma/Bus.scala                   |   2 +-
 rocev2/src/rdma/Constants.scala             |   7 +-
 rocev2/src/rdma/RecvQ.scala                 | 107 ++--
 rocev2/src/rdma/RespHandler.scala           |   2 +-
 rocev2/src/rdma/Utils.scala                 |  33 +-
 rocev2/test/src/rdma/CacheTest.scala        |   4 +-
 rocev2/test/src/rdma/DataStructSim.scala    | 322 +++++++----
 rocev2/test/src/rdma/QPTest.scala           |   4 +-
 rocev2/test/src/rdma/RecvQTest.scala        | 590 ++++++++++++++------
 rocev2/test/src/rdma/RespHandlerTest.scala  |   2 +-
 rocev2/test/src/rdma/RetryHandlerTest.scala |   2 +-
 rocev2/test/src/rdma/SendQTest.scala        |   8 +-
 run.sh                                      |  10 +-
 13 files changed, 749 insertions(+), 344 deletions(-)

diff --git a/rocev2/src/rdma/Bus.scala b/rocev2/src/rdma/Bus.scala
index 664bcd2..b93d1a0 100644
--- a/rocev2/src/rdma/Bus.scala
+++ b/rocev2/src/rdma/Bus.scala
@@ -1748,7 +1748,7 @@ case class WorkComp() extends Bundle {
   def setOpCodeFromRqReqOpCode(reqOpCode: Bits): this.type = {
     when(OpCode.isSendReqPkt(reqOpCode)) {
       opcode := WorkCompOpCode.RECV
-    } elsewhen (OpCode.isWriteWithImmReqPkt(reqOpCode)) {
+    } elsewhen (OpCode.isWriteImmReqPkt(reqOpCode)) {
       opcode := WorkCompOpCode.RECV_RDMA_WITH_IMM
     } otherwise {
       report(
diff --git a/rocev2/src/rdma/Constants.scala b/rocev2/src/rdma/Constants.scala
index e180ac0..30165b2 100644
--- a/rocev2/src/rdma/Constants.scala
+++ b/rocev2/src/rdma/Constants.scala
@@ -949,7 +949,7 @@ object OpCode extends Enumeration {
       }
     }.result
 
-  def isWriteWithImmReqPkt(opcode: Bits): Bool =
+  def isWriteImmReqPkt(opcode: Bits): Bool =
     new Composite(opcode, "OpCode_isWriteWithImmReqPkt") {
       val result = Bool()
       switch(opcode) {
@@ -965,6 +965,11 @@ object OpCode extends Enumeration {
       }
     }.result
 
+  def needRxBuf(opcode: Bits): Bool =
+    new Composite(opcode, "OpCode_needRxBuf") {
+      val result = isSendReqPkt(opcode) || isWriteImmReqPkt(opcode)
+    }.result
+
   def hasImmDt(opcode: Bits): Bool =
     new Composite(opcode, "OpCode_hasImmDt") {
       val result = Bool()
diff --git a/rocev2/src/rdma/RecvQ.scala b/rocev2/src/rdma/RecvQ.scala
index 0e4d839..da968b0 100644
--- a/rocev2/src/rdma/RecvQ.scala
+++ b/rocev2/src/rdma/RecvQ.scala
@@ -1,6 +1,7 @@
 package rdma
 
 import spinal.core._
+import spinal.core.formal._
 import spinal.lib._
 import RdmaConstants._
 import ConstantSettings._
@@ -204,7 +205,6 @@ class ReqCommCheck(busWidth: BusWidth.Value) extends Component {
   val io = new Bundle {
     val qpAttr = in(QpAttrData())
     val epsnInc = out(EPsnInc())
-//    val nakNotifier = out(NakNotifier())
     val clearRnrOrNakSeq = out(RetryNakClear())
     val flush = in(Bool())
     val rx = slave(RdmaDataBus(busWidth))
@@ -400,7 +400,7 @@ class ReqCommCheck(busWidth: BusWidth.Value) extends Component {
     io.epsnInc.incVal := ePsnIncrease(inputPktFrag, io.qpAttr.pmtu)
     io.epsnInc.preReqOpCode := inputPktFrag.bth.opcode
 
-    io.seqErrNotifier.pulse := input.lastFire && isSeqErr
+    io.seqErrNotifier.pulse := retryNakTriggerCond(input.lastFire, isSeqErr)
     io.seqErrNotifier.psn := inputPktFrag.bth.psn
     io.seqErrNotifier.preOpCode := io.qpAttr.rqPreReqOpCode
   }
@@ -425,10 +425,10 @@ class ReqRnrCheck(busWidth: BusWidth.Value) extends Component {
 
   val isSendOrWriteImmReq =
     OpCode.isSendReqPkt(inputPktFrag.bth.opcode) ||
-      OpCode.isWriteWithImmReqPkt(inputPktFrag.bth.opcode)
+      OpCode.isWriteImmReqPkt(inputPktFrag.bth.opcode)
   val isLastOrOnlySendOrWriteImmReq =
     OpCode.isSendLastOrOnlyReqPkt(inputPktFrag.bth.opcode) ||
-      OpCode.isWriteWithImmReqPkt(inputPktFrag.bth.opcode)
+      OpCode.isWriteImmReqPkt(inputPktFrag.bth.opcode)
 
   // RNR check for send/write
   val needRxBuf =
@@ -488,7 +488,10 @@ class ReqRnrCheck(busWidth: BusWidth.Value) extends Component {
       result
     }
 
-  io.rnrNotifier.pulse := io.rx.checkRst.lastFire && hasRnrErr
+  io.rnrNotifier.pulse := retryNakTriggerCond(
+    io.rx.checkRst.lastFire,
+    hasRnrErr
+  )
   io.rnrNotifier.psn := io.rx.checkRst.pktFrag.bth.psn
   io.rnrNotifier.preOpCode := io.rx.checkRst.preOpCode
 }
@@ -586,6 +589,7 @@ class DupReqHandlerAndReadAtomicRstCacheQuery(
     io.readAtomicRstCache.req,
     io.readAtomicRstCache.resp,
     waitQueueDepth = READ_ATOMIC_RESULT_CACHE_QUERY_FIFO_DEPTH,
+    waitQueueName = "dupReadAtomicReqQueryRstCacheWaitQueue",
     buildQuery = buildReadAtomicRstCacheQuery,
     queryCond = readAtomicRstCacheQueryCond,
     expectResp = expectReadAtomicRstCacheResp,
@@ -940,10 +944,14 @@ class ReqAddrInfoExtractor(busWidth: BusWidth.Value) extends Component {
       }.result
   )
 
-  val ackReq =
-    ((isSendReq || isWriteReq) && io.rx.reqWithRxBuf.pktFrag.bth.ackreq) || isReadReq || isAtomicReq
-  val rqOutPsnRangeFifoPushCond =
-    io.rx.reqWithRxBuf.isFirst && (io.rx.reqWithRxBuf.hasNak || ackReq)
+//  val ackReq =
+//    ((isSendReq || isWriteReq) && io.rx.reqWithRxBuf.pktFrag.bth.ackreq) || isReadReq || isAtomicReq
+  val expectResp = rqGenRespCond(
+    io.rx.reqWithRxBuf.pktFrag.bth,
+    io.rx.reqWithRxBuf.hasNak,
+    io.rx.reqWithRxBuf.ackAeth
+  )
+  val rqOutPsnRangeFifoPushCond = io.rx.reqWithRxBuf.isFirst && expectResp
   val (rqOutPsnRangeFifoPush, txNormal) = StreamConditionalFork2(
     // Not flush when RNR
     inputStreamWithAddrInfo,
@@ -953,7 +961,6 @@ class ReqAddrInfoExtractor(busWidth: BusWidth.Value) extends Component {
   io.tx.reqWithRxBufAndVirtualAddrInfo <-/< txNormal.map { payloadData =>
     val result = cloneOf(io.tx.reqWithRxBufAndVirtualAddrInfo.payloadType)
     result.pktFrag := payloadData._1.pktFrag
-//    result.preOpCode := payloadData._1.preOpCode
     result.hasNak := payloadData._1.hasNak
     result.ackAeth := payloadData._1.ackAeth
     result.rxBufValid := payloadData._1.rxBufValid
@@ -1149,7 +1156,7 @@ class ReqPktLenCheck(busWidth: BusWidth.Value) extends Component {
   when(inputValid && io.rx.reqWithRxBufAndDmaInfo.rxBufValid) {
     assert(
       assertion = OpCode.isSendReqPkt(inputPktFrag.bth.opcode) ||
-        OpCode.isWriteWithImmReqPkt(inputPktFrag.bth.opcode),
+        OpCode.isWriteImmReqPkt(inputPktFrag.bth.opcode),
       message =
         L"${REPORT_TIME} time: it should be send requests that require receive buffer, but opcode=${inputPktFrag.bth.opcode}".toSeq,
       severity = FAILURE
@@ -1231,10 +1238,11 @@ class ReqSplitterAndNakGen(busWidth: BusWidth.Value) extends Component {
     val rx = slave(RqReqWithRxBufAndDmaInfoWithLenCheckBus(busWidth))
     val txSendWrite = master(RqReqWithRxBufAndDmaInfoWithLenCheckBus(busWidth))
     val txReadAtomic = master(RqReqWithRxBufAndDmaInfoWithLenCheckBus(busWidth))
-//    val nakNotifier = out(RqNakNotifier())
     val txErrResp = master(Stream(Acknowledge()))
   }
 
+  val inputValid = io.rx.reqWithRxBufAndDmaInfoWithLenCheck.valid
+  val isFirstFrag = io.rx.reqWithRxBufAndDmaInfoWithLenCheck.isFirst
   val inputPktFrag = io.rx.reqWithRxBufAndDmaInfoWithLenCheck.pktFrag
   val inputHasNak = io.rx.reqWithRxBufAndDmaInfoWithLenCheck.hasNak
   val isRetryNak = io.rx.reqWithRxBufAndDmaInfoWithLenCheck.ackAeth.isRetryNak()
@@ -1242,16 +1250,28 @@ class ReqSplitterAndNakGen(busWidth: BusWidth.Value) extends Component {
   val isWriteReq = OpCode.isWriteReqPkt(inputPktFrag.bth.opcode)
   val isReadReq = OpCode.isReadReqPkt(inputPktFrag.bth.opcode)
   val isAtomicReq = OpCode.isAtomicReqPkt(inputPktFrag.bth.opcode)
+//  val isFirstOrOnlyReqPkt = OpCode.isFirstOrOnlyReqPkt(inputPktFrag.bth.opcode)
+//  val isLastOrOnlyReqPkt = OpCode.isFirstOrOnlyReqPkt(inputPktFrag.bth.opcode)
+
+  // TODO: refactor this assertion for inputHasNak
+  when(inputValid && !isFirstFrag) {
+    assert(
+      assertion = stable(inputHasNak) || rose(inputHasNak),
+      message =
+        L"${REPORT_TIME} time: inputHasNak should keep stable or rise throughout whole request, opcode=${inputPktFrag.bth.opcode}, PSN=${inputPktFrag.bth.psn}, isFirstFrag=${isFirstFrag}".toSeq,
+      severity = FAILURE
+    )
+  }
 
   val (reqHasNakStream, reqNormalStream) = StreamConditionalFork2(
     io.rx.reqWithRxBufAndDmaInfoWithLenCheck.throwWhen(io.flush),
     forkCond = inputHasNak
   )
-  //  val (reqNormalStream, reqHasNakStream) = StreamDeMuxByOneCondition(
-  //    io.rx.reqWithRxBufAndDmaInfoWithLenCheck
-  //      .throwWhen(io.rxQCtrl.stateErrFlush),
-  //    condition = inputHasNak
-  //  )
+//  val (reqNormalStream, reqHasNakStream) = StreamDeMuxByOneCondition(
+//    io.rx.reqWithRxBufAndDmaInfoWithLenCheck
+//      .throwWhen(io.rxQCtrl.stateErrFlush),
+//    condition = inputHasNak
+//  )
 
   val (sendWriteIdx, readAtomicIdx) = (0, 1)
   val twoStreams = StreamDeMuxByConditions(
@@ -1272,9 +1292,14 @@ class ReqSplitterAndNakGen(busWidth: BusWidth.Value) extends Component {
     inputPktFrag.bth.psn,
     io.qpAttr.dqpn
   )
+  val errRespGenCond = rqGenRespCond(
+    reqHasNakStream.pktFrag.bth,
+    hasNak = True,
+    aeth = reqHasNakStream.ackAeth
+  )
   io.txErrResp <-/< reqHasNakStream
     // NOTE: response at the last fragment when error
-    .takeWhen(reqHasNakStream.isLast)
+    .takeWhen(reqHasNakStream.isLast && errRespGenCond)
     .translateWith(nakResp)
 
 //  io.nakNotifier.setFromAeth(
@@ -1523,7 +1548,7 @@ class SendWriteRespGenerator(busWidth: BusWidth.Value) extends Component {
   val isWriteReqPkt = OpCode.isWriteReqPkt(inputPktFrag.bth.opcode)
   val isSendOrWriteReq = isSendReqPkt || isWriteReqPkt
   val isWriteWithImmReqPkt =
-    OpCode.isWriteWithImmReqPkt(inputPktFrag.bth.opcode)
+    OpCode.isWriteImmReqPkt(inputPktFrag.bth.opcode)
   val isSendOrWriteImmReq = isSendReqPkt || isWriteWithImmReqPkt
 //  val needWorkCompCond = isSendOrWriteImmReq && isLastFragOfLastOrOnlyPkt
   when(inputValid) {
@@ -1553,9 +1578,10 @@ class SendWriteRespGenerator(busWidth: BusWidth.Value) extends Component {
   }
   when(isLastFragOfLastOrOnlyPkt) {
     assert(
-      assertion = io.rx.reqWithRxBufAndDmaInfoWithLenCheck.reqTotalLenValid,
+      assertion =
+        io.rx.reqWithRxBufAndDmaInfoWithLenCheck.reqTotalLenValid ^ inputHasNak,
       message =
-        L"${REPORT_TIME} time: invalid send/write request, reqTotalLenValid=${io.rx.reqWithRxBufAndDmaInfoWithLenCheck.reqTotalLenValid} should be true".toSeq,
+        L"${REPORT_TIME} time: invalid send/write request, reqTotalLenValid=${io.rx.reqWithRxBufAndDmaInfoWithLenCheck.reqTotalLenValid} should be not match inputHasNak=${inputHasNak}".toSeq,
       severity = FAILURE
     )
   }
@@ -1563,18 +1589,21 @@ class SendWriteRespGenerator(busWidth: BusWidth.Value) extends Component {
   // Only response normal ACK
   val sendWriteNormalAck = Acknowledge().setAck(
     AckType.NORMAL,
-//    io.rx.reqWithRxBufAndDmaInfoWithLenCheck.ackAeth,
     inputPktFrag.bth.psn,
     io.qpAttr.dqpn
   )
 
-  val needAckCond = inputPktFrag.bth.ackreq && isLastFrag && !inputHasNak
   val (req4Ack, req4WorkComp) = StreamFork2(
     io.rx.reqWithRxBufAndDmaInfoWithLenCheck
       .throwWhen(io.flush)
-//      .takeWhen(needAckCond || needWorkCompCond)
   )
 
+  val normalRespGenCond = rqGenRespCond(
+    req4Ack.pktFrag.bth,
+    hasNak = False,
+    aeth = req4Ack.ackAeth
+  )
+  val needAckCond = normalRespGenCond && isLastFrag && !inputHasNak
   // Responses to send/write do not wait for DMA write responses
   io.tx <-/< req4Ack
     .takeWhen(needAckCond)
@@ -1647,7 +1676,7 @@ class RqSendWriteWorkCompGenerator(busWidth: BusWidth.Value) extends Component {
 //    val isWriteReqPkt = OpCode.isWriteReqPkt(inputPktFrag.bth.opcode)
 //    val isSendOrWriteReq = isSendReqPkt || isWriteReqPkt
     val isWriteWithImmReqPkt =
-      OpCode.isWriteWithImmReqPkt(inputPktFrag.bth.opcode)
+      OpCode.isWriteImmReqPkt(inputPktFrag.bth.opcode)
     val isSendOrWriteImmReq = isSendReqPkt || isWriteWithImmReqPkt
     val isLastOrOnlyPkt =
       OpCode.isSendWriteLastOrOnlyReqPkt(inputPktFrag.bth.opcode)
@@ -2047,16 +2076,16 @@ class RqOut(busWidth: BusWidth.Value) extends Component {
     .translateWith(
       io.rxDupAtomicResp.asRdmaDataPktFrag(busWidth)
     )
-  val normalRespVec =
+  val normalAndErrorRespVec =
     Vec(rxSendWriteResp, rxReadResp, rxAtomicResp, rxErrResp)
 
   val hasPktToOutput = Bool()
   val opsnInc = cloneOf(io.opsnInc)
-  val (normalRespOutSel, psnOutRangeQueuePop) = SeqOut(
+  val (normalAndErrorRespOutSel, psnOutRangeQueuePop) = SeqOut(
     curPsn = io.qpAttr.epsn,
     flush = io.flush,
     outPsnRangeQueuePush = io.outPsnRangeFifoPush,
-    outDataStreamVec = normalRespVec,
+    outDataStreamVec = normalAndErrorRespVec,
     outputValidateFunc = (
         psnOutRangeFifoPop: RespPsnRange,
         resp: RdmaDataPkt
@@ -2091,8 +2120,8 @@ class RqOut(busWidth: BusWidth.Value) extends Component {
 //      L"${REPORT_TIME} time: psnOutRangeFifo pop PSN start=${psnOutRangeQueuePop.start}, end=${psnOutRangeQueuePop.end}".toSeq
 //    )
 //  }
-  val (normalRespOut, normalRespOut4ReadAtomicRstCache) =
-    StreamFork2(normalRespOutSel)
+  val (normalAndErrorRespOut, normalAndErrorRespOut4ReadAtomicRstCache) =
+    StreamFork2(normalAndErrorRespOutSel)
 
   val dupRespVec =
     Vec(rxDupSendWriteResp, rxDupReadResp, rxDupAtomicResp)
@@ -2102,14 +2131,17 @@ class RqOut(busWidth: BusWidth.Value) extends Component {
 
   val finalOutStream = StreamArbiterFactory().lowerFirst.fragmentLock
     .onArgs(
-      normalRespOut,
+      normalAndErrorRespOut,
       dupRespOut
     )
   io.tx.pktFrag <-/< finalOutStream.throwWhen(io.flush)
 
   val isReadReq = OpCode.isReadReqPkt(psnOutRangeQueuePop.opcode)
   val isAtomicReq = OpCode.isAtomicReqPkt(psnOutRangeQueuePop.opcode)
-  when(hasPktToOutput && (isReadReq || isAtomicReq)) {
+  val isNonReadAtomicResp = OpCode.isNonReadAtomicRespPkt(
+    normalAndErrorRespOut4ReadAtomicRstCache.bth.opcode
+  )
+  when(hasPktToOutput && !isNonReadAtomicResp && (isReadReq || isAtomicReq)) {
     assert(
       assertion = io.readAtomicRstCachePop.valid,
       message =
@@ -2118,10 +2150,11 @@ class RqOut(busWidth: BusWidth.Value) extends Component {
     )
   }
 
-  val normalRespOutJoinReadAtomicRstCachePopCond = (isReadReq || isAtomicReq) &&
-    (normalRespOut4ReadAtomicRstCache.bth.psn === (io.readAtomicRstCachePop.psnStart + (io.readAtomicRstCachePop.pktNum - 1)))
+  val normalRespOutJoinReadAtomicRstCachePopCond =
+    (isReadReq || isAtomicReq) && !isNonReadAtomicResp &&
+      (normalAndErrorRespOut4ReadAtomicRstCache.bth.psn === (io.readAtomicRstCachePop.psnStart + (io.readAtomicRstCachePop.pktNum - 1)))
   val normalRespOutJoinReadAtomicRstCache = FragmentStreamConditionalJoinStream(
-    inputFragStream = normalRespOut4ReadAtomicRstCache,
+    inputFragStream = normalAndErrorRespOut4ReadAtomicRstCache,
     inputStream = io.readAtomicRstCachePop,
     joinCond = normalRespOutJoinReadAtomicRstCachePopCond
   )
@@ -2143,10 +2176,10 @@ class RqOut(busWidth: BusWidth.Value) extends Component {
     assert(
       assertion = checkRespOpCodeMatch(
         reqOpCode = io.readAtomicRstCachePop.opcode,
-        respOpCode = normalRespOut4ReadAtomicRstCache.bth.opcode
+        respOpCode = normalAndErrorRespOut4ReadAtomicRstCache.bth.opcode
       ),
       message =
-        L"${REPORT_TIME} time: io.readAtomicRstCachePop.opcode=${io.readAtomicRstCachePop.opcode} should match normalRespOut4ReadAtomicRstCache.bth.opcode=${normalRespOut4ReadAtomicRstCache.bth.opcode}, normalRespOut4ReadAtomicRstCache.bth.psn=${normalRespOut4ReadAtomicRstCache.bth.psn}, io.readAtomicRstCachePop.psnStart=${io.readAtomicRstCachePop.psnStart}, io.readAtomicRstCachePop.pktNum=${io.readAtomicRstCachePop.pktNum}".toSeq,
+        L"${REPORT_TIME} time: io.readAtomicRstCachePop.opcode=${io.readAtomicRstCachePop.opcode} should match normalRespOut4ReadAtomicRstCache.bth.opcode=${normalAndErrorRespOut4ReadAtomicRstCache.bth.opcode}, normalRespOut4ReadAtomicRstCache.bth.psn=${normalAndErrorRespOut4ReadAtomicRstCache.bth.psn}, io.readAtomicRstCachePop.psnStart=${io.readAtomicRstCachePop.psnStart}, io.readAtomicRstCachePop.pktNum=${io.readAtomicRstCachePop.pktNum}".toSeq,
       severity = FAILURE
     )
   }
diff --git a/rocev2/src/rdma/RespHandler.scala b/rocev2/src/rdma/RespHandler.scala
index 23609dd..bc3aed5 100644
--- a/rocev2/src/rdma/RespHandler.scala
+++ b/rocev2/src/rdma/RespHandler.scala
@@ -568,7 +568,7 @@ class ReadAtomicRespVerifierAndFatalNakNotifier(busWidth: BusWidth.Value)
         result.workCompStatus := WorkCompStatus.LOC_LEN_ERR
       } elsewhen (keyErr || accessErr) {
         ack.aeth.set(AckType.NAK_RMT_ACC)
-        result.workCompStatus := WorkCompStatus.LOC_PROT_ERR
+        result.workCompStatus := WorkCompStatus.LOC_ACCESS_ERR
       }
     }
 
diff --git a/rocev2/src/rdma/Utils.scala b/rocev2/src/rdma/Utils.scala
index 1a88bf3..1078c15 100644
--- a/rocev2/src/rdma/Utils.scala
+++ b/rocev2/src/rdma/Utils.scala
@@ -1117,6 +1117,7 @@ object FragmentStreamForkQueryJoinResp {
       queryStream: Stream[Tquery],
       respStream: Stream[Tresp],
       waitQueueDepth: Int,
+      waitQueueName: String,
       buildQuery: Stream[Fragment[Tin]] => Tquery,
       queryCond: Stream[Fragment[Tin]] => Bool,
       expectResp: Stream[Fragment[Tin]] => Bool,
@@ -1138,7 +1139,7 @@ object FragmentStreamForkQueryJoinResp {
         depth = waitQueueDepth,
         push = input4Queue,
         flush = False, // TODO: enable queue flush
-        queueName = "inputFragQueue"
+        queueName = waitQueueName
       )
       val emptyStream = StreamSource().translateWith(respStream.payload)
       // When not expect query response, join with StreamSource()
@@ -1616,6 +1617,35 @@ object setAllBits {
     }.result
 }
 
+// For normal ACK or non-retry NAK, they are responded at
+// the last fragment of the last or the only request packet.
+object rqGenRespCond {
+  def apply(bth: BTH, hasNak: Bool, aeth: AETH): Bool =
+    new Composite(bth, "rqRespCond") {
+      val opcode = bth.opcode
+      val isSendReq = OpCode.isSendReqPkt(opcode)
+      val isWriteReq = OpCode.isWriteReqPkt(opcode)
+      val isReadReq = OpCode.isReadReqPkt(opcode)
+      val isAtomicReq = OpCode.isAtomicReqPkt(opcode)
+      val isFirstOrOnlyReqPkt = OpCode.isFirstOrOnlyReqPkt(opcode)
+      val isLastOrOnlyReqPkt = OpCode.isLastOrOnlyReqPkt(opcode)
+      val ackReq =
+        ((isSendReq || isWriteReq) && bth.ackreq) || isReadReq || isAtomicReq
+      val respAtFirstOrLastPkt =
+        (hasNak && aeth.isRetryNak()) ? isFirstOrOnlyReqPkt | isLastOrOnlyReqPkt
+      val result = (hasNak && respAtFirstOrLastPkt) || ackReq
+    }.result
+}
+
+// For retry NAK, it is responded at the last fragment of
+// the first or the only request packet.
+object retryNakTriggerCond {
+  def apply(lastFire: Bool, hasRetryNak: Bool): Bool =
+    new Composite(lastFire, "retryNakTriggerCond") {
+      val result = lastFire && hasRetryNak
+    }.result
+}
+
 object mergeRdmaHeader {
   def apply(busWidth: BusWidth.Value, baseHeader: RdmaHeader): Bits =
     new Composite(baseHeader, "mergeRdmaHeader1") {
@@ -2893,6 +2923,7 @@ object AddrCacheQueryAndRespHandler {
       addrCacheRead.req,
       addrCacheRead.resp,
       waitQueueDepth = ADDR_CACHE_QUERY_FIFO_DEPTH,
+      waitQueueName = "AddrCacheQueryWaitQueue",
       buildQuery = buildAddrCacheQuery,
       queryCond = addrCacheQueryCond,
       expectResp = expectAddrCacheResp,
diff --git a/rocev2/test/src/rdma/CacheTest.scala b/rocev2/test/src/rdma/CacheTest.scala
index da07613..e6e56c0 100644
--- a/rocev2/test/src/rdma/CacheTest.scala
+++ b/rocev2/test/src/rdma/CacheTest.scala
@@ -471,7 +471,7 @@ class WorkReqCacheTest extends AnyFunSuite {
     ]()
 
     streamMasterDriver(dut.io.push, dut.clockDomain) {
-      dut.io.push.workReq.opcode #= WorkReqSim.randomSendWriteReadAtomicOpCode()
+      dut.io.push.workReq.opcode #= WorkReqSim.randomRdmaReqOpCode()
     }
     onStreamFire(dut.io.push, dut.clockDomain) {
       inputWorkReqQueue.enqueue(
@@ -612,7 +612,7 @@ class WorkReqCacheTest extends AnyFunSuite {
                 dut.io.push.psnStart.toInt,
                 dut.io.push.pktNum.toInt,
                 dut.io.push.workReq.lenBytes.toLong,
-                WorkReqSim.randomSendWriteReadAtomicOpCode(),
+                WorkReqSim.randomRdmaReqOpCode(),
                 dut.io.push.workReq.raddr.toBigInt,
                 dut.io.push.workReq.rkey.toLong
               )
diff --git a/rocev2/test/src/rdma/DataStructSim.scala b/rocev2/test/src/rdma/DataStructSim.scala
index 17815b6..4131507 100644
--- a/rocev2/test/src/rdma/DataStructSim.scala
+++ b/rocev2/test/src/rdma/DataStructSim.scala
@@ -421,6 +421,17 @@ object WorkReqSim {
     result
   }
 
+  def randomWriteOpCode(): SpinalEnumElement[WorkReqOpCode.type] = {
+    val opCodes = workReqWrite
+    val randIdx = scala.util.Random.nextInt(opCodes.size)
+    val result = opCodes(randIdx)
+    require(
+      opCodes.contains(result),
+      s"${simTime()} time: WR WriteOpCode should contain ${result}"
+    )
+    result
+  }
+
   def randomSendWriteReadOpCode(): SpinalEnumElement[WorkReqOpCode.type] = {
     val opCodes = WorkReqOpCode.RDMA_READ +: (workReqSend ++ workReqWrite)
     val randIdx = scala.util.Random.nextInt(opCodes.size)
@@ -432,8 +443,7 @@ object WorkReqSim {
     result
   }
 
-  def randomSendWriteReadAtomicOpCode()
-      : SpinalEnumElement[WorkReqOpCode.type] = {
+  def randomRdmaReqOpCode(): SpinalEnumElement[WorkReqOpCode.type] = {
     val opCodes =
       WorkReqOpCode.RDMA_READ +: (workReqSend ++ workReqWrite ++ workReqAtomic)
     val randIdx = scala.util.Random.nextInt(opCodes.size)
@@ -1275,7 +1285,7 @@ trait QueryRespSim[
 //  ): Unit
 
   // Need to override if generate multiple responses
-  protected def buildMultiResp(
+  protected def buildMultiFragResp(
       reqData: ReqData,
       respAlwaysSuccess: Boolean
   ): Seq[RespData]
@@ -1298,7 +1308,7 @@ trait QueryRespSim[
       clockDomain: ClockDomain,
       hasProcessDelay: Boolean,
       processDelayCycles: Int,
-      hasMultiResp: Boolean,
+      respHasMultiFrags: Boolean,
       respAlwaysSuccess: Boolean
   ): mutable.Queue[RespData] = {
     val reqStream = queryBus.req
@@ -1358,15 +1368,16 @@ trait QueryRespSim[
 //    println(f"${simTime()} time: hasProcessDelay=${hasProcessDelay}")
 
     // Handle response
-    if (hasMultiResp) {
+    if (respHasMultiFrags) {
       val multiRespQueue = mutable.Queue[RespData]()
       fork {
         clockDomain.waitSampling()
 
         while (true) {
           val inputReq = MiscUtils.safeDeQueue(delayedReqQueue, clockDomain)
-          val multiRespData = buildMultiResp(inputReq, respAlwaysSuccess)
-          multiRespQueue.appendAll(multiRespData)
+          val multiFragRespData =
+            buildMultiFragResp(inputReq, respAlwaysSuccess)
+          multiRespQueue.appendAll(multiFragRespData)
 
 //          println(f"${simTime()} time: multiRespQueue.length=${multiRespQueue.length}")
         }
@@ -1421,7 +1432,7 @@ trait SingleRespQuerySim[
       clockDomain,
       hasProcessDelay = false,
       processDelayCycles = 0,
-      hasMultiResp = false,
+      respHasMultiFrags = false,
       respAlwaysSuccess = true
     )
   }
@@ -1435,7 +1446,7 @@ trait SingleRespQuerySim[
       clockDomain,
       hasProcessDelay = false,
       processDelayCycles = 0,
-      hasMultiResp = false,
+      respHasMultiFrags = false,
       respAlwaysSuccess = false
     )
   }
@@ -1450,7 +1461,7 @@ trait SingleRespQuerySim[
       clockDomain,
       hasProcessDelay = true,
       processDelayCycles = fixedRespDelayCycles,
-      hasMultiResp = false,
+      respHasMultiFrags = false,
       respAlwaysSuccess = true
     )
   }
@@ -1465,7 +1476,7 @@ trait SingleRespQuerySim[
       clockDomain,
       hasProcessDelay = true,
       processDelayCycles = fixedRespDelayCycles,
-      hasMultiResp = false,
+      respHasMultiFrags = false,
       respAlwaysSuccess = false
     )
   }
@@ -1480,7 +1491,7 @@ trait SingleRespQuerySim[
       clockDomain,
       hasProcessDelay = true,
       processDelayCycles = fixedRespDelayCycles,
-      hasMultiResp = true,
+      respHasMultiFrags = true,
       respAlwaysSuccess = true
     )
   }
@@ -1501,7 +1512,7 @@ trait SingleRespQuerySim[
 //  }
 
   // No multiple responses generated for a single request
-  override def buildMultiResp(
+  override def buildMultiFragResp(
       reqData: ReqData,
       respAlwaysSuccess: Boolean
   ): Seq[RespData] = {
@@ -1540,7 +1551,7 @@ trait MultiRespQuerySim[
       clockDomain,
       hasProcessDelay = false,
       processDelayCycles = 0,
-      hasMultiResp = true,
+      respHasMultiFrags = true,
       respAlwaysSuccess = true
     )
   }
@@ -1554,7 +1565,7 @@ trait MultiRespQuerySim[
       clockDomain,
       hasProcessDelay = false,
       processDelayCycles = 0,
-      hasMultiResp = true,
+      respHasMultiFrags = true,
       respAlwaysSuccess = false
     )
   }
@@ -1569,7 +1580,7 @@ trait MultiRespQuerySim[
       clockDomain,
       hasProcessDelay = true,
       processDelayCycles = fixedRespDelayCycles,
-      hasMultiResp = true,
+      respHasMultiFrags = true,
       respAlwaysSuccess = true
     )
   }
@@ -1584,7 +1595,7 @@ trait MultiRespQuerySim[
       clockDomain,
       hasProcessDelay = true,
       processDelayCycles = fixedRespDelayCycles,
-      hasMultiResp = true,
+      respHasMultiFrags = true,
       respAlwaysSuccess = false
     )
   }
@@ -1620,21 +1631,22 @@ object AddrCacheSim
       )
     ] {
 
-  override def onReqFire(req: QpAddrCacheAgentReadReq):
-//      reqQueue: mutable.Queue[
-  (
+//  def setRespFailureType(invalidRequestOrRemoteAccess: Boolean): (KeyValid, SizeValid, AccessValid) = {
+//    if (invalidRequestOrRemoteAccess) {
+//      (false, false, true)
+//    } else {
+//      (true, true, false)
+//    }
+//  }
+
+  override def onReqFire(req: QpAddrCacheAgentReadReq): (
       PSN,
       LRKey,
       Boolean,
       VirtualAddr,
       PktLen
   ) = {
-//      ]
-//  ): Unit = {
-
 //    println(f"${simTime()} time: AddrCacheSim receive request PSN=${req.psn.toInt}%X")
-
-//    reqQueue.enqueue(
     (
       req.psn.toInt,
       req.key.toLong,
@@ -1642,7 +1654,6 @@ object AddrCacheSim
       req.va.toBigInt,
       req.dataLenBytes.toLong
     )
-    //    )
   }
 
   override def buildRespFromReqData(
@@ -1665,8 +1676,8 @@ object AddrCacheSim
       resp.sizeValid #= true
       resp.accessValid #= true
     } else {
-      resp.keyValid #= false
-      resp.sizeValid #= false
+      resp.keyValid #= true
+      resp.sizeValid #= true
       resp.accessValid #= false
     }
 
@@ -1675,18 +1686,13 @@ object AddrCacheSim
     respValid
   }
 
-  override def onRespFire(respData: QpAddrCacheAgentReadResp):
-//      respQueue: mutable.Queue[
-  (
+  override def onRespFire(respData: QpAddrCacheAgentReadResp): (
       PSN,
       KeyValid,
       SizeValid,
       AccessValid,
       PhysicalAddr
   ) = {
-//      ]
-//  ): Unit = {
-//    respQueue.enqueue(
     (
       respData.psn.toInt,
       respData.keyValid.toBoolean,
@@ -1694,7 +1700,6 @@ object AddrCacheSim
       respData.accessValid.toBoolean,
       respData.pa.toBigInt
     )
-    //    )
   }
 }
 
@@ -2094,7 +2099,7 @@ case class DmaReadBusSim(busWidth: BusWidth.Value)
 //    )
   }
 
-  override def buildMultiResp(
+  override def buildMultiFragResp(
       reqData: (
           SpinalEnumElement[DmaInitiator.type],
           PSN,
@@ -2430,10 +2435,12 @@ object RdmaDataPktSim {
       stream: Fragment[RdmaDataPkt],
       psnStart: PSN,
       numWorkReq: Int,
-      isReadRespGen: Boolean,
-      isSendWriteImmReqOnly: Boolean,
-      isSendWriteReqOnly: Boolean,
-      isReadAtomicReqOnly: Boolean,
+      genReadResp: Boolean,
+      genSendReq: Boolean,
+      genWriteReq: Boolean,
+      genWriteImmReq: Boolean,
+      genReadReq: Boolean,
+      genAtomicReq: Boolean,
       pmtuLen: PMTU.Value,
       busWidth: BusWidth.Value,
       maxFragNum: Int
@@ -2452,14 +2459,16 @@ object RdmaDataPktSim {
           OpCode.Value
       ) => Unit
   ): Unit = {
-    require(
-      (isReadRespGen && !isSendWriteImmReqOnly && !isSendWriteReqOnly && !isReadAtomicReqOnly == true) ||
-        (!isReadRespGen && isSendWriteImmReqOnly && !isSendWriteReqOnly && !isReadAtomicReqOnly == true) ||
-        (!isReadRespGen && !isSendWriteImmReqOnly && isSendWriteReqOnly && !isReadAtomicReqOnly == true) ||
-        (!isReadRespGen && !isSendWriteImmReqOnly && !isSendWriteReqOnly && isReadAtomicReqOnly == true) ||
-        (!isReadRespGen && !isSendWriteImmReqOnly && !isSendWriteReqOnly && !isReadAtomicReqOnly == true),
-      s"${simTime()} time: isReadRespGen=${isReadRespGen}, isSendWriteImmReqOnly=${isSendWriteImmReqOnly}, isSendWriteReqOnly=${isSendWriteReqOnly}, isReadAtomicReqOnly=${isReadAtomicReqOnly}, no more than one can be true"
-    )
+    if (genReadResp) {
+      require(
+        !(genSendReq &&
+          genWriteReq &&
+          genWriteImmReq &&
+          genReadReq &&
+          genAtomicReq),
+        s"${simTime()} time: genSendReq=${genSendReq}, genWriteReq=${genWriteReq}, genWriteImmReq=${genWriteImmReq}, genReadReq=${genReadReq}, genAtomicReq=${genAtomicReq}, must all be false when genReadResp=${genReadResp}"
+      )
+    }
 
     val (payloadFragNumItr, pktNumItr, psnStartItr, payloadLenItr) =
       SendWriteReqReadRespInputGen.getItr(
@@ -2476,29 +2485,61 @@ object RdmaDataPktSim {
       val psnStart = psnStartItr.next()
       val payloadLenBytes = payloadLenItr.next()
 
-      val workReqOpCode = if (isSendWriteImmReqOnly) {
-        WorkReqSim.randomSendWriteImmOpCode()
-      } else if (isSendWriteReqOnly) {
-        WorkReqSim.randomSendWriteOpCode()
-      } else if (isReadAtomicReqOnly) {
-        if (totalPktNum == 1) {
+      val shouldGenAtomicReq = genAtomicReq && totalPktNum == 1
+      val workReqOpCode = (
+        genSendReq,
+        genWriteReq,
+        genWriteImmReq,
+        genReadReq,
+        shouldGenAtomicReq
+      ) match {
+        case (true, true, true, true, true) => WorkReqSim.randomRdmaReqOpCode()
+        case (true, true, true, true, false) =>
+          WorkReqSim.randomSendWriteReadOpCode()
+        case (true, true, true, false, false) =>
+          WorkReqSim.randomSendWriteOpCode()
+        case (true, false, true, false, false) =>
+          WorkReqSim.randomSendWriteImmOpCode()
+        case (false, false, false, true, true) =>
           WorkReqSim.randomReadAtomicOpCode()
-        } else {
-          WorkReqOpCode.RDMA_READ
-        }
-      } else if (totalPktNum == 1) {
-        WorkReqSim.randomSendWriteReadAtomicOpCode()
-      } else {
-        WorkReqSim.randomSendWriteReadOpCode()
+        case (true, false, false, false, false) => WorkReqSim.randomSendOpCode()
+        case (false, true, true, false, false) => WorkReqSim.randomWriteOpCode()
+        case (false, false, true, false, false) =>
+          WorkReqOpCode.RDMA_WRITE_WITH_IMM
+        case (false, false, false, true, false) => WorkReqOpCode.RDMA_READ
+        case _ =>
+          if (genReadResp) {
+            WorkReqSim.randomAtomicOpCode()
+          } else {
+            SpinalExit(
+              f"${simTime()} time: invalid WR generation combo, totalPktNum=${totalPktNum}, genSendReq=${genSendReq}, genWriteReq=${genWriteReq}, genWriteImmReq=${genWriteImmReq}, genReadReq=${genReadReq}, genAtomicReq=${genAtomicReq}"
+            )
+          }
       }
 
+//      val workReqOpCode = if (isSendWriteImmReqOnly) {
+//        WorkReqSim.randomSendWriteImmOpCode()
+//      } else if (isSendWriteReqOnly) {
+//        WorkReqSim.randomSendWriteOpCode()
+//      } else if (isReadAtomicReqOnly) {
+//        if (totalPktNum == 1) {
+//          WorkReqSim.randomReadAtomicOpCode()
+//        } else {
+//          WorkReqOpCode.RDMA_READ
+//        }
+//      } else if (totalPktNum == 1) {
+//        WorkReqSim.randomRdmaReqOpCode()
+//      } else {
+//        WorkReqSim.randomSendWriteReadOpCode()
+//      }
+
       val isReadReq = workReqOpCode == WorkReqOpCode.RDMA_READ
       val isAtomicReq = Seq(
         WorkReqOpCode.ATOMIC_CMP_AND_SWP,
         WorkReqOpCode.ATOMIC_FETCH_AND_ADD
       ).contains(workReqOpCode)
 
-      val (pktNum4Loop, reqPktNum, respPktNum) = if (isReadRespGen) {
+      val (pktNum4Loop, reqPktNum, respPktNum) = if (genReadResp) {
         (totalPktNum, 1, totalPktNum)
       } else if (isReadReq) {
         (1, 1, totalPktNum) // Only 1 packet for read request
@@ -2512,7 +2553,7 @@ object RdmaDataPktSim {
 
       // Inner loop
       for (pktIdx <- 0 until pktNum4Loop) {
-        val opcode = if (isReadRespGen) {
+        val opcode = if (genReadResp) {
           WorkReqSim.assignReadRespOpCode(pktIdx, pktNum4Loop)
         } else {
           WorkReqSim.assignReqOpCode(workReqOpCode, pktIdx, pktNum4Loop)
@@ -2520,7 +2561,7 @@ object RdmaDataPktSim {
 
         val headerLenBytes = opcode.getPktHeaderLenBytes()
         val psn = psnStart + pktIdx
-        val pktFragNum = if (!isReadRespGen && (isReadReq || isAtomicReq)) {
+        val pktFragNum = if (!genReadResp && (isReadReq || isAtomicReq)) {
           1 // Only 1 fragment for read/atomic request
         } else {
           computePktFragNum(
@@ -2538,7 +2579,7 @@ object RdmaDataPktSim {
 
         for (fragIdx <- 0 until pktFragNum) {
           val fragLast = fragIdx == pktFragNum - 1
-          val mty = if (!isReadRespGen && (isReadReq || isAtomicReq)) {
+          val mty = if (!genReadResp && (isReadReq || isAtomicReq)) {
             computeHeaderMty(headerLenBytes, busWidth)
           } else {
             computePktFragMty(
@@ -2593,6 +2634,43 @@ object RdmaDataPktSim {
     }
   }
 
+  def writeImmReqPktFragStreamMasterGen(
+      fragment: Fragment[RdmaDataPkt],
+      psnStart: PSN,
+      numWorkReq: Int,
+      pmtuLen: PMTU.Value,
+      busWidth: BusWidth.Value,
+      maxFragNum: Int
+  )(
+      innerLoopFunc: (
+          PSN,
+          PsnStart,
+          FragLast,
+          FragIdx,
+          FragNum,
+          PktIdx,
+          ReqPktNum,
+          RespPktNum,
+          PktLen,
+          HeaderLen, // TODO: remove this field
+          OpCode.Value
+      ) => Unit
+  ): Unit =
+    rdmaPktFragStreamMasterGenHelper(
+      fragment,
+      psnStart,
+      numWorkReq,
+      genReadResp = false,
+      genSendReq = false,
+      genWriteReq = false,
+      genWriteImmReq = true,
+      genReadReq = false,
+      genAtomicReq = false,
+      pmtuLen = pmtuLen,
+      busWidth = busWidth,
+      maxFragNum = maxFragNum
+    )(innerLoopFunc)
+
   def sendWriteImmReqPktFragStreamMasterGen(
       fragment: Fragment[RdmaDataPkt],
       psnStart: PSN,
@@ -2619,10 +2697,12 @@ object RdmaDataPktSim {
       fragment,
       psnStart,
       numWorkReq,
-      isReadRespGen = false,
-      isSendWriteImmReqOnly = true,
-      isSendWriteReqOnly = false,
-      isReadAtomicReqOnly = false,
+      genReadResp = false,
+      genSendReq = true,
+      genWriteReq = false,
+      genWriteImmReq = true,
+      genReadReq = false,
+      genAtomicReq = false,
       pmtuLen = pmtuLen,
       busWidth = busWidth,
       maxFragNum = maxFragNum
@@ -2654,10 +2734,12 @@ object RdmaDataPktSim {
       fragment,
       psnStart,
       numWorkReq,
-      isReadRespGen = false,
-      isSendWriteImmReqOnly = false,
-      isSendWriteReqOnly = true,
-      isReadAtomicReqOnly = false,
+      genReadResp = false,
+      genSendReq = true,
+      genWriteReq = true,
+      genWriteImmReq = true,
+      genReadReq = false,
+      genAtomicReq = false,
       pmtuLen = pmtuLen,
       busWidth = busWidth,
       maxFragNum = maxFragNum
@@ -2688,10 +2770,12 @@ object RdmaDataPktSim {
     fragment,
     psnStart,
     numWorkReq,
-    isReadRespGen = false,
-    isSendWriteImmReqOnly = false,
-    isSendWriteReqOnly = false,
-    isReadAtomicReqOnly = true,
+    genReadResp = false,
+    genSendReq = false,
+    genWriteReq = false,
+    genWriteImmReq = false,
+    genReadReq = true,
+    genAtomicReq = true,
     pmtuLen = pmtuLen,
     busWidth = busWidth,
     maxFragNum = maxFragNum
@@ -2722,10 +2806,12 @@ object RdmaDataPktSim {
     fragment,
     psnStart,
     numWorkReq,
-    isReadRespGen = false,
-    isSendWriteImmReqOnly = false,
-    isSendWriteReqOnly = false,
-    isReadAtomicReqOnly = false,
+    genReadResp = false,
+    genSendReq = true,
+    genWriteReq = true,
+    genWriteImmReq = true,
+    genReadReq = true,
+    genAtomicReq = true,
     pmtuLen = pmtuLen,
     busWidth = busWidth,
     maxFragNum = maxFragNum
@@ -2798,7 +2884,7 @@ object RdmaDataPktSim {
           WorkReqSim.randomSendWriteOpCode()
         } else {
           if (totalPktNum == 1) {
-            WorkReqSim.randomSendWriteReadAtomicOpCode()
+            WorkReqSim.randomRdmaReqOpCode()
           } else {
             WorkReqSim.randomSendWriteReadOpCode()
           }
@@ -3749,24 +3835,23 @@ object RqNakSim {
 object QpCtrlSim {
   import PsnSim._
 
-  object RqSubState extends SpinalEnum {
+  object RqSubState extends Enumeration {
     val WAITING, NORMAL, NAK_SEQ_TRIGGERED, NAK_SEQ, RNR_TRIGGERED, RNR_TIMEOUT,
-        RNR =
-      newElement()
+        RNR = Value
   }
 
-  object SqSubState extends SpinalEnum {
+  object SqSubState extends Enumeration {
     val WAITING, NORMAL, RETRY_FLUSH, RETRY_WORK_REQ, DRAINING, DRAINED,
-        COALESCE, ERR_FLUSH = newElement()
+        COALESCE, ERR_FLUSH = Value
   }
 
 //  val rxQCtrl = RxQCtrl()
 //  val txQCtrl = TxQCtrl()
 //  val qpAttr = QpAttrData()
-//  val rqSubState = RqSubState()
-//  val sqSubState = SqSubState()
+  var rqSubState = RqSubState.WAITING
+  var sqSubState = SqSubState.WAITING
 
-  private def initQpAttrData(qpAttr: QpAttrData, pmtuLen: PMTU.Value): Unit = {
+  def initQpAttrData(qpAttr: QpAttrData, pmtuLen: PMTU.Value): Unit = {
     qpAttr.npsn #= INIT_PSN
     qpAttr.epsn #= INIT_PSN
     qpAttr.rqOutPsn #= INIT_PSN -% 1
@@ -3789,8 +3874,20 @@ object QpCtrlSim {
     qpAttr.respTimeOut #= DEFAULT_RESP_TIME_OUT
 
     qpAttr.state #= QpState.RESET
-//    rqSubState #= RqSubState.WAITING
-//    sqSubState #= SqSubState.WAITING
+    rqSubState = RqSubState.WAITING
+    sqSubState = SqSubState.WAITING
+  }
+
+  def resetQpAttr2Normal(qpAttr: QpAttrData): Unit = {
+    qpAttr.state #= QpState.RTS
+    qpAttr.epsn #= SimSettings.INIT_PSN
+    qpAttr.npsn #= INIT_PSN
+    qpAttr.epsn #= INIT_PSN
+    qpAttr.rqOutPsn #= INIT_PSN -% 1
+    qpAttr.sqOutPsn #= INIT_PSN -% 1
+
+    rqSubState = RqSubState.NORMAL
+    sqSubState = SqSubState.NORMAL
   }
 
   private def rqHasFatalNak(rqNotifier: RqNotifier): Boolean = {
@@ -3896,12 +3993,12 @@ object QpCtrlSim {
   ) = {
     initQpAttrData(qpAttr, pmtuLen)
     qpAttr.state #= QpState.RTR
-    var rqSubState = RqSubState.NORMAL
+    rqSubState = RqSubState.NORMAL
 
     fork {
       while (true) {
-//        updateQpAttr(thatQpAttr)
         clockDomain.waitSampling()
+//        println(f"${simTime()} time: rqSubState=${rqSubState}")
 
         // Update PSN
         if (rqSubState == RqSubState.NORMAL) {
@@ -3927,9 +4024,9 @@ object QpCtrlSim {
             qpAttr.epsn #= rqNotifier.nak.rnr.psn.toInt
             qpAttr.rqPreReqOpCode #= rqNotifier.nak.rnr.preOpCode.toInt
 
-            println(
-              f"${simTime()} time: received rqNotifier.nak.rnr.pulse=${rqNotifier.nak.rnr.pulse.toBoolean} and rqNotifier.nak.rnr.psn=${rqNotifier.nak.rnr.psn.toInt}, RqSubState.RNR_TRIGGERED=${RqSubState.RNR_TRIGGERED}"
-            )
+//            println(
+//              f"${simTime()} time: received rqNotifier.nak.rnr.pulse=${rqNotifier.nak.rnr.pulse.toBoolean} and rqNotifier.nak.rnr.psn=${rqNotifier.nak.rnr.psn.toInt}, RqSubState.RNR_TRIGGERED=${RqSubState.RNR_TRIGGERED}"
+//            )
           } else if (
             rqSubState == RqSubState.RNR_TRIGGERED && rqNotifier.retryNakHasSent.pulse.toBoolean
           ) {
@@ -3937,9 +4034,9 @@ object QpCtrlSim {
             sleep(qpAttr.negotiatedRnrTimeOut.toLong)
             rqSubState = RqSubState.RNR
 
-            println(
-              f"${simTime()} time: received rqNotifier.retryNakHasSent.pulse=${rqNotifier.retryNakHasSent.pulse.toBoolean}, RqSubState.RNR=${RqSubState.RNR}"
-            )
+//            println(
+//              f"${simTime()} time: received rqNotifier.retryNakHasSent.pulse=${rqNotifier.retryNakHasSent.pulse.toBoolean}, RqSubState.RNR=${RqSubState.RNR}"
+//            )
           } else if (
             rqSubState == RqSubState.NORMAL && rqNotifier.nak.seqErr.pulse.toBoolean
           ) {
@@ -3947,26 +4044,26 @@ object QpCtrlSim {
 //            qpAttr.epsn #= rqNotifier.nak.seqErr.psn.toInt
 //            qpAttr.rqPreReqOpCode #= rqNotifier.nak.seqErr.preOpCode.toInt
 
-            println(
-              f"${simTime()} time: received rqNotifier.nak.seqErr.pulse=${rqNotifier.nak.seqErr.pulse.toBoolean} and rqNotifier.nak.seqErr.psn=${rqNotifier.nak.seqErr.psn.toInt}%X, RqSubState.NAK_SEQ_TRIGGERED=${RqSubState.NAK_SEQ_TRIGGERED}"
-            )
+//            println(
+//              f"${simTime()} time: received rqNotifier.nak.seqErr.pulse=${rqNotifier.nak.seqErr.pulse.toBoolean} and rqNotifier.nak.seqErr.psn=${rqNotifier.nak.seqErr.psn.toInt}%X, RqSubState.NAK_SEQ_TRIGGERED=${RqSubState.NAK_SEQ_TRIGGERED}"
+//            )
           } else if (
             rqSubState == RqSubState.NAK_SEQ_TRIGGERED && rqNotifier.retryNakHasSent.pulse.toBoolean
           ) {
             rqSubState = RqSubState.NAK_SEQ
 
-            println(
-              f"${simTime()} time: received rqNotifier.retryNakHasSent.pulse=${rqNotifier.retryNakHasSent.pulse.toBoolean}, RqSubState.NAK_SEQ=${RqSubState.NAK_SEQ}"
-            )
+//            println(
+//              f"${simTime()} time: received rqNotifier.retryNakHasSent.pulse=${rqNotifier.retryNakHasSent.pulse.toBoolean}, RqSubState.NAK_SEQ=${RqSubState.NAK_SEQ}"
+//            )
           } else if (
             (rqSubState == RqSubState.NAK_SEQ || rqSubState == RqSubState.RNR) &&
             rqNotifier.clearRetryNakFlush.pulse.toBoolean
           ) {
             rqSubState = RqSubState.NORMAL
 
-            println(
-              f"${simTime()} time: received rqNotifier.clearRetryNakFlush.pulse=${rqNotifier.clearRetryNakFlush.pulse.toBoolean}, RqSubState.NORMAL=${RqSubState.NORMAL}"
-            )
+//            println(
+//              f"${simTime()} time: received rqNotifier.clearRetryNakFlush.pulse=${rqNotifier.clearRetryNakFlush.pulse.toBoolean}, RqSubState.NORMAL=${RqSubState.NORMAL}"
+//            )
           }
         }
         // TODO: check RQ behavior under ERR
@@ -3982,10 +4079,7 @@ object QpCtrlSim {
         val rnrWait4Clear = rqSubState == RqSubState.RNR
         // RQ flush
         rxQCtrl.stateErrFlush #= isQpStateWrong
-//        rxQCtrl.nakSeqTriggered #= nakSeqTriggered
         rxQCtrl.nakSeqFlush #= nakSeqTriggered || nakSeqWait4Clear
-//        rxQCtrl.rnrTriggered #= rnrTriggered
-//        rxQCtrl.rnrTimeOut #= rnrTimeOut
         rxQCtrl.rnrFlush #= rnrTriggered || rnrTimeOut || rnrWait4Clear
         rxQCtrl.isRetryNakNotCleared #= rnrTriggered || rnrTimeOut || rnrWait4Clear || nakSeqTriggered || nakSeqWait4Clear
       }
@@ -4002,12 +4096,12 @@ object QpCtrlSim {
   ) = {
     initQpAttrData(qpAttr, pmtuLen)
     qpAttr.state #= QpState.RTS
-    var sqSubState = SqSubState.NORMAL
+    sqSubState = SqSubState.NORMAL
 
     fork {
       while (true) {
-//        updateQpAttr(thatQpAttr)
         clockDomain.waitSampling()
+//        println(f"${simTime()} time: sqSubState=${sqSubState}")
 
         // Update PSN
         if (sqSubState == SqSubState.NORMAL) {
diff --git a/rocev2/test/src/rdma/QPTest.scala b/rocev2/test/src/rdma/QPTest.scala
index 9de9a90..5e457a3 100644
--- a/rocev2/test/src/rdma/QPTest.scala
+++ b/rocev2/test/src/rdma/QPTest.scala
@@ -133,7 +133,7 @@ class QPTest extends AnyFunSuite {
         } else if (noDmaRead) {
           WorkReqSim.randomReadAtomicOpCode()
         } else {
-          WorkReqSim.randomSendWriteReadAtomicOpCode()
+          WorkReqSim.randomRdmaReqOpCode()
         }
         val pktLen = if (workReqOpCode.isAtomicReq()) {
           ATOMIC_DATA_LEN.toLong
@@ -258,7 +258,7 @@ class QPTest extends AnyFunSuite {
             WorkReqOpCode.RDMA_READ
           } else {
             // TODO: check if RQ supports atomic
-            WorkReqSim.randomSendWriteReadAtomicOpCode()
+            WorkReqSim.randomRdmaReqOpCode()
           }
 //        println(
 //          f"${simTime()} time: WR opcode=${workReqOpCode}, pktNum=${pktNum}, totalFragNum=${totalFragNum}, psnStart=${psnStart}, totalLenBytes=${totalLenBytes}"
diff --git a/rocev2/test/src/rdma/RecvQTest.scala b/rocev2/test/src/rdma/RecvQTest.scala
index ff7a28d..dfa16ce 100644
--- a/rocev2/test/src/rdma/RecvQTest.scala
+++ b/rocev2/test/src/rdma/RecvQTest.scala
@@ -650,12 +650,10 @@ class ReqAddrInfoExtractorTest extends AnyFunSuite {
   val simCfg = SimConfig.allOptimisation.withWave
     .compile(new ReqAddrInfoExtractor(busWidth))
 
-  def testFunc(inputHasNak: Boolean): Unit =
+  def testFunc(inputHasNak: Boolean, retryNakOrFatalNak: Boolean): Unit =
     simCfg.doSim { dut =>
       dut.clockDomain.forkStimulus(SIM_CYCLE_TIME)
 
-//      dut.io.qpAttr.pmtu #= pmtuLen.id
-//      dut.io.rxQCtrl.stateErrFlush #= false
       QpCtrlSim.assignDefaultQpAttrAndFlush(
         dut.io.qpAttr,
         pmtuLen,
@@ -740,11 +738,21 @@ class ReqAddrInfoExtractorTest extends AnyFunSuite {
 
           dut.io.rx.reqWithRxBuf.rxBufValid #= opcode.needRxBuf()
           dut.io.rx.reqWithRxBuf.rxBuf.lenBytes #= payloadLenBytes
-          dut.io.rx.reqWithRxBuf.pktFrag.bth.ackreq #=
-            opcode.isLastOrOnlyReqPkt()
+          if (inputHasNak) {
+            // If hasNak is true, ackReq does not matter
+            dut.io.rx.reqWithRxBuf.pktFrag.bth.ackreq #= false
+          } else {
+            // If normal, ackReq for the first, the last or the only request packet
+            dut.io.rx.reqWithRxBuf.pktFrag.bth.ackreq #= opcode
+              .isFirstOrOnlyReqPkt() || opcode.isLastOrOnlyReqPkt()
+          }
           dut.io.rx.reqWithRxBuf.hasNak #= inputHasNak
           if (inputHasNak) {
-            dut.io.rx.reqWithRxBuf.ackAeth.setAsRnrNak()
+            if (retryNakOrFatalNak) {
+              dut.io.rx.reqWithRxBuf.ackAeth.setAsRnrNak()
+            } else {
+              dut.io.rx.reqWithRxBuf.ackAeth.setAsInvReqNak()
+            }
             dut.io.rx.reqWithRxBuf.rxBufValid #= false
           }
 //        println(
@@ -758,9 +766,12 @@ class ReqAddrInfoExtractorTest extends AnyFunSuite {
         val opcode = OpCode(dut.io.rx.reqWithRxBuf.pktFrag.bth.opcodeFull.toInt)
         val isLastFrag = dut.io.rx.reqWithRxBuf.last.toBoolean
         val pktLenBytes = dut.io.rx.reqWithRxBuf.rxBuf.lenBytes.toLong
+//        val ackType = AckTypeSim.decodeFromAeth(dut.io.rx.reqWithRxBuf.ackAeth)
 //        val (_, _, pktLen) = RethSim.extract(pktFragData, busWidth)
 
-//        println(f"${simTime()} time: PSN=${psn}, opcode=${opcode}, ackReq=${ackReq}, isLastFrag=${isLastFrag}, pktLenBytes=${pktLenBytes}%X")
+//        println(
+//          f"${simTime()} time: input PSN=${psn}, opcode=${opcode}, ackReq=${ackReq}, isLastFrag=${isLastFrag}, pktLenBytes=${pktLenBytes}%X"
+//        )
         inputQueue.enqueue(
           (
             psn,
@@ -773,10 +784,16 @@ class ReqAddrInfoExtractorTest extends AnyFunSuite {
             isLastFrag
           )
         )
-        if (
-          isLastFrag && (inputHasNak || ackReq ||
-            opcode.isReadReqPkt() || opcode.isAtomicReqPkt())
-        ) {
+
+        val ackCond =
+          !inputHasNak && (ackReq || opcode.isLastOrOnlyReqPkt() || opcode
+            .isReadReqPkt() || opcode.isAtomicReqPkt())
+        val retryNakCond =
+          inputHasNak && retryNakOrFatalNak && opcode.isFirstOrOnlyReqPkt()
+        val fatalNakCond =
+          inputHasNak && !retryNakOrFatalNak && opcode.isLastOrOnlyReqPkt()
+
+        if (isLastFrag && (ackCond || retryNakCond || fatalNakCond)) {
           val singlePktNum = 1
           val pktNum = if (opcode.isReadReqPkt()) {
             MiscUtils.computePktNum(pktLenBytes, pmtuLen)
@@ -787,7 +804,10 @@ class ReqAddrInfoExtractorTest extends AnyFunSuite {
           inputPsnRangeQueue.enqueue(
             (opcode, psn, psnEnd)
           )
-//          println(f"${simTime()} time: opcode=${opcode}, psn=${psn}, psnEnd=${psnEnd}")
+
+//          println(
+//            f"${simTime()} time: opcode=${opcode}, psn=${psn}%X, psnEnd=${psnEnd}, ackCond=${ackCond}, ackReq=${ackReq}"
+//          )
         }
       }
 
@@ -796,24 +816,34 @@ class ReqAddrInfoExtractorTest extends AnyFunSuite {
         dut.clockDomain
       )
       onStreamFire(dut.io.tx.reqWithRxBufAndVirtualAddrInfo, dut.clockDomain) {
+        val psn = dut.io.tx.reqWithRxBufAndVirtualAddrInfo.pktFrag.bth.psn.toInt
         val pktFragData =
           dut.io.tx.reqWithRxBufAndVirtualAddrInfo.pktFrag.data.toBigInt
         val opcode =
           OpCode(
             dut.io.tx.reqWithRxBufAndVirtualAddrInfo.pktFrag.bth.opcodeFull.toInt
           )
+        val ackReq =
+          dut.io.tx.reqWithRxBufAndVirtualAddrInfo.pktFrag.bth.ackreq.toBoolean
+        val pktLenBytes =
+          dut.io.tx.reqWithRxBufAndVirtualAddrInfo.virtualAddrInfo.dlen.toLong
+        val isLastFrag = dut.io.tx.reqWithRxBufAndVirtualAddrInfo.last.toBoolean
         outputQueue.enqueue(
           (
-            dut.io.tx.reqWithRxBufAndVirtualAddrInfo.pktFrag.bth.psn.toInt,
+            psn,
             opcode,
-            dut.io.tx.reqWithRxBufAndVirtualAddrInfo.pktFrag.bth.ackreq.toBoolean,
+            ackReq,
             pktFragData,
-            dut.io.tx.reqWithRxBufAndVirtualAddrInfo.virtualAddrInfo.dlen.toLong,
+            pktLenBytes,
             dut.io.tx.reqWithRxBufAndVirtualAddrInfo.rxBufValid.toBoolean,
             dut.io.tx.reqWithRxBufAndVirtualAddrInfo.hasNak.toBoolean,
-            dut.io.tx.reqWithRxBufAndVirtualAddrInfo.last.toBoolean
+            isLastFrag
           )
         )
+
+//        println(
+//          f"${simTime()} time: output PSN=${psn}, opcode=${opcode}, ackReq=${ackReq}, isLastFrag=${isLastFrag}, pktLenBytes=${pktLenBytes}%X"
+//        )
       }
 
       streamSlaveRandomizer(dut.io.rqOutPsnRangeFifoPush, dut.clockDomain)
@@ -826,29 +856,35 @@ class ReqAddrInfoExtractorTest extends AnyFunSuite {
             dut.io.rqOutPsnRangeFifoPush.end.toInt
           )
         )
-//        println(f"${simTime()} time: opcode=${opcode}, dut.io.rqOutPsnRangeFifoPush.start=${dut.io.rqOutPsnRangeFifoPush.start.toInt}, dut.io.rqOutPsnRangeFifoPush.end=${dut.io.rqOutPsnRangeFifoPush.end.toInt}")
+
+//        println(
+//          f"${simTime()} time: opcode=${opcode}, dut.io.rqOutPsnRangeFifoPush.start=${dut.io.rqOutPsnRangeFifoPush.start.toInt}, dut.io.rqOutPsnRangeFifoPush.end=${dut.io.rqOutPsnRangeFifoPush.end.toInt}"
+//        )
       }
 
-      MiscUtils.checkExpectedOutputMatch(
+      MiscUtils.checkExpectedOutputMatchAlways(
         dut.clockDomain,
-        inputQueue,
-        outputQueue,
-        MATCH_CNT
+        inputPsnRangeQueue,
+        outputPsnRangeQueue
       )
       MiscUtils.checkExpectedOutputMatch(
         dut.clockDomain,
-        inputPsnRangeQueue,
-        outputPsnRangeQueue,
+        inputQueue,
+        outputQueue,
         MATCH_CNT
       )
     }
 
   test("ReqAddrInfoExtractor normal case") {
-    testFunc(inputHasNak = false)
+    testFunc(inputHasNak = false, retryNakOrFatalNak = false)
+  }
+
+  test("ReqAddrInfoExtractor retry NAK case") {
+    testFunc(inputHasNak = true, retryNakOrFatalNak = true)
   }
 
-  test("ReqAddrInfoExtractor invalid input case") {
-    testFunc(inputHasNak = true)
+  test("ReqAddrInfoExtractor fatal NAK case") {
+    testFunc(inputHasNak = true, retryNakOrFatalNak = false)
   }
 }
 
@@ -1394,7 +1430,7 @@ class ReqSplitterAndNakGenTest extends AnyFunSuite {
         val pktNum = pktNumItr.next()
         val psnStart = psnStartItr.next()
         val payloadLenBytes = payloadLenItr.next()
-        val workReqOpCode = WorkReqSim.randomSendWriteReadAtomicOpCode()
+        val workReqOpCode = WorkReqSim.randomRdmaReqOpCode()
 //        println(
 //          f"${simTime()} time: WR opcode=${workReqOpCode}, pktNum=${pktNum}, totalFragNum=${totalFragNum}, psnStart=${psnStart}, totalLenBytes=${totalLenBytes}"
 //        )
@@ -2063,8 +2099,6 @@ class SendWriteRespGeneratorTest extends AnyFunSuite {
   def testFunc(hasNak: Boolean): Unit = simCfg.doSim { dut =>
     dut.clockDomain.forkStimulus(SIM_CYCLE_TIME)
 
-//    dut.io.qpAttr.pmtu #= pmtuLen.id
-//    dut.io.rxQCtrl.stateErrFlush #= false
     QpCtrlSim.assignDefaultQpAttrAndFlush(
       dut.io.qpAttr,
       pmtuLen,
@@ -2082,15 +2116,6 @@ class SendWriteRespGeneratorTest extends AnyFunSuite {
     val expectedAckQueue =
       mutable.Queue[(PSN, SpinalEnumElement[AckType.type])]()
     val outputAckQueue = mutable.Queue[(PSN, SpinalEnumElement[AckType.type])]()
-//    val outputWorkCompQueue =
-//      mutable.Queue[
-//        (
-//            SpinalEnumElement[WorkCompOpCode.type],
-//            SpinalEnumElement[WorkCompStatus.type],
-//            WorkReqId,
-//            PktLen
-//        )
-//      ]()
 
     RdmaDataPktSim.pktFragStreamMasterDriver(
       dut.io.rx.reqWithRxBufAndDmaInfoWithLenCheck,
@@ -2131,7 +2156,8 @@ class SendWriteRespGeneratorTest extends AnyFunSuite {
         dut.io.rx.reqWithRxBufAndDmaInfoWithLenCheck.rxBuf.lenBytes #= payloadLenBytes
         dut.io.rx.reqWithRxBufAndDmaInfoWithLenCheck.dmaInfo.dlen #= payloadLenBytes
 
-        dut.io.rx.reqWithRxBufAndDmaInfoWithLenCheck.reqTotalLenValid #= fragLast
+        // If hasNak is true, reqTotalLenValid should be false
+        dut.io.rx.reqWithRxBufAndDmaInfoWithLenCheck.reqTotalLenValid #= fragLast && !hasNak
         dut.io.rx.reqWithRxBufAndDmaInfoWithLenCheck.hasNak #= hasNak
         dut.io.rx.reqWithRxBufAndDmaInfoWithLenCheck.rxBufValid #= opcode
           .needRxBuf()
@@ -3497,7 +3523,7 @@ class RqOutTest extends AnyFunSuite {
       val workReqOpCode = if (readRespOnly) {
         WorkReqOpCode.RDMA_READ
       } else {
-        WorkReqSim.randomSendWriteReadAtomicOpCode()
+        WorkReqSim.randomRdmaReqOpCode()
 //        WorkReqOpCode.RDMA_WRITE_WITH_IMM
       }
       val psnEnd = psnStart +% (pktNum - 1)
@@ -3982,7 +4008,7 @@ class RecvQTest extends AnyFunSuite {
       dut.reqCommCheck.io.tx.checkRst.hasNak.simPublic()
       dut.reqCommCheck.clearFlushStage.isReadAtomicRstCacheFull.simPublic()
       dut.reqCommCheck.io.txDupReq.pktFrag.valid.simPublic()
-//      dut.reqCommCheck.checkStage.pendingReqNum.simPublic()
+      //      dut.reqCommCheck.checkStage.pendingReqNum.simPublic()
 
       dut.reqRnrCheck.io.tx.reqWithRxBuf.valid.simPublic()
       dut.reqRnrCheck.io.tx.reqWithRxBuf.hasNak.simPublic()
@@ -4028,10 +4054,6 @@ class RecvQTest extends AnyFunSuite {
     // TODO: check RQ support empty requests
   }
 
-  test("RecvQ fatal error case") {
-    // TODO: check RQ output fatal error
-  }
-
   test("RecvQ send/write only normal case") {
     normalTestFunc(allKindReq = false, sendWriteOrReadAtomic = true)
   }
@@ -4059,7 +4081,7 @@ class RecvQTest extends AnyFunSuite {
       dut.io.rxQCtrl
     )
 
-    val recvWorkReqIdQueue = mutable.Queue[WorkReqId]()
+//    val recvWorkReqIdQueue = mutable.Queue[WorkReqId]()
     val expectedWorkCompQueue =
       mutable.Queue[(WorkReqId, SpinalEnumElement[WorkCompStatus.type])]()
     val outputWorkCompQueue =
@@ -4162,9 +4184,9 @@ class RecvQTest extends AnyFunSuite {
         getRdmaPktDataFunc = (r: RdmaDataPkt) => r,
         pendingReqNumExceed = {
           // TODO: refactor pendingReqNumExceed logic
-//            val isExceeded = dut.reqCommCheck.checkStage.pendingReqNum.toInt + 1 >=
-//              QpCtrlSim.getMaxPendingReadAtomicWorkReqNum(dut.io.qpAttr)
-//            isExceeded
+//          val isExceeded = dut.reqCommCheck.checkStage.pendingReqNum.toInt + 1 >=
+//            QpCtrlSim.getMaxPendingReadAtomicWorkReqNum(dut.io.qpAttr)
+//          isExceeded
           false
         },
         readAtomicRstCacheFull = {
@@ -4247,7 +4269,7 @@ class RecvQTest extends AnyFunSuite {
         readAtomicRstCacheFull = {
           // TODO: refactor readAtomicRstCacheFull logic
           // dut.io.rx.pktFrag.valid must be false when readAtomicRstCacheFull is true
-//          dut.reqCommCheck.checkStage.isReadAtomicRstCacheFull.toBoolean
+          //          dut.reqCommCheck.checkStage.isReadAtomicRstCacheFull.toBoolean
           val isFull =
             dut.readAtomicRstCache.io.occupancy.toInt + 1 >= QpCtrlSim
               .getMaxPendingReadAtomicWorkReqNum(dut.io.qpAttr)
@@ -4317,7 +4339,7 @@ class RecvQTest extends AnyFunSuite {
       // Just random generate RR
     }
     onStreamFire(dut.io.rxWorkReq, dut.clockDomain) {
-      recvWorkReqIdQueue.enqueue(dut.io.rxWorkReq.id.toBigInt)
+//      recvWorkReqIdQueue.enqueue(dut.io.rxWorkReq.id.toBigInt)
       expectedWorkCompQueue.enqueue(
         (dut.io.rxWorkReq.id.toBigInt, WorkCompStatus.SUCCESS)
       )
@@ -4349,11 +4371,12 @@ class RecvQTest extends AnyFunSuite {
 
     streamSlaveAlwaysReady(dut.io.sendWriteWorkComp, dut.clockDomain)
     onStreamFire(dut.io.sendWriteWorkComp, dut.clockDomain) {
+      val workCompOpCode = dut.io.sendWriteWorkComp.opcode.toEnum
       List(
         WorkCompOpCode.RECV,
         WorkCompOpCode.RECV_RDMA_WITH_IMM
-      ) should contain(dut.io.sendWriteWorkComp.opcode.toEnum) withClue
-        f"${simTime()} time: dut.io.sendWriteWorkComp.opcode=${dut.io.sendWriteWorkComp.opcode.toEnum} should be RECV or RECV_RDMA_WITH_IMM"
+      ) should contain(workCompOpCode) withClue
+        f"${simTime()} time: dut.io.sendWriteWorkComp.opcode=${workCompOpCode} should be RECV or RECV_RDMA_WITH_IMM"
       outputWorkCompQueue.enqueue(
         (
           dut.io.sendWriteWorkComp.id.toBigInt,
@@ -4511,9 +4534,9 @@ class RecvQTest extends AnyFunSuite {
                   _, // headerLenBytes,
                   opcode
               ) =>
-//                  println(
-//                    f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, fragLast=${fragLast}"
-//                  )
+                //                  println(
+                //                    f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, fragLast=${fragLast}"
+                //                  )
                 normalReqQueue.enqueue(
                   (
                     psn,
@@ -4551,9 +4574,10 @@ class RecvQTest extends AnyFunSuite {
                   _, // headerLenBytes,
                   opcode
               ) =>
-//          println(
-//            f"${simTime()} time: PSN=${readAtomicReqPsn}%X, opcode=${opcode}, pktNum=${pktNum}%X, payloadLenBytes=${payloadLenBytes}%X"
-//          )
+                //                println(
+                //                  f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, pktNum=${pktNum}%X, payloadLenBytes=${payloadLenBytes}%X"
+                //                )
+
                 normalReqQueue.enqueue(
                   (
                     psn,
@@ -4598,13 +4622,13 @@ class RecvQTest extends AnyFunSuite {
             expectedAtomicRespQueue.appendAll(normalAtomicRespQueue)
 
             dut.clockDomain.waitSamplingWhere {
-//              println(f"${simTime()} time: expectedReadRespQueue.size=${expectedReadRespQueue.size}, expectedAtomicRespQueue.size=${expectedAtomicRespQueue.size}")
+              //              println(f"${simTime()} time: expectedReadRespQueue.size=${expectedReadRespQueue.size}, expectedAtomicRespQueue.size=${expectedAtomicRespQueue.size}")
               expectedReadRespQueue.isEmpty && expectedAtomicRespQueue.isEmpty
             }
           }
-//          println(
-//            f"${simTime()} time: normal responses done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, start retry"
-//          )
+          //          println(
+          //            f"${simTime()} time: normal responses done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, start retry"
+          //          )
 
           val ePsn = dut.io.qpAttr.epsn.toInt
           val rqOutPsn = dut.io.qpAttr.rqOutPsn.toInt
@@ -4631,7 +4655,7 @@ class RecvQTest extends AnyFunSuite {
             expectedAtomicRespQueue.appendAll(normalAtomicRespQueue)
 
             dut.clockDomain.waitSamplingWhere {
-//              println(f"${simTime()} time: expectedReadRespQueue.size=${expectedReadRespQueue.size}, expectedAtomicRespQueue.size=${expectedAtomicRespQueue.size}")
+              //              println(f"${simTime()} time: expectedReadRespQueue.size=${expectedReadRespQueue.size}, expectedAtomicRespQueue.size=${expectedAtomicRespQueue.size}")
               expectedReadRespQueue.isEmpty && expectedAtomicRespQueue.isEmpty
             }
           }
@@ -4640,9 +4664,9 @@ class RecvQTest extends AnyFunSuite {
           dut.io.qpAttr.rqOutPsn.toInt shouldBe rqOutPsn withClue
             f"${simTime()} time: after retry, rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt} should remain @ ${rqOutPsn}"
 
-//          println(
-//            f"${simTime()} time: retry responses done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, finish retry"
-//          )
+          //          println(
+          //            f"${simTime()} time: retry responses done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, finish retry"
+          //          )
           normalReqQueue.clear()
           normalAckQueue.clear()
           normalReadRespQueue.clear()
@@ -4676,13 +4700,13 @@ class RecvQTest extends AnyFunSuite {
           pktFrag.mty #= mty
           pktFrag.last #= fragLast
 
-//          if (opcode.isReadReqPkt()) {
-//            val (virtualAddr, rKey, pktLen) =
-//              RethSim.extract(pktFragData, busWidth)
-//            println(
-//              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, pktLen=${pktLen}%X"
-//            )
-//          }
+          //          if (opcode.isReadReqPkt()) {
+          //            val (virtualAddr, rKey, pktLen) =
+          //              RethSim.extract(pktFragData, busWidth)
+          //            println(
+          //              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, pktLen=${pktLen}%X"
+          //            )
+          //          }
 
           val reqValid = true
           reqValid
@@ -4690,12 +4714,12 @@ class RecvQTest extends AnyFunSuite {
       )
 
       onStreamFire(dut.io.rx.pktFrag, dut.clockDomain) {
-//        val opcode = OpCode(dut.io.rx.pktFrag.bth.opcodeFull.toInt)
-//        val ackreq = dut.io.rx.pktFrag.bth.ackreq.toBoolean
-//        val fragLast = dut.io.rx.pktFrag.last.toBoolean
-//        println(
-//          f"${simTime()} time: input opcode=${opcode}, PSN=${dut.io.rx.pktFrag.bth.psn.toInt}%X, ePSN=${dut.io.qpAttr.epsn.toInt}%X, fragLast=${fragLast}, ackreq=${ackreq}"
-//        )
+        //        val opcode = OpCode(dut.io.rx.pktFrag.bth.opcodeFull.toInt)
+        //        val ackreq = dut.io.rx.pktFrag.bth.ackreq.toBoolean
+        //        val fragLast = dut.io.rx.pktFrag.last.toBoolean
+        //        println(
+        //          f"${simTime()} time: input opcode=${opcode}, PSN=${dut.io.rx.pktFrag.bth.psn.toInt}%X, ePSN=${dut.io.qpAttr.epsn.toInt}%X, fragLast=${fragLast}, ackreq=${ackreq}"
+        //        )
       }
 
       streamMasterDriverAlwaysValid(dut.io.rxWorkReq, dut.clockDomain) {
@@ -4730,9 +4754,9 @@ class RecvQTest extends AnyFunSuite {
           SpinalExit(f"${simTime()} time: invalid opcode=${opcode}, PSN=${psn}")
         }
 
-//        println(
-//          f"${simTime()} time: response: opcode=${opcode}, PSN=${psn}"
-//        )
+        //        println(
+        //          f"${simTime()} time: response: opcode=${opcode}, PSN=${psn}"
+        //        )
       }
 
       // TODO: support atomic requests
@@ -4744,13 +4768,13 @@ class RecvQTest extends AnyFunSuite {
       )
 
       if (sendWriteOrReadAtomic) {
-//          MiscUtils.checkCondChangeOnceAndHoldAfterwards(
-//            dut.clockDomain,
-//            cond =
-//              dut.io.rx.pktFrag.valid.toBoolean && dut.io.rx.pktFrag.ready.toBoolean,
-//            clue =
-//              f"${simTime()} time: dut.io.rx.pktFrag.fire=${dut.io.rx.pktFrag.valid.toBoolean && dut.io.rx.pktFrag.ready.toBoolean} should be true always when sendWriteOrReadAtomic=${sendWriteOrReadAtomic}"
-//          )
+        //          MiscUtils.checkCondChangeOnceAndHoldAfterwards(
+        //            dut.clockDomain,
+        //            cond =
+        //              dut.io.rx.pktFrag.valid.toBoolean && dut.io.rx.pktFrag.ready.toBoolean,
+        //            clue =
+        //              f"${simTime()} time: dut.io.rx.pktFrag.fire=${dut.io.rx.pktFrag.valid.toBoolean && dut.io.rx.pktFrag.ready.toBoolean} should be true always when sendWriteOrReadAtomic=${sendWriteOrReadAtomic}"
+        //          )
       } else { // For read/atomic requests
         MiscUtils.checkConditionAlwaysHold(dut.clockDomain)(
           cond = !dut.io.sendWriteWorkComp.valid.toBoolean,
@@ -4759,7 +4783,7 @@ class RecvQTest extends AnyFunSuite {
         )
       }
 
-//      waitUntil(req4RetryQueue.size > MATCH_CNT)
+      //      waitUntil(req4RetryQueue.size > MATCH_CNT)
 
       if (sendWriteOrReadAtomic) {
         MiscUtils.checkExpectedOutputMatch(
@@ -4866,9 +4890,9 @@ class RecvQTest extends AnyFunSuite {
                 _, // headerLenBytes,
                 opcode
             ) =>
-//              println(
-//                f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, respPktNum=${respPktNum}%X"
-//              )
+              //              println(
+              //                f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, respPktNum=${respPktNum}%X"
+              //              )
 
               val pktFragData = dut.io.rx.pktFrag.data.toBigInt
               normalReqQueue.enqueue(
@@ -4955,9 +4979,9 @@ class RecvQTest extends AnyFunSuite {
                 if (isWrongRetryAtomicReq) {
                   val (virtualAddr, rKey, atomicSwap, atomicComp) =
                     AtomicEthSim.extract(pktFragData, busWidth)
-//                  println(
-//                    f"${simTime()} time: virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, atomicSwap=${atomicSwap}%X, atomicComp=${atomicComp}%X"
-//                  )
+                  //                  println(
+                  //                    f"${simTime()} time: virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, atomicSwap=${atomicSwap}%X, atomicComp=${atomicComp}%X"
+                  //                  )
 
                   // Duplicated atomic requests not match original ones have no responses
                   val wrongRetryAtomicReqPktFragData = AtomicEthSim.set(
@@ -5006,7 +5030,7 @@ class RecvQTest extends AnyFunSuite {
           expectedAtomicRespQueue.appendAll(normalAtomicRespQueue)
 
           dut.clockDomain.waitSamplingWhere {
-//              println(f"${simTime()} time: expectedReadRespQueue.size=${expectedReadRespQueue.size}, expectedAtomicRespQueue.size=${expectedAtomicRespQueue.size}")
+            //            println(f"${simTime()} time: expectedReadRespQueue.size=${expectedReadRespQueue.size}, expectedAtomicRespQueue.size=${expectedAtomicRespQueue.size}")
             expectedReadRespQueue.isEmpty && expectedAtomicRespQueue.isEmpty
           }
 
@@ -5027,7 +5051,7 @@ class RecvQTest extends AnyFunSuite {
           expectedAtomicRespQueue.appendAll(correctDupAtomicRespQueue)
 
           dut.clockDomain.waitSamplingWhere {
-//              println(f"${simTime()} time: expectedReadRespQueue.size=${expectedReadRespQueue.size}, expectedAtomicRespQueue.size=${expectedAtomicRespQueue.size}")
+            //            println(f"${simTime()} time: expectedReadRespQueue.size=${expectedReadRespQueue.size}, expectedAtomicRespQueue.size=${expectedAtomicRespQueue.size}")
             expectedReadRespQueue.isEmpty && expectedAtomicRespQueue.isEmpty
           }
 
@@ -5036,9 +5060,10 @@ class RecvQTest extends AnyFunSuite {
           dut.io.qpAttr.rqOutPsn.toInt shouldBe rqOutPsn withClue
             f"${simTime()} time: after retry, rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt} should remain @ ${rqOutPsn}"
 
-          println(
-            f"${simTime()} time: retry responses done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, finish retry"
-          )
+          //          println(
+          //            f"${simTime()} time: retry responses done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, finish retry"
+          //          )
+
           normalReqQueue.clear()
           partialRetryReadAndMaybeWrongRetryAtomicReqQueue.clear()
           normalReadRespQueue.clear()
@@ -5074,18 +5099,18 @@ class RecvQTest extends AnyFunSuite {
           pktFrag.mty #= mty
           pktFrag.last #= fragLast
 
-//          if (opcode.isReadReqPkt()) {
-//            val (virtualAddr, rKey, pktLen) =
-//              RethSim.extract(pktFragData, busWidth)
-//            println(
-//              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, pktLen=${pktLen}%X"
-//            )
-//          } else if (opcode.isAtomicReqPkt()) {
-//            val (virtualAddr, rKey, atomicSwap, atomicComp) = AtomicEthSim.extract(pktFragData, busWidth)
-//            println(
-//              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, atomicSwap=${atomicSwap}%X, atomicComp=${atomicComp}%X"
-//            )
-//          }
+          //          if (opcode.isReadReqPkt()) {
+          //            val (virtualAddr, rKey, pktLen) =
+          //              RethSim.extract(pktFragData, busWidth)
+          //            println(
+          //              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, pktLen=${pktLen}%X"
+          //            )
+          //          } else if (opcode.isAtomicReqPkt()) {
+          //            val (virtualAddr, rKey, atomicSwap, atomicComp) = AtomicEthSim.extract(pktFragData, busWidth)
+          //            println(
+          //              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, atomicSwap=${atomicSwap}%X, atomicComp=${atomicComp}%X"
+          //            )
+          //          }
 
           val reqValid = true
           reqValid
@@ -5093,12 +5118,12 @@ class RecvQTest extends AnyFunSuite {
       )
 
       onStreamFire(dut.io.rx.pktFrag, dut.clockDomain) {
-//        val opcode = OpCode(dut.io.rx.pktFrag.bth.opcodeFull.toInt)
-//        val ackreq = dut.io.rx.pktFrag.bth.ackreq.toBoolean
-//        val fragLast = dut.io.rx.pktFrag.last.toBoolean
-//        println(
-//          f"${simTime()} time: input opcode=${opcode}, PSN=${dut.io.rx.pktFrag.bth.psn.toInt}%X, ePSN=${dut.io.qpAttr.epsn.toInt}%X, fragLast=${fragLast}, ackreq=${ackreq}"
-//        )
+        //        val opcode = OpCode(dut.io.rx.pktFrag.bth.opcodeFull.toInt)
+        //        val ackreq = dut.io.rx.pktFrag.bth.ackreq.toBoolean
+        //        val fragLast = dut.io.rx.pktFrag.last.toBoolean
+        //        println(
+        //          f"${simTime()} time: input opcode=${opcode}, PSN=${dut.io.rx.pktFrag.bth.psn.toInt}%X, ePSN=${dut.io.qpAttr.epsn.toInt}%X, fragLast=${fragLast}, ackreq=${ackreq}"
+        //        )
       }
 
       streamSlaveAlwaysReady(dut.io.tx.pktFrag, dut.clockDomain)
@@ -5120,9 +5145,9 @@ class RecvQTest extends AnyFunSuite {
           SpinalExit(f"${simTime()} time: invalid opcode=${opcode}, PSN=${psn}")
         }
 
-//        println(
-//          f"${simTime()} time: response: PSN=${psn}%X, opcode=${opcode}, fragLast=${fragLast}"
-//        )
+        //        println(
+        //          f"${simTime()} time: response: PSN=${psn}%X, opcode=${opcode}, fragLast=${fragLast}"
+        //        )
       }
 
       // TODO: support atomic requests
@@ -5146,14 +5171,14 @@ class RecvQTest extends AnyFunSuite {
       )
     }
 
-  test("RecvQ RNR retry case") {
-    retryTestFunc(rnrRetryOrSeqNakRetry = true)
-  }
-
   test("RecvQ SEQ NAK retry case") {
     retryTestFunc(rnrRetryOrSeqNakRetry = false)
   }
 
+  test("RecvQ RNR retry case") {
+    retryTestFunc(rnrRetryOrSeqNakRetry = true)
+  }
+
   def retryTestFunc(rnrRetryOrSeqNakRetry: Boolean): Unit =
     simCfg.doSim { dut =>
       dut.clockDomain.forkStimulus(SIM_CYCLE_TIME)
@@ -5291,40 +5316,14 @@ class RecvQTest extends AnyFunSuite {
             bothNormalAndRetryReqQueue.appendAll(sendWriteImmReqWrongPsnQueue)
           }
           expectedAckQueue.appendAll(retryAckQueue)
-          /*
-          val needRetry = if (rnrRetryOrSeqNakRetry) { // RNR retry case
-            bothNormalAndRetryReqQueue.appendAll(sendWriteImmReqQueue)
-            val sendWriteReqOpCode = sendWriteImmReqQueue.last._2
-            if (sendWriteReqOpCode.needRxBuf()) {
-              // Append retry responses
-              expectedAckQueue.appendAll(retryAckQueue)
-              true // Need retry
-            } else {
-              // Append normal write responses
-              expectedAckQueue.appendAll(normalAckQueue)
-              false // No need retry
-            }
-          } else { // SEQ NAK retry case
-            val sendWriteReqWrongPsnQueue = sendWriteImmReqQueue.map { reqData =>
-              val (psn, opcode, padCnt, ackreq, fragLast, pktFragData, mty) =
-                reqData
-              val wrongPsn = psn +% 1 // Make wrong PSN for SEQ NAK
-              (wrongPsn, opcode, padCnt, ackreq, fragLast, pktFragData, mty)
-            }
-            bothNormalAndRetryReqQueue.appendAll(sendWriteReqWrongPsnQueue)
-            // Append retry responses
-            expectedAckQueue.appendAll(retryAckQueue)
-            true // Need retry
-          }
-           */
+
           // Wait until requests processed
           dut.clockDomain.waitSamplingWhere(
             bothNormalAndRetryReqQueue.isEmpty && expectedAckQueue.isEmpty
           )
-
-          println(
-            f"${simTime()} time: requests with retry NAK done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, start retry"
-          )
+//          println(
+//            f"${simTime()} time: requests with retry NAK done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, start retry"
+//          )
 
           // Recv WR ready
           dut.io.rxWorkReq.randomize()
@@ -5363,29 +5362,19 @@ class RecvQTest extends AnyFunSuite {
               sendWriteImmReqQueueWithoutDupReq
             )
 
-            println(
-              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, pktLen=${pktLen}%X, remainingPktLen=${remainingPktLen}%X "
-            )
+//            println(
+//              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, pktLen=${pktLen}%X, remainingPktLen=${remainingPktLen}%X "
+//            )
           }
 
-//          val ePsn = dut.io.qpAttr.epsn.toInt
-//          val rqOutPsn = dut.io.qpAttr.rqOutPsn.toInt
-//        val latestReqPsn = normalAckQueue.last._1
-//        latestReqPsn shouldBe dut.io.qpAttr.rqOutPsn.toInt withClue
-//          f"${simTime()} time: latestReqPsn=${latestReqPsn}%X should == dut.io.qpAttr.rqOutPsn.toInt=${dut.io.qpAttr.rqOutPsn.toInt}%X"
-
           // Append normal send/write responses
           expectedAckQueue.appendAll(normalAckQueue)
 
           dut.clockDomain.waitSamplingWhere(expectedAckQueue.isEmpty)
-//          dut.io.qpAttr.epsn.toInt shouldBe ePsn withClue
-//            f"${simTime()} time: after retry, ePSN=${dut.io.qpAttr.epsn.toInt} should remain @ ${ePsn}"
-//          dut.io.qpAttr.rqOutPsn.toInt shouldBe rqOutPsn withClue
-//            f"${simTime()} time: after retry, rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt} should remain @ ${rqOutPsn}"
+//          println(
+//            f"${simTime()} time: retry requests done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, finish retry"
+//          )
 
-          println(
-            f"${simTime()} time: retry requests done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, finish retry"
-          )
           sendWriteImmReqQueue.clear()
           retryAckQueue.clear()
           normalAckQueue.clear()
@@ -5481,6 +5470,243 @@ class RecvQTest extends AnyFunSuite {
           val ackType = AckTypeSim.decodeFromCodeAndValue(aethCode, aethValue)
           outputAckQueue.enqueue((psn, ackType))
 
+          fragLast shouldBe true withClue
+            f"${simTime()} time: dut.io.tx.pktFrag.last=${fragLast} should be true"
+
+//          println(
+//            f"${simTime()} time: response PSN=${psn}%X, opcode=${opcode}, ackType=${ackType}, fragLast=${fragLast}"
+//          )
+        } else {
+          SpinalExit(
+            f"${simTime()} time: expect ACK, but opcode=${opcode}, PSN=${psn}"
+          )
+        }
+
+//        println(
+//          f"${simTime()} time: response: opcode=${opcode}, PSN=${psn}%X, fragLast=${fragLast}"
+//        )
+      }
+
+      // TODO: support atomic requests
+      MiscUtils.checkConditionAlwaysHold(dut.clockDomain)(
+        cond =
+          !(dut.io.dma.atomic.wr.req.valid.toBoolean || dut.io.dma.atomic.rd.req.valid.toBoolean),
+        clue =
+          f"${simTime()} time: dut.io.dma.atomic.wr.req.valid=${dut.io.dma.atomic.wr.req.valid.toBoolean}, dut.io.dma.atomic.rd.req.valid=${dut.io.dma.atomic.rd.req.valid.toBoolean} should be false"
+      )
+
+      MiscUtils.checkExpectedOutputMatchAlways(
+        dut.clockDomain,
+        expectedWorkCompQueue,
+        outputWorkCompQueue
+      )
+      MiscUtils.checkExpectedOutputMatch(
+        dut.clockDomain,
+        expectedAckQueue,
+        outputAckQueue,
+        MATCH_CNT
+      )
+    }
+
+  test("RecvQ remote access fatal NAK case") {
+    fatalErrorTestFunc()
+  }
+
+  def fatalErrorTestFunc(): Unit =
+    simCfg.doSim { dut =>
+      dut.clockDomain.forkStimulus(SIM_CYCLE_TIME)
+
+      QpCtrlSim.connectRecvQ(
+        dut.clockDomain,
+        pmtuLen,
+        dut.io.psnInc,
+        dut.io.notifier,
+        dut.io.qpAttr,
+        dut.io.rxQCtrl
+      )
+
+      val inputReqQueue = mutable.Queue[
+        (PSN, OpCode.Value, PadCnt, AckReq, FragLast, PktFragData, MTY)
+      ]()
+      val expectedAckQueue =
+        mutable.Queue[(PSN, SpinalEnumElement[AckType.type])]()
+      val outputAckQueue =
+        mutable.Queue[(PSN, SpinalEnumElement[AckType.type])]()
+
+      val expectedWorkCompQueue =
+        mutable.Queue[(WorkReqId, SpinalEnumElement[WorkCompStatus.type])]()
+      val outputWorkCompQueue =
+        mutable.Queue[(WorkReqId, SpinalEnumElement[WorkCompStatus.type])]()
+      // Input to DUT
+      val _ = RqDmaBusSim.reqStreamAlwaysFireAndRespSuccess(
+        dut.io.dma,
+        dut.clockDomain,
+        busWidth
+      )
+
+      // AddrCache query failure, result in remote access NAK
+      val _ = AddrCacheSim.reqStreamAlwaysFireAndRespFailure(
+        dut.io.addrCacheRead,
+        dut.clockDomain
+      )
+
+      fork {
+        val pendingReqLimit = 1
+
+        // Retry control loop
+        while (true) {
+          QpCtrlSim.resetQpAttr2Normal(dut.io.qpAttr)
+          val expectedPsn = SimSettings.INIT_PSN
+
+          // Generate send/write requests
+          RdmaDataPktSim.rdmaReqPktFragStreamMasterGen(
+            dut.io.rx.pktFrag,
+            expectedPsn,
+            pendingReqLimit,
+            pmtuLen,
+            busWidth,
+            maxFragNum
+          ) {
+            (
+                psn,
+                _, // psnStart
+                fragLast,
+                _, // fragIdx,
+                _, // pktFragNum,
+                _, // pktIdx,
+                _, // reqPktNum,
+                _, // respPktNum,
+                _, // payloadLenBytes,
+                _, // headerLenBytes,
+                opcode
+            ) =>
+              inputReqQueue.enqueue(
+                (
+                  psn,
+                  opcode,
+                  dut.io.rx.pktFrag.bth.padCnt.toInt,
+                  dut.io.rx.pktFrag.bth.ackreq.toBoolean,
+                  fragLast,
+                  dut.io.rx.pktFrag.data.toBigInt,
+                  dut.io.rx.pktFrag.mty.toBigInt
+                )
+              )
+
+              if (opcode.isLastOrOnlyReqPkt() && fragLast) {
+                expectedAckQueue.enqueue((psn, AckType.NAK_RMT_ACC))
+              }
+//              println(
+//                f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, payloadLenBytes=${payloadLenBytes}%X, fragLast=${fragLast}"
+//              )
+          }
+
+//          sleep(0)
+//          println(
+//            f"${simTime()} time: qpAttr.state=${dut.io.qpAttr.state.toEnum}"
+//          )
+
+          // Wait until requests processed
+          dut.clockDomain.waitSamplingWhere(
+            inputReqQueue.isEmpty && expectedAckQueue.isEmpty
+          )
+//          println(
+//            f"${simTime()} time: requests with fatal NAK done, dut.io.qpAttr.epsn=${dut.io.qpAttr.epsn.toInt}%X, dut.io.qpAttr.rqOutPsn=${dut.io.qpAttr.rqOutPsn.toInt}%X, start again"
+//          )
+
+          inputReqQueue.clear()
+          expectedAckQueue.clear()
+        }
+      }
+
+      streamMasterPayloadFromQueueNoRandomDelay(
+        dut.io.rx.pktFrag,
+        dut.clockDomain,
+        inputReqQueue,
+        payloadAssignFunc = (
+            pktFrag: Fragment[RdmaDataPkt],
+            payloadData: (
+                PSN,
+                OpCode.Value,
+                PadCnt,
+                AckReq,
+                FragLast,
+                PktFragData,
+                MTY
+            )
+        ) => {
+          val (psn, opcode, padCnt, ackreq, fragLast, pktFragData, mty) =
+            payloadData
+          pktFrag.bth.psn #= psn
+          pktFrag.bth.opcodeFull #= opcode.id
+          pktFrag.bth.padCnt #= padCnt
+          pktFrag.bth.ackreq #= ackreq
+          pktFrag.data #= pktFragData
+          pktFrag.mty #= mty
+          pktFrag.last #= fragLast
+
+//          if (opcode.isWriteReqPkt()) {
+//            val (virtualAddr, rKey, pktLen) =
+//              RethSim.extract(pktFragData, busWidth)
+//            println(
+//              f"${simTime()} time: PSN=${psn}%X, opcode=${opcode}, virtualAddr=${virtualAddr}%X, rKey=${rKey}%X, pktLen=${pktLen}%X"
+//            )
+//          }
+
+          val reqValid = true
+          reqValid
+        }
+      )
+
+      onStreamFire(dut.io.rx.pktFrag, dut.clockDomain) {
+//        val opcode = OpCode(dut.io.rx.pktFrag.bth.opcodeFull.toInt)
+//        val psn = dut.io.rx.pktFrag.bth.psn.toInt
+//        val fragLast = dut.io.rx.pktFrag.last.toBoolean
+//
+//        if (fragLast) {
+//          expectedAckQueue.enqueue((psn, AckType.NAK_RMT_ACC))
+//        }
+
+//        println(
+//          f"${simTime()} time: input opcode=${opcode}, PSN=${dut.io.rx.pktFrag.bth.psn.toInt}%X, ePSN=${dut.io.qpAttr.epsn.toInt}%X, fragLast=${fragLast}, ackreq=${ackreq}"
+//        )
+      }
+
+      streamMasterDriverAlwaysValid(dut.io.rxWorkReq, dut.clockDomain) {
+        // Just random generate RR
+      }
+      onStreamFire(dut.io.rxWorkReq, dut.clockDomain) {
+        expectedWorkCompQueue.enqueue(
+          (dut.io.rxWorkReq.id.toBigInt, WorkCompStatus.REM_ACCESS_ERR)
+        )
+      }
+
+      streamSlaveAlwaysReady(dut.io.sendWriteWorkComp, dut.clockDomain)
+      onStreamFire(dut.io.sendWriteWorkComp, dut.clockDomain) {
+        val workCompOpCode = dut.io.sendWriteWorkComp.opcode.toEnum
+        List(
+          WorkCompOpCode.RECV,
+          WorkCompOpCode.RECV_RDMA_WITH_IMM
+        ) should contain(workCompOpCode) withClue
+          f"${simTime()} time: dut.io.sendWriteWorkComp.opcode=${workCompOpCode} should be RECV or RECV_RDMA_WITH_IMM"
+        outputWorkCompQueue.enqueue(
+          (
+            dut.io.sendWriteWorkComp.id.toBigInt,
+            dut.io.sendWriteWorkComp.status.toEnum
+          )
+        )
+      }
+      streamSlaveAlwaysReady(dut.io.tx.pktFrag, dut.clockDomain)
+      onStreamFire(dut.io.tx.pktFrag, dut.clockDomain) {
+        val opcode = OpCode(dut.io.tx.pktFrag.bth.opcodeFull.toInt)
+        val psn = dut.io.tx.pktFrag.bth.psn.toInt
+        val fragLast = dut.io.tx.pktFrag.last.toBoolean
+
+        if (opcode.isAckRespPkt()) {
+          val (_, aethCode, aethValue, _) =
+            AethSim.extract(dut.io.tx.pktFrag.data.toBigInt, busWidth)
+          val ackType = AckTypeSim.decodeFromCodeAndValue(aethCode, aethValue)
+          outputAckQueue.enqueue((psn, ackType))
+
 //          println(
 //            f"${simTime()} time: response: PSN=${psn}%X, opcode=${opcode}, ackType=${ackType}, fragLast=${fragLast}"
 //          )
@@ -5488,7 +5714,9 @@ class RecvQTest extends AnyFunSuite {
           fragLast shouldBe true withClue
             f"${simTime()} time: dut.io.tx.pktFrag.last=${fragLast} should be true"
         } else {
-          SpinalExit(f"${simTime()} time: invalid opcode=${opcode}, PSN=${psn}")
+          SpinalExit(
+            f"${simTime()} time: expect NAK, but opcode=${opcode}, PSN=${psn}"
+          )
         }
 
 //        println(
@@ -5496,12 +5724,22 @@ class RecvQTest extends AnyFunSuite {
 //        )
       }
 
-      // TODO: support atomic requests
+      // No DMA operation
       MiscUtils.checkConditionAlwaysHold(dut.clockDomain)(
         cond =
           !(dut.io.dma.atomic.wr.req.valid.toBoolean || dut.io.dma.atomic.rd.req.valid.toBoolean),
         clue =
-          f"${simTime()} time: dut.io.dma.atomic.wr.req.valid or dut.io.dma.atomic.rd.req.valid=${dut.io.dma.atomic.wr.req.valid.toBoolean || dut.io.dma.atomic.rd.req.valid.toBoolean} should be false"
+          f"${simTime()} time: dut.io.dma.atomic.wr.req.valid=${dut.io.dma.atomic.wr.req.valid.toBoolean}, dut.io.dma.atomic.rd.req.valid=${dut.io.dma.atomic.rd.req.valid.toBoolean} should be false"
+      )
+      MiscUtils.checkConditionAlwaysHold(dut.clockDomain)(
+        cond = !dut.io.dma.sendWrite.req.valid.toBoolean,
+        clue =
+          f"${simTime()} time: dut.io.dma.sendWrite.req.valid=${dut.io.dma.sendWrite.req.valid.toBoolean} should be false"
+      )
+      MiscUtils.checkConditionAlwaysHold(dut.clockDomain)(
+        cond = !dut.io.dma.read.req.valid.toBoolean,
+        clue =
+          f"${simTime()} time: dut.io.dma.read.req.valid=${dut.io.dma.read.req.valid.toBoolean} should be false"
       )
 
       MiscUtils.checkExpectedOutputMatchAlways(
diff --git a/rocev2/test/src/rdma/RespHandlerTest.scala b/rocev2/test/src/rdma/RespHandlerTest.scala
index 6acfe26..d2346ec 100644
--- a/rocev2/test/src/rdma/RespHandlerTest.scala
+++ b/rocev2/test/src/rdma/RespHandlerTest.scala
@@ -693,7 +693,7 @@ class ReadAtomicRespVerifierAndFatalNakNotifierTest extends AnyFunSuite {
           val expectedWorkCompStatus = if (addrCacheQuerySuccess) {
             WorkCompStatus.SUCCESS
           } else {
-            WorkCompStatus.LOC_LEN_ERR
+            WorkCompStatus.LOC_ACCESS_ERR
           }
 
           inputWorkReqAndAckQueue.enqueue(
diff --git a/rocev2/test/src/rdma/RetryHandlerTest.scala b/rocev2/test/src/rdma/RetryHandlerTest.scala
index a2ba1d0..1ece743 100644
--- a/rocev2/test/src/rdma/RetryHandlerTest.scala
+++ b/rocev2/test/src/rdma/RetryHandlerTest.scala
@@ -69,7 +69,7 @@ class RetryHandlerTest extends AnyFunSuite {
       streamMasterDriver(dut.io.retryWorkReqIn, dut.clockDomain) {
         val curPsn = nextPsn
         dut.io.retryWorkReqIn.scanOutData.psnStart #= curPsn
-        val workReqOpCode = WorkReqSim.randomSendWriteReadAtomicOpCode()
+        val workReqOpCode = WorkReqSim.randomRdmaReqOpCode()
         dut.io.retryWorkReqIn.scanOutData.workReq.opcode #= workReqOpCode
         val pktLen = if (workReqOpCode.isAtomicReq()) {
           ATOMIC_DATA_LEN.toLong
diff --git a/rocev2/test/src/rdma/SendQTest.scala b/rocev2/test/src/rdma/SendQTest.scala
index 0638d55..2fe2e75 100644
--- a/rocev2/test/src/rdma/SendQTest.scala
+++ b/rocev2/test/src/rdma/SendQTest.scala
@@ -83,7 +83,7 @@ class WorkReqValidatorTest extends AnyFunSuite {
       val pktNum = MiscUtils.computePktNum(pktLen.toLong, pmtuLen)
 //      val psnEnd = psnStart +% (pktNum - 1)
 
-      val workReqOpCode = WorkReqSim.randomSendWriteReadAtomicOpCode()
+      val workReqOpCode = WorkReqSim.randomRdmaReqOpCode()
       dut.io.workReq.opcode #= workReqOpCode
       dut.io.workReq.lenBytes #= pktLen
       val noFlags = 0
@@ -110,7 +110,7 @@ class WorkReqValidatorTest extends AnyFunSuite {
       if (!normalOrErrorCase) { // Error case
         val workCompStatus = if (addrCacheQueryErrOrFlushErr) {
           // AddrCache query response error
-          WorkCompStatus.LOC_LEN_ERR
+          WorkCompStatus.LOC_ACCESS_ERR
         } else { // Flush error
           WorkCompStatus.WR_FLUSH_ERR
         }
@@ -341,7 +341,7 @@ class WorkReqCacheAndOutPsnRangeHandlerTest extends AnyFunSuite {
       dut.io.normalWorkReq
     }
     streamMasterDriver(inputWorkReqStream, dut.clockDomain) {
-      val workReqOpCode = WorkReqSim.randomSendWriteReadAtomicOpCode()
+      val workReqOpCode = WorkReqSim.randomRdmaReqOpCode()
       inputWorkReqStream.workReq.opcode #= workReqOpCode
       val workReqLen = inputWorkReqStream.workReq.lenBytes.toLong
       val pktNum = MiscUtils.computePktNum(workReqLen, pmtuLen)
@@ -747,7 +747,7 @@ class SqOutTest extends AnyFunSuite {
       val pktNum = pktNumItr.next()
       val psnStart = psnStartItr.next()
       val payloadLenBytes = payloadLenItr.next()
-      val workReqOpCode = WorkReqSim.randomSendWriteReadAtomicOpCode()
+      val workReqOpCode = WorkReqSim.randomRdmaReqOpCode()
       val psnEnd = psnStart +% (pktNum - 1)
 
       val isReadReq = workReqOpCode.isReadReq()
diff --git a/run.sh b/run.sh
index 91aecca..32ff7dd 100755
--- a/run.sh
+++ b/run.sh
@@ -76,9 +76,13 @@ $MILL rocev2.test.testSim rdma.ReadDmaReqInitiatorTest
 $MILL rocev2.test.testSim rdma.RqReadDmaRespHandlerTest
 $MILL rocev2.test.testSim rdma.ReadRespGeneratorTest
 $MILL rocev2.test.testSim rdma.RqOutTest
-# Slow test
-$MILL rocev2.test.testSim rdma.SendWriteRespGeneratorTest
-$MILL rocev2.test.testSim rdma.RqSendWriteWorkCompGeneratorTest
+$MILL rocev2.test.testSim rdma.SendWriteRespGeneratorTest # Slow test
+$MILL rocev2.test.testSim rdma.RqSendWriteWorkCompGeneratorTest # Slow test
+
+if [ "$CI_ENV" = "true" ]; then
+  rm -rf simWorkspace/ # Delete simulation data to save space for GitHub CI
+fi
+
 # E2E test
 $MILL rocev2.test.testSim rdma.RecvQTest