From 6d55545018ba40e57d50b115e30390ee41c795e4 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Sun, 7 Jan 2024 02:22:58 +0800 Subject: [PATCH 01/16] support SG for SQ --- src/DataTypes.bsv | 21 ++++++++++++++------- src/Settings.bsv | 2 +- src/Utils.bsv | 1 + 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index bc52236..b298f7f 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -34,6 +34,8 @@ typedef 8 QP_CAP_CNT_WIDTH; // Derived settings typedef AETH_VALUE_WIDTH TIMER_WIDTH; +typedef TAdd#(1, TLog#(MAX_SGE)) SGE_NUM_WIDTH; + // 12 + 4 + 28 = 44 bytes typedef TAdd#(TAdd#(BTH_BYTE_WIDTH, XRCETH_BYTE_WIDTH), ATOMIC_ETH_BYTE_WIDTH) HEADER_MAX_BYTE_LENGTH; @@ -80,6 +82,8 @@ typedef TSub#(TSub#(ADDR_WIDTH, TLB_CACHE_INDEX_WIDTH), PAGE_OFFSET_WIDTH) TLB_C typedef Bit#(DATA_BUS_WIDTH) DATA; typedef Bit#(DATA_BUS_BYTE_WIDTH) ByteEn; +typedef Bit#(SGE_NUM_WIDTH) ScatterGatherListIdx; + typedef Bit#(HEADER_MAX_DATA_WIDTH) HeaderData; typedef Bit#(HEADER_MAX_BYTE_EN_WIDTH) HeaderByteEn; typedef Bit#(HEADER_MAX_BYTE_NUM_WIDTH) HeaderByteNum; @@ -246,17 +250,17 @@ typedef struct { } DmaReadMetaData deriving(Bits, FShow); typedef struct { - DmaReqSrcType initiator; - QPN sqpn; + DmaReqSrcType initiator; // TODO: remove it + QPN sqpn; // TODO: remove it ADDR startAddr; PktLen len; - WorkReqID wrID; + WorkReqID wrID; // TODO: remove it } DmaReadReq deriving(Bits, FShow); typedef struct { - DmaReqSrcType initiator; - QPN sqpn; - WorkReqID wrID; + DmaReqSrcType initiator; // TODO: remove it + QPN sqpn; // TODO: remove it + WorkReqID wrID; // TODO: remove it Bool isRespErr; DataStream dataStream; } DmaReadResp deriving(Bits, FShow); @@ -485,7 +489,10 @@ typedef enum { IBV_WR_BIND_MW = 8, IBV_WR_SEND_WITH_INV = 9, IBV_WR_TSO = 10, - IBV_WR_DRIVER1 = 11 + IBV_WR_DRIVER1 = 11, + IBV_WR_RDMA_READ_RESP = 12, // Not defined in rdma-core + IBV_WR_FLUSH = 14, + IBV_WR_ATOMIC_WRITE = 15 } WorkReqOpCode deriving(Bits, Eq, FShow); typedef enum { diff --git a/src/Settings.bsv b/src/Settings.bsv index 2753070..7c546cf 100644 --- a/src/Settings.bsv +++ b/src/Settings.bsv @@ -13,7 +13,7 @@ typedef TExp#(31) MAX_MR_SIZE; // 2GB typedef TExp#(21) PAGE_SIZE_CAP; // 2MB typedef 4 MAX_QP; typedef 32 MAX_QP_WR; -typedef 1 MAX_SGE; +typedef 8 MAX_SGE; typedef 8 MAX_CQ; typedef MAX_QP_WR MAX_CQE; typedef 256 MAX_MR; diff --git a/src/Utils.bsv b/src/Utils.bsv index b9f9b35..b8c30e2 100644 --- a/src/Utils.bsv +++ b/src/Utils.bsv @@ -1250,6 +1250,7 @@ function Bool containWorkReqFlag( // return !isZero(pack(flags & enum2Flag(flag))); endfunction +// TODO: remove this function, it should consider start address to calculate PktNum // The returned PktNum might be zero or one less than actual PktNum function Tuple2#(PktNum, PmtuResidue) truncateLenByPMTU(Length len, PMTU pmtu); return case (pmtu) From 9a2011a0285885ca2cfa83dd991fc603aa1b1a7b Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Sun, 7 Jan 2024 20:53:41 +0800 Subject: [PATCH 02/16] refactor calcPadCnt --- src/DataTypes.bsv | 2 +- src/Utils.bsv | 10 +++++++--- test/SimGenRdmaReqResp.bsv | 2 +- test/Utils4Test.bsv | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index b298f7f..548687b 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -92,7 +92,7 @@ typedef Bit#(TAdd#(1, HEADER_MAX_DATA_WIDTH)) HeaderBitNum; typedef Bit#(TLog#(TAdd#(1, HEADER_MAX_FRAG_NUM))) HeaderFragNum; typedef Bit#(DATA_BUS_BYTE_NUM_WIDTH) BusByteWidthMask; -typedef Bit#(PAD_WIDTH) PadMask; +// typedef Bit#(PAD_WIDTH) PadMask; typedef Bit#(TAdd#(1, DATA_BUS_BIT_NUM_WIDTH)) BusBitNum; typedef Bit#(TAdd#(1, DATA_BUS_BYTE_NUM_WIDTH)) ByteEnBitNum; diff --git a/src/Utils.bsv b/src/Utils.bsv index b8c30e2..8786563 100644 --- a/src/Utils.bsv +++ b/src/Utils.bsv @@ -798,9 +798,13 @@ function Bool isSupportedReqOpCodeRQ(TypeQP qpt, RdmaOpCode opcode); endcase endfunction -function PAD calcPadCnt(Length len); - PadMask padMask = maxBound; - PAD tmpCnt = truncate(len) & padMask; +// function PAD calcPadCnt(Length len); +function PAD calcPadCnt(Bit#(nSz) len) provisos( + Add#(PAD_WIDTH, anysize, nSz) +); + // PadMask padMask = maxBound; + // PAD tmpCnt = truncate(len) & padMask; + PAD tmpCnt = truncate(len); PAD padCnt = (1 << valueOf(PAD_WIDTH)) - tmpCnt; return padCnt; endfunction diff --git a/test/SimGenRdmaReqResp.bsv b/test/SimGenRdmaReqResp.bsv index 0860d2d..a21fe18 100644 --- a/test/SimGenRdmaReqResp.bsv +++ b/test/SimGenRdmaReqResp.bsv @@ -826,7 +826,7 @@ module mkTestSimGenRdmaResp(Empty); // if (refDataStream.isLast) begin // let lastFragValidByteNum = calcByteEnBitNumInSim(refDataStream.byteEn); - // let padCnt = calcPadCnt(zeroExtend(lastFragValidByteNum)); + // let padCnt = calcPadCnt(lastFragValidByteNum); // let lastFragValidByteNumWithPadding = lastFragValidByteNum + zeroExtend(padCnt); // let lastFragByteEnWithPadding = genByteEn(lastFragValidByteNumWithPadding); diff --git a/test/Utils4Test.bsv b/test/Utils4Test.bsv index 6dce9e4..f9f01da 100644 --- a/test/Utils4Test.bsv +++ b/test/Utils4Test.bsv @@ -191,7 +191,7 @@ endfunction // This function should be used in simulation only function ByteEn addPadding2LastFragByteEn(ByteEn lastFragByteEn); let lastFragValidByteNum = calcByteEnBitNumInSim(lastFragByteEn); - let padCnt = calcPadCnt(zeroExtend(lastFragValidByteNum)); + let padCnt = calcPadCnt(lastFragValidByteNum); let lastFragValidByteNumWithPadding = lastFragValidByteNum + zeroExtend(padCnt); let lastFragByteEnWithPadding = genByteEn(lastFragValidByteNumWithPadding); return lastFragByteEnWithPadding; From 32b91a1fedbba0be0955589a2d4dea36961901fc Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Mon, 8 Jan 2024 00:08:05 +0800 Subject: [PATCH 03/16] refactor HeaderFragNum --- src/DataTypes.bsv | 10 +++++----- src/Utils.bsv | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index 548687b..8a2169e 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -39,15 +39,15 @@ typedef TAdd#(1, TLog#(MAX_SGE)) SGE_NUM_WIDTH; // 12 + 4 + 28 = 44 bytes typedef TAdd#(TAdd#(BTH_BYTE_WIDTH, XRCETH_BYTE_WIDTH), ATOMIC_ETH_BYTE_WIDTH) HEADER_MAX_BYTE_LENGTH; -typedef TDiv#(DATA_BUS_WIDTH, 8) DATA_BUS_BYTE_WIDTH; -typedef TLog#(DATA_BUS_BYTE_WIDTH) DATA_BUS_BYTE_NUM_WIDTH; -typedef TLog#(DATA_BUS_WIDTH) DATA_BUS_BIT_NUM_WIDTH; +typedef TDiv#(DATA_BUS_WIDTH, 8) DATA_BUS_BYTE_WIDTH; // 32 +typedef TLog#(DATA_BUS_BYTE_WIDTH) DATA_BUS_BYTE_NUM_WIDTH; // 5 +typedef TLog#(DATA_BUS_WIDTH) DATA_BUS_BIT_NUM_WIDTH; // 8 typedef TDiv#(HEADER_MAX_BYTE_LENGTH, DATA_BUS_BYTE_WIDTH) HEADER_MAX_FRAG_NUM; // 2 (bus 256b), 1 (bus 512b) typedef TMul#(DATA_BUS_WIDTH, HEADER_MAX_FRAG_NUM) HEADER_MAX_DATA_WIDTH; // 512 typedef TMul#(DATA_BUS_BYTE_WIDTH, HEADER_MAX_FRAG_NUM) HEADER_MAX_BYTE_EN_WIDTH; // 64 -// typedef TLog#(TAdd#(1, HEADER_MAX_BYTE_LENGTH)) HEADER_MAX_BYTE_NUM_WIDTH; // 6 typedef TLog#(TAdd#(1, HEADER_MAX_BYTE_EN_WIDTH)) HEADER_MAX_BYTE_NUM_WIDTH; // 7 +typedef TLog#(TAdd#(1, HEADER_MAX_FRAG_NUM)) HEADER_FRAG_NUM_WIDTH; // 2 (bus 256b), 1 (bus 512b) typedef TLog#(MAX_PMTU) MAX_PMTU_WIDTH; // 12 // typedef TLog#(TLog#(MAX_PMTU)) PMTU_VALUE_MAX_WIDTH; // 4 @@ -89,7 +89,7 @@ typedef Bit#(HEADER_MAX_BYTE_EN_WIDTH) HeaderByteEn; typedef Bit#(HEADER_MAX_BYTE_NUM_WIDTH) HeaderByteNum; // typedef Bit#(TLog#(TAdd#(1, HEADER_MAX_BYTE_EN_WIDTH))) HeaderByteNum; typedef Bit#(TAdd#(1, HEADER_MAX_DATA_WIDTH)) HeaderBitNum; -typedef Bit#(TLog#(TAdd#(1, HEADER_MAX_FRAG_NUM))) HeaderFragNum; +typedef Bit#(HEADER_FRAG_NUM_WIDTH) HeaderFragNum; typedef Bit#(DATA_BUS_BYTE_NUM_WIDTH) BusByteWidthMask; // typedef Bit#(PAD_WIDTH) PadMask; diff --git a/src/Utils.bsv b/src/Utils.bsv index 8786563..836d117 100644 --- a/src/Utils.bsv +++ b/src/Utils.bsv @@ -283,6 +283,8 @@ endfunction function Tuple2#(HeaderFragNum, ByteEnBitNum) calcHeaderFragNumAndLastFragValidByeNum( HeaderByteNum headerLen +) provisos( + Add#(HEADER_FRAG_NUM_WIDTH, DATA_BUS_BYTE_NUM_WIDTH, HEADER_MAX_BYTE_NUM_WIDTH) ); let headerLastFragValidByteNum = calcLastFragValidByteNum(headerLen); // BusByteWidthMask busByteWidthMask = maxBound; @@ -294,7 +296,7 @@ function Tuple2#(HeaderFragNum, ByteEnBitNum) calcHeaderFragNumAndLastFragValidB Bit#(TSub#(HEADER_MAX_BYTE_NUM_WIDTH, DATA_BUS_BYTE_NUM_WIDTH)) truncatedHeaderLen = truncateLSB(headerLen); HeaderFragNum headerFragNum = - truncatedHeaderLen + zeroExtend(pack(!isZero(headerLastFragByteNumResidue))); + truncatedHeaderLen + zeroExtend(pack(!isZeroR(headerLastFragByteNumResidue))); return tuple2(headerFragNum, headerLastFragValidByteNum); endfunction From bd40473e14618c1d80bac40653edeeadab3d66d9 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Wed, 10 Jan 2024 23:21:00 +0800 Subject: [PATCH 04/16] refactor IdxSGE --- src/DataTypes.bsv | 24 +++++++++++++----------- src/PrimUtils.bsv | 4 ++++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index 8a2169e..88ea08e 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -34,7 +34,8 @@ typedef 8 QP_CAP_CNT_WIDTH; // Derived settings typedef AETH_VALUE_WIDTH TIMER_WIDTH; -typedef TAdd#(1, TLog#(MAX_SGE)) SGE_NUM_WIDTH; +typedef TLog#(MAX_SGE) SGE_IDX_WIDTH; +typedef TAdd#(1, SGE_IDX_WIDTH) SGE_NUM_WIDTH; // 12 + 4 + 28 = 44 bytes typedef TAdd#(TAdd#(BTH_BYTE_WIDTH, XRCETH_BYTE_WIDTH), ATOMIC_ETH_BYTE_WIDTH) HEADER_MAX_BYTE_LENGTH; @@ -82,20 +83,21 @@ typedef TSub#(TSub#(ADDR_WIDTH, TLB_CACHE_INDEX_WIDTH), PAGE_OFFSET_WIDTH) TLB_C typedef Bit#(DATA_BUS_WIDTH) DATA; typedef Bit#(DATA_BUS_BYTE_WIDTH) ByteEn; -typedef Bit#(SGE_NUM_WIDTH) ScatterGatherListIdx; +typedef Bit#(SGE_IDX_WIDTH) IdxSGL; +typedef Bit#(SGE_NUM_WIDTH) NumSGE; -typedef Bit#(HEADER_MAX_DATA_WIDTH) HeaderData; -typedef Bit#(HEADER_MAX_BYTE_EN_WIDTH) HeaderByteEn; -typedef Bit#(HEADER_MAX_BYTE_NUM_WIDTH) HeaderByteNum; -// typedef Bit#(TLog#(TAdd#(1, HEADER_MAX_BYTE_EN_WIDTH))) HeaderByteNum; -typedef Bit#(TAdd#(1, HEADER_MAX_DATA_WIDTH)) HeaderBitNum; -typedef Bit#(HEADER_FRAG_NUM_WIDTH) HeaderFragNum; +typedef Bit#(HEADER_MAX_DATA_WIDTH) HeaderData; +typedef Bit#(HEADER_MAX_BYTE_EN_WIDTH) HeaderByteEn; +typedef Bit#(HEADER_MAX_BYTE_NUM_WIDTH) HeaderByteNum; +typedef Bit#(TAdd#(1, HEADER_MAX_DATA_WIDTH)) HeaderBitNum; +typedef Bit#(HEADER_FRAG_NUM_WIDTH) HeaderFragNum; -typedef Bit#(DATA_BUS_BYTE_NUM_WIDTH) BusByteWidthMask; +typedef Bit#(DATA_BUS_BIT_NUM_WIDTH) BusBitWidthMask; // 8 +typedef Bit#(DATA_BUS_BYTE_NUM_WIDTH) BusByteWidthMask; // 5 // typedef Bit#(PAD_WIDTH) PadMask; -typedef Bit#(TAdd#(1, DATA_BUS_BIT_NUM_WIDTH)) BusBitNum; -typedef Bit#(TAdd#(1, DATA_BUS_BYTE_NUM_WIDTH)) ByteEnBitNum; +typedef Bit#(TAdd#(1, DATA_BUS_BIT_NUM_WIDTH)) BusBitNum; // 9 +typedef Bit#(TAdd#(1, DATA_BUS_BYTE_NUM_WIDTH)) ByteEnBitNum; // 6 // typedef Bit#(TLog#(TAdd#(1, MAX_QP_WR))) PendingReqCnt; // typedef Bit#(TLog#(TAdd#(1, MAX_QP_RD_ATOM))) PendingReadAtomicReqCnt; diff --git a/src/PrimUtils.bsv b/src/PrimUtils.bsv index aad1674..63aaf3b 100644 --- a/src/PrimUtils.bsv +++ b/src/PrimUtils.bsv @@ -46,6 +46,10 @@ function Bool isTwo(Bit#(nSz) bits) provisos(Add#(2, anysize, nSz)); return isZero(bits >> 2) && unpack(bits[1]) && !unpack(lsb(bits)); endfunction +function Bool isTwoR(Bit#(nSz) bits) provisos(Add#(2, anysize, nSz)); + return isZero(bits >> 2) && unpack(bits[1]) && !unpack(lsb(bits)); +endfunction + // function Bool isAllOnes(Bit#(nSz) bits); // Bool ret = unpack(&bits); // return ret; From 6bc8c61cf2e81fd3f4d670ab1472f719fa952ba3 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Thu, 18 Jan 2024 00:03:41 +0800 Subject: [PATCH 05/16] add truncateByPMTU --- run_one.sh | 4 ++-- src/DataTypes.bsv | 4 ++-- src/PayloadConAndGen.bsv | 4 ++-- src/ReqGenSQ.bsv | 2 +- src/Utils.bsv | 35 +++++++++++++++++++++++++++++++++-- test/TestPayloadConAndGen.bsv | 6 +++--- test/TestUtils.bsv | 2 +- test/Utils4Test.bsv | 2 +- 8 files changed, 45 insertions(+), 14 deletions(-) diff --git a/run_one.sh b/run_one.sh index cad3ab1..6eb1759 100755 --- a/run_one.sh +++ b/run_one.sh @@ -9,11 +9,11 @@ if [ -f "$BASH_PROFILE" ]; then source $BASH_PROFILE fi -TEST_LOG=test.log +TEST_LOG=run.log TEST_DIR=test cd $TEST_DIR truncate -s 0 $TEST_LOG -FILES=`ls TestReqHandleRQ.bsv` +FILES=`ls TestPayloadGen.bsv` for FILE in $FILES; do # echo $FILE TESTCASES=`grep -Phzo 'doc.*?\nmodule\s+\S+(?=\()' $FILE | xargs -0 -I {} echo "{}" | grep module | cut -d ' ' -f 2` diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index 88ea08e..2905a11 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -107,7 +107,7 @@ typedef Bit#(QP_CAP_CNT_WIDTH) InlineDataSize; typedef Bit#(QP_CAP_CNT_WIDTH) ScatterGatherElemCnt; // typedef Bit#(PMTU_VALUE_MAX_WIDTH) PmtuValueWidth; -typedef Bit#(MAX_PMTU_WIDTH) PmtuResidue; +typedef Bit#(MAX_PMTU_WIDTH) ResiduePMTU; typedef Bit#(TOTAL_FRAG_NUM_WIDTH) TotalFragNum; typedef Bit#(PMTU_FRAG_NUM_WIDTH) PktFragNum; typedef Bit#(PKT_NUM_WIDTH) PktNum; @@ -170,7 +170,7 @@ typedef enum { RETRY_REASON_TIMEOUT } RetryReason deriving(Bits, Eq, FShow); -// DATA and ByteEn are left algined +// DATA and ByteEn are left aligned typedef struct { DATA data; ByteEn byteEn; diff --git a/src/PayloadConAndGen.bsv b/src/PayloadConAndGen.bsv index 4ff7785..d6bdbf8 100644 --- a/src/PayloadConAndGen.bsv +++ b/src/PayloadConAndGen.bsv @@ -37,7 +37,7 @@ module mkAddrChunkSrv#(Bool clearAll, Bool isSQ)(AddrChunkSrv); FIFOF#(AddrChunkResp) respQ <- mkFIFOF; Reg#(PktLen) fullPktLenReg <- mkRegU; - Reg#(PmtuResidue) residueReg <- mkRegU; + Reg#(ResiduePMTU) residueReg <- mkRegU; Reg#(Bool) isZeroResidueReg <- mkRegU; Reg#(PktNum) pktNumReg <- mkRegU; @@ -667,7 +667,7 @@ module mkPayloadGenerator#( let lastFragValidByteNum = calcLastFragValidByteNum(totalDmaLen); let lastFragValidByteNumWithPadding = lastFragValidByteNum + zeroExtend(padCnt); let lastFragByteEnWithPadding = genByteEn(lastFragValidByteNumWithPadding); - let pktFragNum = calcFragNumByPmtu(payloadGenReq.pmtu); + let pktFragNum = calcFragNumByPMTU(payloadGenReq.pmtu); immAssert( !isZero(lastFragValidByteNumWithPadding), "lastFragValidByteNumWithPadding assertion @ mkPayloadGenerator", diff --git a/src/ReqGenSQ.bsv b/src/ReqGenSQ.bsv index a3eec79..af93907 100644 --- a/src/ReqGenSQ.bsv +++ b/src/ReqGenSQ.bsv @@ -487,7 +487,7 @@ module mkReqGenSQ#( // Pipeline FIFO FIFOF#(Tuple7#( - PendingWorkReq, PktNum, PmtuResidue, Bool, Bool, Bool, Bool + PendingWorkReq, PktNum, ResiduePMTU, Bool, Bool, Bool, Bool )) workReqPayloadGenQ <- mkFIFOF; FIFOF#(Tuple3#(PendingWorkReq, PktNum, WorkReqInfo)) workReqPktNumQ <- mkFIFOF; FIFOF#(Tuple2#(PendingWorkReq, WorkReqInfo)) workReqPsnQ <- mkFIFOF; diff --git a/src/Utils.bsv b/src/Utils.bsv index 836d117..db52629 100644 --- a/src/Utils.bsv +++ b/src/Utils.bsv @@ -265,7 +265,7 @@ function Bool pktLenGtPMTU(PktLen pktLen, PMTU pmtu); endcase; endfunction -function PktFragNum calcFragNumByPmtu(PMTU pmtu) provisos( +function PktFragNum calcFragNumByPMTU(PMTU pmtu) provisos( // Check DATA_BUS_BYTE_WIDTH must be power of 2 Add#(TLog#(DATA_BUS_BYTE_WIDTH), 1, TLog#(TAdd#(1, DATA_BUS_BYTE_WIDTH))) ); @@ -1256,9 +1256,40 @@ function Bool containWorkReqFlag( // return !isZero(pack(flags & enum2Flag(flag))); endfunction +function ResiduePMTU truncateByPMTU(Bit#(nSz) bits, PMTU pmtu) provisos( + Add#(MAX_PMTU_WIDTH, anysizeJ, nSz), + Add#(TSub#(MAX_PMTU_WIDTH, 1), anysizeK, nSz), + Add#(TSub#(MAX_PMTU_WIDTH, 2), anysizeL, nSz), + Add#(TSub#(MAX_PMTU_WIDTH, 3), anysizeM, nSz), + Add#(TSub#(MAX_PMTU_WIDTH, 4), anysizeN, nSz) +); + return case (pmtu) + IBV_MTU_256 : begin + Bit#(TSub#(MAX_PMTU_WIDTH, 4)) residue = truncate(bits); // [7 : 0] + zeroExtend(residue); + end + IBV_MTU_512 : begin + Bit#(TSub#(MAX_PMTU_WIDTH, 3)) residue = truncate(bits); // [8 : 0] + zeroExtend(residue); + end + IBV_MTU_1024: begin + Bit#(TSub#(MAX_PMTU_WIDTH, 2)) residue = truncate(bits); // [9 : 0] + zeroExtend(residue); + end + IBV_MTU_2048: begin + Bit#(TSub#(MAX_PMTU_WIDTH, 1)) residue = truncate(bits); // [10 : 0] + zeroExtend(residue); + end + IBV_MTU_4096: begin + Bit#(MAX_PMTU_WIDTH) residue = truncate(bits); // [11 : 0] + zeroExtend(residue); + end + endcase; +endfunction + // TODO: remove this function, it should consider start address to calculate PktNum // The returned PktNum might be zero or one less than actual PktNum -function Tuple2#(PktNum, PmtuResidue) truncateLenByPMTU(Length len, PMTU pmtu); +function Tuple2#(PktNum, ResiduePMTU) truncateLenByPMTU(Length len, PMTU pmtu); return case (pmtu) IBV_MTU_256 : begin Bit#(8) residue = truncate(len); // [7 : 0] diff --git a/test/TestPayloadConAndGen.bsv b/test/TestPayloadConAndGen.bsv index 4f15e09..e1e064f 100644 --- a/test/TestPayloadConAndGen.bsv +++ b/test/TestPayloadConAndGen.bsv @@ -40,7 +40,7 @@ module mkTestAddrChunkSrv(Empty); let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); - Reg#(PmtuResidue) residueReg <- mkRegU; + Reg#(ResiduePMTU) residueReg <- mkRegU; Reg#(PktNum) pktNumReg <- mkRegU; Reg#(ADDR) nextAddrReg <- mkRegU; Reg#(Length) totalLenReg <- mkRegU; @@ -389,7 +389,7 @@ module mkTestDmaWriteCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); let totalPktNum = calcPktNumByLenOnly(payloadLen, pmtu); let { totalFragNum, lastFragByteEn, lastFragValidByteNum } = calcTotalFragNumByLength(payloadLen); - let pktFragNum = calcFragNumByPmtu(pmtu); + let pktFragNum = calcFragNumByPMTU(pmtu); pendingReqStatsQ.enq(tuple3(totalFragNum, lastFragByteEn, pktFragNum)); // $display( // "time=%0t: issueReq", $time, @@ -810,7 +810,7 @@ module mkTestPayloadGenSegmentAndPaddingCase(Empty); }; payloadGenerator.srvPort.request.put(payloadGenReq); - let maxPktFragNum = calcFragNumByPmtu(pmtu); + let maxPktFragNum = calcFragNumByPMTU(pmtu); let { totalFragNum, lastFragByteEn, lastFragValidByteNum } = calcTotalFragNumByLength(payloadLen); let lastFragByteEnWithPadding = addPadding2LastFragByteEn(lastFragByteEn); diff --git a/test/TestUtils.bsv b/test/TestUtils.bsv index 0bd095f..8700cfb 100644 --- a/test/TestUtils.bsv +++ b/test/TestUtils.bsv @@ -32,7 +32,7 @@ module mkTestSegmentDataStream(Empty); let refDataStream = refDataStreamPipeOut.first; refDataStreamPipeOut.deq; - let maxPktFragNum = calcFragNumByPmtu(pmtu); + let maxPktFragNum = calcFragNumByPMTU(pmtu); if (refDataStream.isLast) begin pmtuFragCntReg <= 0; end diff --git a/test/Utils4Test.bsv b/test/Utils4Test.bsv index f9f01da..4f83e0d 100644 --- a/test/Utils4Test.bsv +++ b/test/Utils4Test.bsv @@ -215,7 +215,7 @@ module mkSegmentDataStreamByPmtu#( if (!setFirstReg && curData.isFirst) begin let pmtu = pmtuPipeIn.first; pmtuPipeIn.deq; - let pktFragNum = calcFragNumByPmtu(pmtu); + let pktFragNum = calcFragNumByPMTU(pmtu); pktFragNumReg <= pktFragNum; fragCntReg <= pktFragNum - 2; end From 5ee5d786000124a44a810bce17955a2f36a9804b Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Sun, 21 Jan 2024 23:32:20 +0800 Subject: [PATCH 06/16] support SG in PayloadGen --- src/PayloadGen.bsv | 2044 +++++++++++++++++++++++++++++++++++++++ src/Utils.bsv | 2 +- test/TestPayloadGen.bsv | 1401 +++++++++++++++++++++++++++ 3 files changed, 3446 insertions(+), 1 deletion(-) create mode 100644 src/PayloadGen.bsv create mode 100644 test/TestPayloadGen.bsv diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv new file mode 100644 index 0000000..4c9b59e --- /dev/null +++ b/src/PayloadGen.bsv @@ -0,0 +1,2044 @@ +import Arbitration :: *; +import BRAMFIFO :: *; +import ClientServer :: *; +import Connectable :: *; +import FIFOF :: *; +import GetPut :: *; +import PAClib :: *; +import Vector :: *; + +import DataTypes :: *; +import Headers :: *; +import PrimUtils :: *; +import Settings :: *; +import Utils :: *; + +typedef Bit#(32) AddrIPv4; +typedef Bit#(128) AddrIPv6; + +typedef union tagged { + AddrIPv4 IPv4; + AddrIPv6 IPv6; +} IP deriving(Bits); + +instance FShow#(IP); + function Fmt fshow(IP ipAddr); + case (ipAddr) matches + tagged IPv4 .ipv4: begin + return $format( + "ipv4=%d.%d.%d.%d", + ipv4[31 : 24], ipv4[23: 16], ipv4[15 : 8], ipv4[7 : 0] + ); + end + tagged IPv6 .ipv6: begin + return $format( + "ipv6=%h:%h:%h:%h:%h:%h:%h:%h", + ipv6[127 : 112], ipv6[111: 96], ipv6[95 : 80], ipv6[79 : 64], + ipv6[63 : 48], ipv6[47: 32], ipv6[31 : 16], ipv6[15 : 0] + ); + end + endcase + endfunction +endinstance + +typedef union tagged { + IMM ImmDt; + RKEY RKey2Inv; +} ImmDtOrInvRKey deriving(Bits, FShow); + +typedef struct { + WorkReqID id; // TODO: remove it + WorkReqOpCode opcode; + FlagsType#(WorkReqSendFlag) flags; + TypeQP qpType; + PSN curPSN; + PMTU pmtu; + IP dqpIP; + ScatterGatherList sgl; + ADDR raddr; + RKEY rkey; + // ADDR laddr; + // Length len; + // LKEY lkey; + QPN sqpn; // TODO: remove it + Bool solicited; // Relevant only for the Send and RDMA Write with immediate data + // Maybe#(Long) comp; + // Maybe#(Long) swap; + Maybe#(ImmDtOrInvRKey) immDtOrInvRKey; + Maybe#(QPN) srqn; // for XRC + Maybe#(QPN) dqpn; // for UD + Maybe#(QKEY) qkey; // for UD +} WorkQueueElem deriving(Bits); + +instance FShow#(WorkQueueElem); + function Fmt fshow(WorkQueueElem wqe); + return $format( + "WorkQueueElem { opcode=", fshow(wqe.opcode), + ", flags=", fshow(wqe.flags), + ", qpType=", fshow(wqe.qpType), + ", curPSN=%h", wqe.curPSN, + ", pmtu=", fshow(wqe.pmtu), + ", dqpIP=", fshow(wqe.dqpIP), + ", sgl=", fshow(wqe.sgl), + ", rkey=%h", wqe.rkey, + ", raddr=%h", wqe.raddr, + // ", sqpn=%h", wqe.sqpn, + // ", sqpn=%h", wqe.sqpn, + ", solicited=", fshow(wqe.solicited), + // ", comp=", fshow(wqe.comp), ", swap=", fshow(wqe.swap), + ", immDtOrInvRKey=", fshow(wqe.immDtOrInvRKey), + ", srqn=", fshow(wqe.srqn), + ", dqpn=", fshow(wqe.dqpn), + ", qkey=", fshow(wqe.qkey), " }" + ); + endfunction +endinstance + +typedef struct { + ADDR laddr; + Length len; + LKEY lkey; + Bool isFirst; + Bool isLast; +} ScatterGatherElem deriving(Bits, FShow); + +typedef Vector#(MAX_SGE, ScatterGatherElem) ScatterGatherList; + +typedef struct { + // The last fragment ByteEn for each packet of the SGE + ByteEnBitNum curPktLastFragValidByteNum; + PktLen pktLen; + PMTU pmtu; + Bool isFirst; + Bool isLast; + Bool isFirstSGE; + Bool isLastSGE; +} PktMetaDataSGE deriving(Bits, FShow); +// typedef struct { +// PktLen firstPktLen; +// PktLen lastPktLen; +// PktNum sgePktNum; +// Length sgeLen; +// PMTU pmtu; +// Bool isFirst; +// Bool isLast; +// } DmaReadMetaDataSGE deriving(Bits, FShow); +// typedef struct { +// PktLen firstPktLen; +// PktFragNum firstPktFragNum; +// ByteEnBitNum firstPktLastFragValidByteNum; +// // PAD firstPktPadCnt; +// // ByteEn firstPktLastFragByteEn; +// PktLen lastPktLen; +// PktFragNum lastPktFragNum; +// ByteEnBitNum lastPktLastFragValidByteNum; +// // PAD lastPktPadCnt; +// // ByteEn lastPktLastFragByteEn; +// Length sgeLen; +// PktNum sgePktNum; +// PMTU pmtu; +// Bool isSinglePktSGE; +// Bool isFirst; +// Bool isLast; +// } DetailMetaDataSGE deriving(Bits, FShow); + +typedef struct { + // PktLen lastPktLen; + // PktFragNum lastPktFragNum; + // PktNum sgePktNum; + ByteEnBitNum lastFragValidByteNum; + Bool isFirst; + Bool isLast; +} MergedMetaDataSGE deriving(Bits, FShow); + +typedef struct { + QPN sqpn; // TODO: remove it + WorkReqID wrID; // TODO: remove it + Length totalLen; + NumSGE sgeNum; + PMTU pmtu; + // PktFragNum pmtuFragNum; + // PktLen pmtuLen; +} TotalPayloadMetaData deriving(Bits, FShow); + +typedef struct { + // QPN sqpn; // TODO: remove it + // WorkReqID wrID; // TODO: remove it + PktLen firstPktLen; + PktFragNum firstPktFragNum; + ByteEnBitNum firstPktLastFragValidByteNum; + // PAD firstPktPadCnt; + // ByteEn firstPktLastFragByteEn; + // PktLen lastPktLen; + // PktFragNum lastPktFragNum; + // ByteEnBitNum lastPktLastFragValidByteNum; + // PAD lastPktPadCnt; + // ByteEn lastPktLastFragByteEn; + ByteEnBitNum origLastFragValidByteNum; + // PktLen pmtuLen; + PktNum adjustedPktNum; + PktNum origPktNum; + PMTU pmtu; + // Length totalLen; +} AdjustedTotalPayloadMetaData deriving(Bits, FShow); + +typedef struct { + // DmaReqSrcType initiator; + ScatterGatherList sgl; + QPN sqpn; // TODO: remove it + // ADDR startAddr; + // Length len; + WorkReqID wrID; // TODO: remove it +} DmaReadMetaDataSGL deriving(Bits, FShow); + +typedef struct { + ADDR startAddr; + Length len; + PMTU pmtu; + Bool isFirst; + Bool isLast; +} AddrChunkReq deriving(Bits, FShow); + +typedef struct { + ADDR chunkAddr; + PktLen chunkLen; + Bool isFirst; + Bool isLast; + Bool isOrigFirst; + Bool isOrigLast; +} AddrChunkResp deriving(Bits, FShow); + +function ADDR alignAddrByPMTU(ADDR addr, PMTU pmtu); + return case (pmtu) + IBV_MTU_256 : begin + // 8 = log2(256) + { addr[valueOf(ADDR_WIDTH)-1 : 8], 8'b0 }; + end + IBV_MTU_512 : begin + // 9 = log2(512) + { addr[valueOf(ADDR_WIDTH)-1 : 9], 9'b0 }; + end + IBV_MTU_1024: begin + // 10 = log2(1024) + { addr[valueOf(ADDR_WIDTH)-1 : 10], 10'b0 }; + end + IBV_MTU_2048: begin + // 11 = log2(2048) + { addr[valueOf(ADDR_WIDTH)-1 : 11], 11'b0 }; + end + IBV_MTU_4096: begin + // 12 = log2(4096) + { addr[valueOf(ADDR_WIDTH)-1 : 12], 12'b0 }; + end + endcase; +endfunction + +function Tuple5#(PktLen, PktLen, PktLen, PktNum, ADDR) calcPktNumAndPktLenByAddrAndPMTU( + ADDR startAddr, Length len, PMTU pmtu +); + let oneAsPSN = 1; + let pmtuAlignedStartAddr = alignAddrByPMTU(startAddr, pmtu); + let secondChunkStartAddr = addrAddPsnMultiplyPMTU(pmtuAlignedStartAddr, oneAsPSN, pmtu); + let pmtuLen = calcPmtuLen(pmtu); + + Tuple4#(PktLen, PktNum, PktLen, PktLen) tmpTuple = case (pmtu) + IBV_MTU_256 : begin + Bit#(8) addrLowPart = truncate(startAddr); // [7 : 0] + Bit#(8) lenLowPart = truncate(len); + Bit#(8) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 8)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + IBV_MTU_512 : begin + Bit#(9) addrLowPart = truncate(startAddr); // [8 : 0] + Bit#(9) lenLowPart = truncate(len); + Bit#(9) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 9)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + IBV_MTU_1024: begin + Bit#(10) addrLowPart = truncate(startAddr); // [9 : 0] + Bit#(10) lenLowPart = truncate(len); + Bit#(10) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 10)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + IBV_MTU_2048: begin + Bit#(11) addrLowPart = truncate(startAddr); // [10 : 0] + Bit#(11) lenLowPart = truncate(len); + Bit#(11) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 11)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + IBV_MTU_4096: begin + Bit#(12) addrLowPart = truncate(startAddr); // [11 : 0] + Bit#(12) lenLowPart = truncate(len); + Bit#(12) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 12)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + endcase; + + let { pmtuMask, truncatedPktNum, addrLowPart, lenLowPart } = tmpTuple; + let maxFirstPktLen = pmtuLen - addrLowPart; + let tmpSum = addrLowPart + lenLowPart; + ResiduePMTU residue = truncateByPMTU(tmpSum, pmtu); + PktLen tmpLastPktLen = zeroExtend(residue); + + let pmtuInvMask = ~pmtuMask; + let residuePktNum = |(pmtuMask & tmpSum); + let extraPktNum = |(pmtuInvMask & tmpSum); + Bool hasResidue = unpack(residuePktNum); + Bool hasExtraPkt = unpack(extraPktNum); + let notFullPkt = isZeroR(truncatedPktNum); + + let totalPktNum = truncatedPktNum + zeroExtend(residuePktNum) + zeroExtend(extraPktNum); + // let firstPktLen = notFullPkt ? (hasExtraPkt ? maxFirstPktLen : lenLowPart) : maxFirstPktLen; + let firstPktLen = (notFullPkt && !hasExtraPkt) ? lenLowPart : maxFirstPktLen; + // let lastPktLen = notFullPkt ? (hasResidue ? tmpLastPktLen : 0) : (hasResidue ? tmpLastPktLen : pmtuLen); + let lastPktLen = notFullPkt ? (hasExtraPkt ? tmpLastPktLen : lenLowPart) : (hasResidue ? tmpLastPktLen : pmtuLen); + // let isSinglePkt = isLessOrEqOneR(totalPktNum); + + return tuple5(pmtuLen, firstPktLen, lastPktLen, totalPktNum, secondChunkStartAddr); +endfunction + +function PktFragNum calcFragNumByPktLen(PktLen pktLen) provisos( + Add#(PMTU_FRAG_NUM_WIDTH, DATA_BUS_BYTE_NUM_WIDTH, PKT_LEN_WIDTH) +); + BusByteWidthMask lastFragByteNumResidue = truncate(pktLen); + // Bit#(TSub#(PKT_LEN_WIDTH, DATA_BUS_BYTE_NUM_WIDTH)) truncatedPktLen = + PktFragNum truncatedPktLen = truncateLSB(pktLen); + let pktFragNum = truncatedPktLen + zeroExtend(pack(!isZeroR(lastFragByteNumResidue))); + return pktFragNum; +endfunction + +function Tuple2#(PktLen, PktNum) calcPktNumAndLastPktLenByPMTU(Length len, PMTU pmtu); + let pmtuLen = calcPmtuLen(pmtu); + + Tuple2#(PktNum, PktLen) tmpTuple = case (pmtu) + IBV_MTU_256 : begin + Bit#(8) lenLowPart = truncate(len); // [7 : 0] + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 8)) truncatedLen = truncateLSB(len); + tuple2(zeroExtend(truncatedLen), zeroExtend(lenLowPart)); + end + IBV_MTU_512 : begin + Bit#(9) lenLowPart = truncate(len); // [8 : 0] + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 9)) truncatedLen = truncateLSB(len); + tuple2(zeroExtend(truncatedLen), zeroExtend(lenLowPart)); + end + IBV_MTU_1024: begin + Bit#(10) lenLowPart = truncate(len); // [9 : 0] + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 10)) truncatedLen = truncateLSB(len); + tuple2(zeroExtend(truncatedLen), zeroExtend(lenLowPart)); + end + IBV_MTU_2048: begin + Bit#(11) lenLowPart = truncate(len); // [10 : 0] + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 11)) truncatedLen = truncateLSB(len); + tuple2(zeroExtend(truncatedLen), zeroExtend(lenLowPart)); + end + IBV_MTU_4096: begin + Bit#(12) lenLowPart = truncate(len); // [11 : 0] + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 12)) truncatedLen = truncateLSB(len); + tuple2(zeroExtend(truncatedLen), zeroExtend(lenLowPart)); + end + endcase; + + let { truncatedPktNum, residuePktLen } = tmpTuple; + let residuePktNum = |residuePktLen; + Bool hasResidue = unpack(residuePktNum); + let noFullPkt = isZeroR(truncatedPktNum); + + let totalPktNum = truncatedPktNum + zeroExtend(residuePktNum); + let lastPktLen = noFullPkt ? (hasResidue ? residuePktLen : 0) : (hasResidue ? residuePktLen : pmtuLen); + // let isSinglePkt = isLessOrEqOneR(totalPktNum); + + return tuple2(lastPktLen, totalPktNum); +endfunction + +function DataStream mergeFragData( + DataStream preFrag, + DataStream curFrag, + ByteEnBitNum preFragInvalidByteNum, + BusBitNum preFragInvalidBitNum +); + let resultFrag = preFrag; + resultFrag.byteEn = truncateLSB({ preFrag.byteEn, curFrag.byteEn } << preFragInvalidByteNum); + resultFrag.data = truncateLSB({ preFrag.data, curFrag.data } << preFragInvalidBitNum); + return resultFrag; +endfunction + +interface AddrChunkSrv; + interface Server#(AddrChunkReq, AddrChunkResp) srvPort; + // interface PipeOut#(DmaReadMetaDataSGE) sgeDmaReadMetaDataPipeOut; + interface PipeOut#(PktMetaDataSGE) sgePktMetaDataPipeOut; + method Bool isIdle(); +endinterface + +module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); + FIFOF#(AddrChunkReq) reqQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + FIFOF#(AddrChunkResp) respQ <- mkFIFOF; + // FIFOF#(DmaReadMetaDataSGE) sgeDmaReadMetaDataOutQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + FIFOF#(PktMetaDataSGE) sgePktMetaDataOutQ <- mkFIFOF; + + Reg#(PktLen) firstPktLenReg <- mkRegU; + Reg#(PktLen) fullPktLenReg <- mkRegU; + Reg#(PktLen) lastPktLenReg <- mkRegU; + Reg#(Bool) isOrigFirstReg <- mkRegU; + Reg#(Bool) isOrigLastReg <- mkRegU; + // Reg#(ResiduePMTU) residueReg <- mkRegU; + // Reg#(Bool) isZeroResidueReg <- mkRegU; + + Reg#(PktNum) pktNumReg <- mkRegU; + Reg#(ADDR) chunkAddrReg <- mkRegU; + Reg#(ADDR) nextAddrReg <- mkRegU; + Reg#(PMTU) pmtuReg <- mkRegU; + Reg#(Bool) isFirstReg <- mkRegU; + Reg#(Bool) busyReg <- mkReg(False); + + PSN oneAsPSN = 1; + + rule resetAndClear if (clearAll); + reqQ.clear; + respQ.clear; + busyReg <= False; + // isFirstReg <= False; + endrule + + // rule debug; + // $display( + // "time=%0t: mkAddrChunkSrv debug", $time, + // ", isSQ=", fshow(isSQ), + // ", reqQ.notEmpty=", fshow(reqQ.notEmpty), + // ", respQ.notFull=", fshow(respQ.notFull), + // ", pktNumReg=%0d", pktNumReg, + // ", fullPktLenReg=%h", fullPktLenReg, + // ", busyReg=", fshow(busyReg), + // ", isFirstReg=", fshow(isFirstReg), + // ", clearAll=", fshow(clearAll) + // ); + // endrule + + rule recvReq if (!clearAll && !busyReg); + let addrChunkReq = reqQ.first; + reqQ.deq; + + immAssert( + !isZeroR(addrChunkReq.len), + "addrChunkReq.len assertion @ mkAddrChunkSrv", + $format( + "addrChunkReq.len=%0d cannot be zero", addrChunkReq.len + ) + ); + + let { + pmtuLen, firstPktLen, lastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU( + addrChunkReq.startAddr, addrChunkReq.len, addrChunkReq.pmtu + ); + // let { tmpPktNum, pmtuResidue } = truncateLenByPMTU( + // addrChunkReq.len, addrChunkReq.pmtu + // ); + // let isZeroResidue = isZeroR(pmtuResidue); + // let totalPktNum = tmpPktNum + (isZeroResidue ? 0 : 1); + // let pmtuLen = calcPmtuLen(addrChunkReq.pmtu); + + firstPktLenReg <= firstPktLen; + fullPktLenReg <= pmtuLen; + lastPktLenReg <= lastPktLen; + pktNumReg <= sgePktNum; + chunkAddrReg <= addrChunkReq.startAddr; + nextAddrReg <= secondChunkStartAddr; + pmtuReg <= addrChunkReq.pmtu; + isOrigFirstReg <= addrChunkReq.isFirst; + isOrigLastReg <= addrChunkReq.isLast; + isFirstReg <= True; + busyReg <= True; + // residueReg <= pmtuResidue; + // isZeroResidueReg <= isZeroResidue; + + // pktNumReg <= totalPktNum - 1; + // chunkAddrReg <= addrAddPsnMultiplyPMTU(addrChunkReq.startAddr, oneAsPSN, addrChunkReq.pmtu); + // busyReg <= !isSinglePkt; + // let addrChunkResp = AddrChunkResp { + // chunkAddr: addrChunkReq.startAddr, + // chunkLen : firstPktLen, + // isFirst : True, + // isLast : isSinglePkt + // }; + // respQ.enq(addrChunkResp); + + // let sgeDmaReadMetaData = DmaReadMetaDataSGE { + // firstPktLen: firstPktLen, + // lastPktLen : lastPktLen, + // sgePktNum : sgePktNum, + // sgeLen : addrChunkReq.len, + // pmtu : addrChunkReq.pmtu, + // isFirst : addrChunkReq.isOrigFirst, + // isLast : addrChunkReq.isOrigLast + // }; + // sgeDmaReadMetaDataOutQ.enq(sgeDmaReadMetaData); + + // $display( + // "time=%0t: mkAddrChunkSrv recvReq", $time, + // ", isSQ=", fshow(isSQ), + // ", sgePktNum=%0d", sgePktNum, + // ", pmtuResidue=%h", pmtuResidue, + // ", addrChunkReq=", fshow(addrChunkReq) + // ); + endrule + + rule genResp if (!clearAll && busyReg); + let isLast = isLessOrEqOneR(pktNumReg); + + busyReg <= !isLast; + pktNumReg <= pktNumReg - 1; + isFirstReg <= False; + chunkAddrReg <= nextAddrReg; + nextAddrReg <= addrAddPsnMultiplyPMTU(nextAddrReg, oneAsPSN, pmtuReg); + + let chunkLen = isFirstReg ? firstPktLenReg : (isLast ? lastPktLenReg : fullPktLenReg); + let addrChunkResp = AddrChunkResp { + chunkAddr : chunkAddrReg, + chunkLen : chunkLen, + // chunkLen : (isLast && !isZeroResidueReg) ? zeroExtend(residueReg) : fullPktLenReg, + isFirst : isFirstReg, + isLast : isLast, + isOrigFirst: isOrigFirstReg, + isOrigLast : isOrigLastReg + }; + respQ.enq(addrChunkResp); + + let firstPktLastFragValidByteNum = calcLastFragValidByteNum(firstPktLenReg); + let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLenReg); + let curPktLastFragValidByteNum = isFirstReg ? + firstPktLastFragValidByteNum : ( + isLast ? lastPktLastFragValidByteNum : fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)) + ); + + let sgePktMetaData = PktMetaDataSGE { + curPktLastFragValidByteNum: curPktLastFragValidByteNum, + pktLen : chunkLen, + pmtu : pmtuReg, + isFirst : isFirstReg, + isLast : isLast, + isFirstSGE : isOrigFirstReg, + isLastSGE : isOrigLastReg + }; + sgePktMetaDataOutQ.enq(sgePktMetaData); + + // $display( + // "time=%0t: mkAddrChunkSrv genResp", $time, + // ", isSQ=", fshow(isSQ), + // ", pktNumReg=%0d", pktNumReg, + // ", chunkAddrReg=%h", chunkAddrReg, + // ", nextAddrReg=%h", nextAddrReg, + // ", addrChunkResp=", fshow(addrChunkResp) + // ); + endrule + + interface srvPort = toGPServer(reqQ, respQ); + // interface sgeDmaReadMetaDataPipeOut = toPipeOut(sgeDmaReadMetaDataOutQ); + interface sgePktMetaDataPipeOut = toPipeOut(sgePktMetaDataOutQ); + method Bool isIdle() = !busyReg && + !reqQ.notEmpty && !sgePktMetaDataOutQ.notEmpty && !respQ.notEmpty; +endmodule + +typedef struct { + // DmaReqSrcType initiator; + DmaReadMetaDataSGL sglDmaReadMetaData; + PMTU pmtu; +} DmaReadCntrlReq deriving(Bits, FShow); + +typedef struct { + DmaReadResp dmaReadResp; + Bool isOrigFirst; + Bool isOrigLast; +} DmaReadCntrlResp deriving(Bits, FShow); + +typedef Server#(DmaReadCntrlReq, DmaReadCntrlResp) DmaCntrlReadSrv; + +interface DmaCntrl; + method Bool isIdle(); + method Action cancel(); +endinterface + +interface DmaReadCntrl; + interface DmaCntrlReadSrv srvPort; + interface DmaCntrl dmaCntrl; + interface PipeOut#(PktMetaDataSGE) sgePktMetaDataPipeOut; + interface PipeOut#(TotalPayloadMetaData) totalPayloadMetaDataPipeOut; + interface PipeOut#(MergedMetaDataSGE) sgeMergedMetaDataPipeOut; +endinterface + +module mkDmaReadCntrl#( + Bool clearAll, DmaReadSrv dmaReadSrv +)(DmaReadCntrl); + FIFOF#(DmaReadCntrlReq) reqQ <- mkFIFOF; + FIFOF#(DmaReadCntrlResp) respQ <- mkFIFOF; + FIFOF#(MergedMetaDataSGE) sgeMergedMetaDataOutQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + FIFOF#(TotalPayloadMetaData) totalPayloadMetaDataOutQ <- mkFIFOF; + + FIFOF#(Tuple2#(QPN, WorkReqID)) pendingDmaCntrlReqQ <- mkFIFOF; // TODO: remove it + FIFOF#(Tuple2#(Bool, Bool)) pendingDmaReadReqQ <- mkFIFOF; + FIFOF#(Tuple2#(ScatterGatherElem, PMTU)) pendingScatterGatherElemQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + + let addrChunkSrv <- mkAddrChunkSrv(clearAll); + + Reg#(Bool) gracefulStopReg[2] <- mkCReg(2, False); + Reg#(Bool) cancelReg[2] <- mkCReg(2, False); + + Reg#(Length) totalLenReg <- mkRegU; + Reg#(NumSGE) sgeNumReg <- mkRegU; + Reg#(IdxSGL) sglIdxReg <- mkReg(0); + + rule resetAndClear if (clearAll); + reqQ.clear; + respQ.clear; + + pendingDmaCntrlReqQ.clear; + pendingScatterGatherElemQ.clear; + pendingDmaReadReqQ.clear; + + cancelReg[1] <= False; + gracefulStopReg[1] <= False; + + sglIdxReg <= 0; + // $display("time=%0t: resetAndClear", $time); + endrule + + (* conflict_free = "recvReq, \ + issueChunkReq, \ + issueDmaReq, \ + recvDmaResp" *) + rule recvReq if (!clearAll && !cancelReg[1]); + let dmaReadCntrlReq = reqQ.first; + + let sglIdx = sglIdxReg; + let sge = dmaReadCntrlReq.sglDmaReadMetaData.sgl[sglIdx]; + immAssert( + !isZeroR(sge.len), + "zero SGE assertion @ mkDmaReadCntrl", + $format( + "sge.len=%d", sge.len, + " should not be zero when sglIdxReg=%d", sglIdxReg + ) + ); + + let mergedLastPktLastFragValidByteNum = + calcLastFragValidByteNum(sge.len); + let sgeMergedMetaData = MergedMetaDataSGE { + lastFragValidByteNum: mergedLastPktLastFragValidByteNum, + isFirst : sge.isFirst, + isLast : sge.isLast + }; + sgeMergedMetaDataOutQ.enq(sgeMergedMetaData); + pendingScatterGatherElemQ.enq(tuple2(sge, dmaReadCntrlReq.pmtu)); + // let addrChunkReq = AddrChunkReq { + // startAddr: sge.laddr, + // len : sge.len, + // pmtu : dmaReadCntrlReq.pmtu, + // isFirst : sge.isFirst, + // isLast : sge.isLast + // }; + // addrChunkSrv.srvPort.request.put(addrChunkReq); + + let curSQPN = dmaReadCntrlReq.sglDmaReadMetaData.sqpn; + let curWorkReqID = dmaReadCntrlReq.sglDmaReadMetaData.wrID; + let totalLen = totalLenReg; + let sgeNum = sgeNumReg; + if (sge.isFirst) begin + totalLen = sge.len; + sgeNum = 1; + + pendingDmaCntrlReqQ.enq(tuple2(curSQPN, curWorkReqID)); + end + else begin + totalLen = totalLenReg + sge.len; + sgeNum = sgeNumReg + 1; + end + totalLenReg <= totalLen; + sgeNumReg <= sgeNum; + // $display( + // "time=%0t: recvReq", $time, + // ", SGE sglIdx=%0d", sglIdx, + // ", sge.laddr=%h", sge.laddr, + // ", mergedLastPktLastFragValidByteNum=%0d", mergedLastPktLastFragValidByteNum, + // ", totalLen=%0d", totalLen, + // ", sgeNum=%0d", sgeNum + // // ", sgePktNum=%0d", sgePktNum, + // // ", firstPktLen=%0d", firstPktLen, + // // ", lastPktLen=%0d", lastPktLen, + // // ", pmtuLen=%0d", pmtuLen, + // // ", firstPktFragNum=%0d", firstPktFragNum, + // // ", lastPktFragNum=%0d", lastPktFragNum + // ); + + // if (sglIdxReg == valueOf(TSub#(MAX_SGE, 1))) begin + if (isAllOnesR(sglIdxReg)) begin + immAssert( + sge.isLast, + "last SGE assertion @ mkDmaReadCntrl", + $format( + "sge.isLast=", fshow(sge.isLast), + " should be true when sglIdxReg=%d", sglIdxReg + ) + ); + end + + if (sge.isLast) begin + reqQ.deq; + sglIdxReg <= 0; + + let totalPayloadMetaData = TotalPayloadMetaData { + sqpn : curSQPN, + wrID : curWorkReqID, + totalLen: totalLen, + sgeNum : sgeNum, + pmtu : dmaReadCntrlReq.pmtu + }; + totalPayloadMetaDataOutQ.enq(totalPayloadMetaData); + // $display( + // "time=%0t: mkDmaReadCntrl recvReq", $time, + // ", sqpn=%h", curSQPN, + // ", totalPayloadMetaData=", fshow(totalPayloadMetaData) + // ); + end + else begin + sglIdxReg <= sglIdxReg + 1; + end + // $display( + // "time=%0t: mkDmaReadCntrl recvReq", $time, + // ", sqpn=%h", curSQPN, + // ", dmaReadCntrlReq=", fshow(dmaReadCntrlReq), + // ", addrChunkReq=", fshow(addrChunkReq) + // ); + endrule + + rule issueChunkReq if (!clearAll && !cancelReg[1]); + let { sge, pmtu } = pendingScatterGatherElemQ.first; + pendingScatterGatherElemQ.deq; + + let addrChunkReq = AddrChunkReq { + startAddr: sge.laddr, + len : sge.len, + pmtu : pmtu, + isFirst : sge.isFirst, + isLast : sge.isLast + }; + addrChunkSrv.srvPort.request.put(addrChunkReq); + endrule + + rule issueDmaReq if (!clearAll && !cancelReg[1]); + let addrChunkResp <- addrChunkSrv.srvPort.response.get; + + let { curSQPN, curWorkReqID } = pendingDmaCntrlReqQ.first; + // let pendingSGE = pendingScatterGatherElemQ.first; + + let dmaReadReq = DmaReadReq { + initiator: DMA_SRC_SQ_RD, + sqpn : curSQPN, + startAddr: addrChunkResp.chunkAddr, + len : addrChunkResp.chunkLen, + wrID : curWorkReqID + }; + + dmaReadSrv.request.put(dmaReadReq); + + // let curSGE = pendingScatterGatherElemQ.first; + // let isFirstDmaReqChunk = addrChunkResp.isFirst && curSGE.isFirst; + // let isLastDmaReqChunk = addrChunkResp.isLast && curSGE.isLast; + let isFirstDmaReqChunk = addrChunkResp.isFirst && addrChunkResp.isOrigFirst; + let isLastDmaReqChunk = addrChunkResp.isLast && addrChunkResp.isOrigLast; + pendingDmaReadReqQ.enq(tuple2(isFirstDmaReqChunk, isLastDmaReqChunk)); + + // if (addrChunkResp.isLast) begin + // pendingScatterGatherElemQ.deq; + // end + if (isLastDmaReqChunk) begin + pendingDmaCntrlReqQ.deq; + end + // $display( + // "time=%0t: mkDmaReadCntrl issueDmaReq", $time, + // ", sqpn=%h", curSQPN, + // ", pendingDmaReadCntrlReq=", fshow(pendingDmaCntrlReqQ.first), + // ", addrChunkResp=", fshow(addrChunkResp), + // ", dmaReadReq=", fshow(dmaReadReq) + // ); + endrule + + rule recvDmaResp if (!clearAll); + let dmaResp <- dmaReadSrv.response.get; + + let { isFirstDmaReqChunk, isLastDmaReqChunk } = pendingDmaReadReqQ.first; + + let isOrigFirst = dmaResp.dataStream.isFirst && isFirstDmaReqChunk; + let isOrigLast = dmaResp.dataStream.isLast && isLastDmaReqChunk; + + let dmaReadCntrlResp = DmaReadCntrlResp { + dmaReadResp: dmaResp, + isOrigFirst: isOrigFirst, + isOrigLast : isOrigLast + }; + respQ.enq(dmaReadCntrlResp); + + if (dmaResp.dataStream.isLast) begin + pendingDmaReadReqQ.deq; + end + // $display( + // "time=%0t: mkDmaReadCntrl recvDmaResp", $time, + // ", isFirst=", fshow(dmaResp.dataStream.isFirst), + // ", isLast=", fshow(dmaResp.dataStream.isLast), + // ", isFirstDmaReqChunk=", fshow(isFirstDmaReqChunk), + // ", isLastDmaReqChunk=", fshow(isLastDmaReqChunk), + // ", isOrigFirst=", fshow(isOrigFirst), + // ", isOrigLast=", fshow(isOrigLast) + // ); + endrule + + rule setGracefulStop if ( + cancelReg[1] && + !gracefulStopReg[1] && + !respQ.notEmpty && + !pendingDmaReadReqQ.notEmpty && + !clearAll + ); + gracefulStopReg[1] <= True; + // $display("time=%0t: mkDmaReadCntrl cancel read done", $time); + endrule + + interface srvPort = toGPServer(reqQ, respQ); + + interface dmaCntrl = interface DmaCntrl; + method Action cancel(); + cancelReg[0] <= True; + gracefulStopReg[0] <= False; + endmethod + + method Bool isIdle() = gracefulStopReg[0]; + endinterface; + + interface totalPayloadMetaDataPipeOut = toPipeOut(totalPayloadMetaDataOutQ); + interface sgeMergedMetaDataPipeOut = toPipeOut(sgeMergedMetaDataOutQ); + interface sgePktMetaDataPipeOut = addrChunkSrv.sgePktMetaDataPipeOut; +endmodule + +// interface MergePayloadEachSGE; +// interface PipeOut#(MergedMetaDataSGE) sgeMergedMetaDataPipeOut; +// interface DataStreamPipeOut sgeMergedPayloadPipeOut; +// endinterface + +typedef enum { + MERGE_SGE_PAYLOAD_INIT, + // MERGE_SGE_PAYLOAD_ONLY_PKT, + MERGE_SGE_PAYLOAD_FIRST_PKT, + MERGE_SGE_PAYLOAD_MID_PKT, + // MERGE_SGE_PAYLOAD_LAST_PKT, + MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT +} MergePayloadStateEachSGE deriving(Bits, Eq, FShow); + +module mkMergePayloadEachSGE#( + Bool clearAll, + PipeOut#(PktMetaDataSGE) sgePktMetaDataPipeIn, + PipeOut#(DataStream) sgePayloadPipeIn +)(DataStreamPipeOut); + FIFOF#(DataStream) pktPayloadOutQ <- mkFIFOF; + + Reg#(ByteEnBitNum) sgeFirstPktLastFragValidByteNumReg <- mkRegU; + Reg#(ByteEnBitNum) sgeFirstPktLastFragInvalidByteNumReg <- mkRegU; + Reg#(BusBitNum) sgeFirstPktLastFragInvalidBitNumReg <- mkRegU; + // Reg#(ByteEnBitNum) sgeLastPktLastFragInvalidByteNumReg <- mkRegU; + // Reg#(BusBitNum) sgeLastPktLastFragInvalidBitNumReg <- mkRegU; + Reg#(Bool) isPreFragValidReg <- mkRegU; + Reg#(Bool) isFirstFragReg <- mkRegU; + Reg#(Bool) sgeHasOnlyPktReg <- mkRegU; + Reg#(Bool) sgePktIsLastReg <- mkRegU; + Reg#(Bool) hasExtraFragReg <- mkRegU; + Reg#(DataStream) prePayloadFragReg <- mkRegU; + Reg#(MergePayloadStateEachSGE) stateReg <- mkReg(MERGE_SGE_PAYLOAD_INIT); + + // TODO: emptyDataStream should be an input parameter + let emptyDataStream = DataStream { + data: 0, + byteEn: 0, + isFirst: False, + isLast: False + }; + + function Action prepareNextPktSGE(PktMetaDataSGE sgeNextPktMetaData); + action + sgePktIsLastReg <= sgeNextPktMetaData.isLast; + // In case sgeFirstPktLastFragValidByteNumReg=32 and curPktLastFragValidByteNum=32 + // expand extra bit for overflow case. + hasExtraFragReg <= !sgeHasOnlyPktReg && (( + { 1'b0, sgeFirstPktLastFragValidByteNumReg } + + { 1'b0, sgeNextPktMetaData.curPktLastFragValidByteNum } + ) > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); + // $display( + // "time=%0t: prepareNextPktSGE", $time, + // ", sgeHasOnlyPktReg=", fshow(sgeHasOnlyPktReg), + // ", sgeFirstPktLastFragValidByteNumReg=%0d", sgeFirstPktLastFragValidByteNumReg, + // ", sgeNextPktMetaData.curPktLastFragValidByteNum=%0d", sgeNextPktMetaData.curPktLastFragValidByteNum, + // ", sgeNextPktMetaData.isLast=", fshow(sgeNextPktMetaData.isLast) + // ); + endaction + endfunction + + function ActionValue#(DataStream) prepareNextSGE(); + actionvalue + let sgePktMetaData = sgePktMetaDataPipeIn.first; + sgePktMetaDataPipeIn.deq; + + let curPktLastFragValidByteNum = sgePktMetaData.curPktLastFragValidByteNum; + let { + curPktLastFragValidBitNum, + curPktLastFragInvalidByteNum, + curPktLastFragInvalidBitNum + } = calcFragBitNumAndByteNum(curPktLastFragValidByteNum); + // let { + // lastPktLastFragValidBitNum, + // lastPktLastFragInvalidByteNum, + // lastPktLastFragInvalidBitNum + // } = calcFragBitNumAndByteNum(sgeDetailMetaData.lastPktLastFragValidByteNum); + + let isOnlyPktSGE = sgePktMetaData.isFirst && sgePktMetaData.isLast; + sgePktIsLastReg <= sgePktMetaData.isLast; + sgeHasOnlyPktReg <= isOnlyPktSGE; + + // if (sgePktMetaData.isLast) begin + // hasExtraFragReg <= !isOnlyPktSGE && (( + // sgeFirstPktLastFragValidByteNumReg + + // curPktLastFragValidByteNum + // ) > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); + // end + + if (sgePktMetaData.isFirst) begin + sgeFirstPktLastFragValidByteNumReg <= curPktLastFragValidByteNum; + sgeFirstPktLastFragInvalidByteNumReg <= curPktLastFragInvalidByteNum; + sgeFirstPktLastFragInvalidBitNumReg <= curPktLastFragInvalidBitNum; + // sgeLastPktLastFragInvalidByteNumReg <= lastPktLastFragInvalidByteNum; + // sgeLastPktLastFragInvalidBitNumReg <= lastPktLastFragInvalidBitNum; + end + + // let curPayloadFrag = sgePayloadPipeIn.first; + // sgePayloadPipeIn.deq; + + // let nextPrePayloadFrag = curPayloadFrag; + // if (isOnlyPktSGE) begin + // stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; + // end + // else begin + // if (curPayloadFrag.isLast) begin // Single fragment first packet + // nextPrePayloadFrag.byteEn = curPayloadFrag.byteEn >> curPktLastFragInvalidByteNum; + // nextPrePayloadFrag.data = curPayloadFrag.data >> curPktLastFragInvalidBitNum; + + // stateReg <= MERGE_SGE_PAYLOAD_MID_PKT; + // end + // else begin + // stateReg <= MERGE_SGE_PAYLOAD_FIRST_PKT; + // end + // end + let curPayloadFrag = sgePayloadPipeIn.first; + let nextPrePayloadFrag = curPayloadFrag; + if (curPayloadFrag.isLast && !isOnlyPktSGE) begin // Single fragment first packet + nextPrePayloadFrag = emptyDataStream; + isPreFragValidReg <= False; + // nextPrePayloadFrag.isLast = False; + end + else begin + sgePayloadPipeIn.deq; + isPreFragValidReg <= True; + end + + if (isOnlyPktSGE) begin + stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; + end + else begin + stateReg <= MERGE_SGE_PAYLOAD_FIRST_PKT; + end + + // $display( + // "time=%0t: prepareNextSGE", $time, + // ", sgePktMetaData.isFirst=", fshow(sgePktMetaData.isFirst), + // ", sgePktMetaData.isLast=", fshow(sgePktMetaData.isLast), + // ", curPktLastFragValidByteNum=%0d", curPktLastFragValidByteNum, + // ", curPktLastFragValidBitNum=%0d", curPktLastFragValidBitNum, + // ", curPktLastFragInvalidByteNum=%0d", curPktLastFragInvalidByteNum, + // ", curPktLastFragInvalidBitNum=%0d", curPktLastFragInvalidBitNum, + // ", sgeFirstPktLastFragValidByteNumReg=%0d", sgeFirstPktLastFragValidByteNumReg, + // ", sgeFirstPktLastFragInvalidByteNumReg=%0d", sgeFirstPktLastFragInvalidByteNumReg, + // ", sgeFirstPktLastFragInvalidBitNumReg=%0d", sgeFirstPktLastFragInvalidBitNumReg, + // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), + // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast) + // ); + + isFirstFragReg <= True; + return nextPrePayloadFrag; + endactionvalue + endfunction + + rule resetAndClear if (clearAll); + pktPayloadOutQ.clear; + // sgeMergedMetaDataOutQ.clear; + stateReg <= MERGE_SGE_PAYLOAD_INIT; + endrule + + rule init if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_INIT); + let nextPrePayloadFrag <- prepareNextSGE; + prePayloadFragReg <= nextPrePayloadFrag; + endrule +/* + rule mergeOnlyPktSGE if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_ONLY_PKT); + let nextPayloadFrag = emptyDataStream; + if (prePayloadFragReg.isLast) begin + if ( + sgePktIsLastReg && !( + sgeDetailMetaDataPipeIn.notEmpty && sgePayloadPipeIn.notEmpty + ) + ) begin + // Wait for next SGL, if no next SGE metadata or payload + stateReg <= MERGE_SGE_PAYLOAD_INIT; + end + else begin + nextPayloadFrag <- prepareNextSGE; + end + end + else begin + nextPayloadFrag = sgePayloadPipeIn.first; + sgePayloadPipeIn.deq; + end + prePayloadFragReg <= nextPayloadFrag; + + immAssert( + isOneR(sgeRemainingPktNumReg), + "sgeRemainingPktNumReg assertion @ mkMergePayloadEachSGE", + $format( + "sgeRemainingPktNumReg=%0d", sgeRemainingPktNumReg, + " should be one when stateReg=", fshow(stateReg) + ) + ); + + pktPayloadOutQ.enq(prePayloadFragReg); + endrule +*/ + rule mergeFirstPktSGE if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_FIRST_PKT); + let sgeNextPktMetaData = sgePktMetaDataPipeIn.first; + + let curPayloadFrag = sgePayloadPipeIn.first; + sgePayloadPipeIn.deq; + + let nextPrePayloadFrag = curPayloadFrag; + if (curPayloadFrag.isLast) begin + sgePktMetaDataPipeIn.deq; + + // Only right shift the last fragment of the first packet + nextPrePayloadFrag.byteEn = curPayloadFrag.byteEn >> sgeFirstPktLastFragInvalidByteNumReg; + nextPrePayloadFrag.data = curPayloadFrag.data >> sgeFirstPktLastFragInvalidBitNumReg; + + // sgePktIsLastReg <= sgeNextPktMetaData.isLast; + if (sgeNextPktMetaData.isLast) begin + stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; + prepareNextPktSGE(sgeNextPktMetaData); + // hasExtraFragReg <= checkExtraFrag(sgeNextPktMetaData.curPktLastFragValidByteNum); + end + else begin + stateReg <= MERGE_SGE_PAYLOAD_MID_PKT; + end + end + isPreFragValidReg <= True; + nextPrePayloadFrag.isLast = False; + prePayloadFragReg <= nextPrePayloadFrag; + + immAssert( + !sgePktIsLastReg, + "sgePktIsLastReg assertion @ mkMergePayloadEachSGE", + $format( + "sgePktIsLastReg=", fshow(sgePktIsLastReg), + " should be false when stateReg=", fshow(stateReg) + ) + ); + + if (isPreFragValidReg) begin + isFirstFragReg <= False; + let outPayloadFrag = prePayloadFragReg; + pktPayloadOutQ.enq(outPayloadFrag); + // $display( + // "time=%0t: mergeFirstPktSGE", $time, + // ", sgeNextPktMetaData.isFirst=", fshow(sgeNextPktMetaData.isFirst), + // ", sgeNextPktMetaData.isLast=", fshow(sgeNextPktMetaData.isLast), + // ", sgeNextPktMetaData.curPktLastFragValidByteNum=%0d", sgeNextPktMetaData.curPktLastFragValidByteNum, + // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), + // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), + // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn, + // ", outPayloadFrag.isFirst=", fshow(outPayloadFrag.isFirst), + // ", outPayloadFrag.isLast=", fshow(outPayloadFrag.isLast), + // ", outPayloadFrag.byteEn=%h", outPayloadFrag.byteEn + // ); + end + endrule + + rule mergeMidPktSGE if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_MID_PKT); + let sgeNextPktMetaData = sgePktMetaDataPipeIn.first; + + let curPayloadFrag = sgePayloadPipeIn.first; + sgePayloadPipeIn.deq; + + if (curPayloadFrag.isLast) begin + sgePktMetaDataPipeIn.deq; + // $display( + // "time=%0t: mergeMidPktSGE, sgePktMetaDataPipeIn.deq", $time, + // ", sgeNextPktMetaData.curPktLastFragValidByteNum=%0d", sgeNextPktMetaData.curPktLastFragValidByteNum + // ); + + // sgePktIsLastReg <= sgeNextPktMetaData.isLast; + if (sgeNextPktMetaData.isLast) begin + stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; + prepareNextPktSGE(sgeNextPktMetaData); + // hasExtraFragReg <= checkExtraFrag(sgeNextPktMetaData.curPktLastFragValidByteNum); + end + end + let nextPrePayloadFrag = curPayloadFrag; + nextPrePayloadFrag.isLast = False; + prePayloadFragReg <= nextPrePayloadFrag; + + let mergedFrag = mergeFragData( + prePayloadFragReg, curPayloadFrag, + sgeFirstPktLastFragInvalidByteNumReg, sgeFirstPktLastFragInvalidBitNumReg + ); + isFirstFragReg <= False; + let outPayloadFrag = mergedFrag; + outPayloadFrag.isFirst = isFirstFragReg; + // outPayloadFrag.isLast = False; + pktPayloadOutQ.enq(outPayloadFrag); + // $display( + // "time=%0t: mergeMidPktSGE", $time, + // ", sgeNextPktMetaData.isFirst=", fshow(sgeNextPktMetaData.isFirst), + // ", sgeNextPktMetaData.isLast=", fshow(sgeNextPktMetaData.isLast), + // ", sgeNextPktMetaData.curPktLastFragValidByteNum=%0d", sgeNextPktMetaData.curPktLastFragValidByteNum, + // // ", sgeFirstPktLastFragInvalidByteNumReg=%0d", sgeFirstPktLastFragInvalidByteNumReg, + // // ", sgeFirstPktLastFragInvalidBitNumReg=%0d", sgeFirstPktLastFragInvalidBitNumReg, + // // ", sgeNextPktMetaData.isLast=", fshow(sgeNextPktMetaData.isLast), + // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), + // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), + // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn, + // ", outPayloadFrag.isFirst=", fshow(outPayloadFrag.isFirst), + // ", outPayloadFrag.isLast=", fshow(outPayloadFrag.isLast), + // ", outPayloadFrag.byteEn=%h", outPayloadFrag.byteEn + // ); + endrule + + rule mergeLastOrOnlyPktSGE if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT); + let nextPayloadFrag = emptyDataStream; + let outPayloadFrag = prePayloadFragReg; + + let isLastFrag = prePayloadFragReg.isLast; + if (prePayloadFragReg.isLast) begin + if (sgePktMetaDataPipeIn.notEmpty && sgePayloadPipeIn.notEmpty) begin + nextPayloadFrag <- prepareNextSGE; + end + else begin + // Wait for a packet of next SGE, if no next SGE packet metadata or payload + stateReg <= MERGE_SGE_PAYLOAD_INIT; + end + end + else begin + nextPayloadFrag = sgePayloadPipeIn.first; + sgePayloadPipeIn.deq; + + nextPayloadFrag.isFirst = False; + if (!sgeHasOnlyPktReg && !hasExtraFragReg && nextPayloadFrag.isLast) begin + // No extra fragment + stateReg <= MERGE_SGE_PAYLOAD_INIT; + isLastFrag = True; + end + end + prePayloadFragReg <= nextPayloadFrag; + + immAssert( + sgePktIsLastReg, + "sgePktIsLastReg assertion @ mkMergePayloadEachSGE", + $format( + "sgePktIsLastReg=", fshow(sgePktIsLastReg), + " should be true when stateReg=", fshow(stateReg) + ) + ); + + if (!sgeHasOnlyPktReg) begin + outPayloadFrag = mergeFragData( + prePayloadFragReg, nextPayloadFrag, + sgeFirstPktLastFragInvalidByteNumReg, sgeFirstPktLastFragInvalidBitNumReg + ); + outPayloadFrag.isLast = isLastFrag; + end + pktPayloadOutQ.enq(outPayloadFrag); + // $display( + // "time=%0t: mergeLastOrOnlyPktSGE", $time, + // ", sgeHasOnlyPktReg=", fshow(sgeHasOnlyPktReg), + // ", hasExtraFragReg=", fshow(hasExtraFragReg), + // ", prePayloadFragReg.isFirst=", fshow(prePayloadFragReg.isFirst), + // ", prePayloadFragReg.isLast=", fshow(prePayloadFragReg.isLast), + // ", prePayloadFragReg.byteEn=%h", prePayloadFragReg.byteEn, + // ", nextPayloadFrag.isFirst=", fshow(nextPayloadFrag.isFirst), + // ", nextPayloadFrag.isLast=", fshow(nextPayloadFrag.isLast), + // ", nextPayloadFrag.byteEn=%h", nextPayloadFrag.byteEn, + // ", outPayloadFrag.isFirst=", fshow(outPayloadFrag.isFirst), + // ", outPayloadFrag.isLast=", fshow(outPayloadFrag.isLast), + // ", outPayloadFrag.byteEn=%h", outPayloadFrag.byteEn + // ); + endrule + + return toPipeOut(pktPayloadOutQ); +endmodule + +typedef enum { + MERGE_SGL_PAYLOAD_INIT, + MERGE_SGL_PAYLOAD_FIRST_OR_MID_SGE, + // MERGE_SGL_PAYLOAD_FIRST_SGE, + // MERGE_SGL_PAYLOAD_MID_SGE, + MERGE_SGL_PAYLOAD_LAST_OR_ONLY_SGE + // MERGE_SGL_PAYLOAD_LAST_SGE, + // MERGE_SGL_PAYLOAD_ONLY_SGE +} MergePayloadStateAllSGE deriving(Bits, Eq, FShow); + +module mkMergePayloadAllSGE#( + Bool clearAll, + PipeOut#(MergedMetaDataSGE) sgeMergedMetaDataPipeIn, + PipeOut#(DataStream) sgeMergedPayloadPipeIn +)(DataStreamPipeOut); + FIFOF#(DataStream) pktPayloadOutQ <- mkFIFOF; + + // Reg#(ByteEnBitNum) curValidByteNumReg <- mkRegU; + Reg#(ByteEnBitNum) curInvalidByteNumReg <- mkRegU; + Reg#(BusBitNum) curInvalidBitNumReg <- mkRegU; + // Reg#(ByteEnBitNum) nextValidByteNumReg <- mkRegU; + Reg#(ByteEnBitNum) nextInvalidByteNumReg <- mkRegU; + Reg#(BusBitNum) nextInvalidBitNumReg <- mkRegU; + + Reg#(Bool) isFirstFragReg <- mkRegU; + Reg#(Bool) sgeIsOnlyReg <- mkRegU; + Reg#(Bool) sgeIsLastReg <- mkRegU; + Reg#(Bool) hasLessFragReg <- mkRegU; + Reg#(DataStream) prePayloadFragReg <- mkRegU; + Reg#(MergePayloadStateAllSGE) stateReg <- mkReg(MERGE_SGL_PAYLOAD_INIT); + + BusByteWidthMask busByteNumMask = maxBound; + BusBitWidthMask busBitNumMask = maxBound; + + // TODO: emptyDataStream should be an input parameter + let emptyDataStream = DataStream { + data: 0, + byteEn: 0, + isFirst: False, + isLast: False + }; + + function ActionValue#(DataStream) preprocessNextSGL(); + actionvalue + let sgeMergedMetaData = sgeMergedMetaDataPipeIn.first; + sgeMergedMetaDataPipeIn.deq; + + let isOnlySGE = sgeMergedMetaData.isFirst && sgeMergedMetaData.isLast; + sgeIsOnlyReg <= isOnlySGE; + sgeIsLastReg <= sgeMergedMetaData.isLast; + isFirstFragReg <= True; + + let lastFragValidByteNum = sgeMergedMetaData.lastFragValidByteNum; + let hasLessFrag = lastFragValidByteNum < fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)); + let curPayloadFrag = sgeMergedPayloadPipeIn.first; + let nextPrePayloadFrag = curPayloadFrag; + + let nextHasLessFrag = hasLessFrag; + // If first SGE is single fragment without full byteEn, + // Wait for the first fragment of next SGE to merge + if (!isOnlySGE && curPayloadFrag.isLast && hasLessFrag) begin + nextPrePayloadFrag = emptyDataStream; + nextHasLessFrag = True; + end + else begin + sgeMergedPayloadPipeIn.deq; + nextHasLessFrag = False; + end + hasLessFragReg <= nextHasLessFrag; + + let { + lastFragValidBitNum, lastFragInvalidByteNum, lastFragInvalidBitNum + } = calcFragBitNumAndByteNum(lastFragValidByteNum); + + // curValidByteNumReg <= nextHasLessFrag ? 0 : fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)); + curInvalidByteNumReg <= nextHasLessFrag ? fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)) : 0; + curInvalidBitNumReg <= nextHasLessFrag ? fromInteger(valueOf(DATA_BUS_WIDTH)) : 0; + // nextValidByteNumReg <= lastFragValidByteNum; + nextInvalidByteNumReg <= lastFragInvalidByteNum; + nextInvalidBitNumReg <= lastFragInvalidBitNum; + + case ({ pack(sgeMergedMetaData.isFirst), pack(sgeMergedMetaData.isLast) }) + 2'b11, 2'b01: begin + stateReg <= MERGE_SGL_PAYLOAD_LAST_OR_ONLY_SGE; + end + 2'b10, 2'b00: begin + stateReg <= MERGE_SGL_PAYLOAD_FIRST_OR_MID_SGE; + end + endcase + // $display( + // "time=%0t: preprocessNextSGL", $time, + // ", isOnlySGE=", fshow(isOnlySGE), + // ", hasLessFrag=", fshow(hasLessFrag), + // ", nextHasLessFrag=", fshow(nextHasLessFrag), + // ", sgeMergedMetaData.isFirst=", fshow(sgeMergedMetaData.isFirst), + // ", sgeMergedMetaData.isLast=", fshow(sgeMergedMetaData.isLast), + // ", lastFragValidByteNum=%0d", lastFragValidByteNum, + // ", lastFragValidBitNum=%0d", lastFragValidBitNum, + // ", lastFragInvalidByteNum=%0d", lastFragInvalidByteNum, + // ", lastFragInvalidBitNum=%0d", lastFragInvalidBitNum, + // // ", curValidByteNumReg=%0d", curValidByteNumReg, + // ", curInvalidByteNumReg=%0d", curInvalidByteNumReg, + // ", curInvalidBitNumReg=%0d", curInvalidBitNumReg, + // // ", nextValidByteNumReg=%0d", nextValidByteNumReg, + // ", nextInvalidByteNumReg=%0d", nextInvalidByteNumReg, + // ", nextInvalidBitNumReg=%0d", nextInvalidBitNumReg, + // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), + // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), + // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn, + // ", nextPrePayloadFrag.isFirst=", fshow(nextPrePayloadFrag.isFirst), + // ", nextPrePayloadFrag.isLast=", fshow(nextPrePayloadFrag.isLast), + // ", nextPrePayloadFrag.byteEn=%h", nextPrePayloadFrag.byteEn + // ); + return nextPrePayloadFrag; + endactionvalue + endfunction + + function Action preprocessNextSGE(); + action + let sgeMergedMetaData = sgeMergedMetaDataPipeIn.first; + sgeMergedMetaDataPipeIn.deq; + + // let isOnlySGE = sgeMergedMetaData.isFirst && sgeMergedMetaData.isLast; + // sgeIsOnlyReg <= isOnlySGE; + sgeIsLastReg <= sgeMergedMetaData.isLast; + + let lastFragValidByteNum = sgeMergedMetaData.lastFragValidByteNum; + let { + lastFragValidBitNum, lastFragInvalidByteNum, lastFragInvalidBitNum + } = calcFragBitNumAndByteNum(lastFragValidByteNum); + + // let sumValidByteNum = nextValidByteNumReg + lastFragValidByteNum; + let sumInvalidByteNum = nextInvalidByteNumReg + lastFragInvalidByteNum; + let sumInvalidBitNum = nextInvalidBitNumReg + lastFragInvalidBitNum; + + // let hasLessFrag = !sgeIsOnlyReg && !(sumValidByteNum > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); + let hasLessFrag = !sgeIsOnlyReg && (sumInvalidByteNum >= fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); + hasLessFragReg <= hasLessFrag; + + // let curValidByteNum = nextValidByteNumReg; + let curInvalidByteNum = nextInvalidByteNumReg; + let curInvalidBitNum = nextInvalidBitNumReg; + // curValidByteNumReg <= curValidByteNum; + curInvalidByteNumReg <= curInvalidByteNum; + curInvalidBitNumReg <= curInvalidBitNum; + + // let nextValidByteNum = sumValidByteNum & zeroExtend(busByteNumMask); + let nextInvalidByteNum = sumInvalidByteNum & zeroExtend(busByteNumMask); + let nextInvalidBitNum = sumInvalidBitNum & zeroExtend(busBitNumMask); + // nextValidByteNumReg <= nextValidByteNum; + nextInvalidByteNumReg <= nextInvalidByteNum; + nextInvalidBitNumReg <= nextInvalidBitNum; + + case ({ pack(sgeMergedMetaData.isFirst), pack(sgeMergedMetaData.isLast) }) + 2'b11, 2'b01: begin + stateReg <= MERGE_SGL_PAYLOAD_LAST_OR_ONLY_SGE; + end + 2'b10, 2'b00: begin + stateReg <= MERGE_SGL_PAYLOAD_FIRST_OR_MID_SGE; + end + endcase + // $display( + // "time=%0t: preprocessNextSGE", $time, + // ", hasLessFrag=", fshow(hasLessFrag), + // ", sgeMergedMetaData.isFirst=", fshow(sgeMergedMetaData.isFirst), + // ", sgeMergedMetaData.isLast=", fshow(sgeMergedMetaData.isLast), + // ", lastFragValidByteNum=%0d", lastFragValidByteNum, + // ", lastFragValidBitNum=%0d", lastFragValidBitNum, + // ", lastFragInvalidByteNum=%0d", lastFragInvalidByteNum, + // ", lastFragInvalidBitNum=%0d", lastFragInvalidBitNum, + // // ", sumValidByteNum=%0d", sumValidByteNum, + // ", sumInvalidByteNum=%0d", sumInvalidByteNum, + // ", sumInvalidBitNum=%0d", sumInvalidBitNum, + // // ", curValidByteNum=%0d", curValidByteNum, + // ", curInvalidByteNum=%0d", curInvalidByteNum, + // ", curInvalidBitNum=%0d", curInvalidBitNum, + // // ", nextValidByteNum=%0d", nextValidByteNum, + // ", nextInvalidByteNum=%0d", nextInvalidByteNum, + // ", nextInvalidBitNum=%0d", nextInvalidBitNum + // ); + endaction + endfunction + + rule resetAndClear if (clearAll); + pktPayloadOutQ.clear; + + stateReg <= MERGE_SGL_PAYLOAD_INIT; + endrule + + rule init if (!clearAll && stateReg == MERGE_SGL_PAYLOAD_INIT); + let nextPayloadFrag <- preprocessNextSGL; + prePayloadFragReg <= nextPayloadFrag; + endrule + + rule mergeFirstOrMidSGE if (!clearAll && stateReg == MERGE_SGL_PAYLOAD_FIRST_OR_MID_SGE); + let curPayloadFrag = sgeMergedPayloadPipeIn.first; + sgeMergedPayloadPipeIn.deq; + + let mergedFrag = mergeFragData( + prePayloadFragReg, curPayloadFrag, + curInvalidByteNumReg, curInvalidBitNumReg + ); + // In case prePayloadFragReg is emptyDataStream, + // then it should set mergeFrag.isFirst as true. + mergedFrag.isFirst = isFirstFragReg; + + let shouldOutput = True; + let nextPrePayloadFrag = mergedFrag; + if (curPayloadFrag.isLast) begin + preprocessNextSGE; + + if (hasLessFragReg) begin + shouldOutput = False; + nextPrePayloadFrag.byteEn = mergedFrag.byteEn >> nextInvalidByteNumReg; + nextPrePayloadFrag.data = mergedFrag.data >> nextInvalidBitNumReg; + end + end + prePayloadFragReg <= nextPrePayloadFrag; + + let outPayloadFrag = mergedFrag; + // outPayloadFrag.isFirst = isFirstFragReg; + outPayloadFrag.isLast = False; + if (shouldOutput) begin + isFirstFragReg <= False; + + pktPayloadOutQ.enq(outPayloadFrag); + end + // $display( + // "time=%0t: mergeFirstOrMidSGE", $time, + // ", shouldOutput=", fshow(shouldOutput), + // ", hasLessFragReg=", fshow(hasLessFragReg), + // ", curInvalidByteNumReg=%0d", curInvalidByteNumReg, + // ", curInvalidBitNumReg=%0d", curInvalidBitNumReg, + // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), + // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), + // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn, + // ", mergedFrag.isFirst=", fshow(mergedFrag.isFirst), + // ", mergedFrag.isLast=", fshow(mergedFrag.isLast), + // ", mergedFrag.byteEn=%h", mergedFrag.byteEn, + // ", nextPrePayloadFrag.isFirst=", fshow(nextPrePayloadFrag.isFirst), + // ", nextPrePayloadFrag.isLast=", fshow(nextPrePayloadFrag.isLast), + // ", nextPrePayloadFrag.byteEn=%h", nextPrePayloadFrag.byteEn, + // ", outPayloadFrag.isFirst=", fshow(outPayloadFrag.isFirst), + // ", outPayloadFrag.isLast=", fshow(outPayloadFrag.isLast), + // ", outPayloadFrag.byteEn=%h", outPayloadFrag.byteEn + // ); + endrule + + rule mergeLastOrOnlySGE if (!clearAll && stateReg == MERGE_SGL_PAYLOAD_LAST_OR_ONLY_SGE); + let nextPayloadFrag = emptyDataStream; + let outPayloadFrag = prePayloadFragReg; + + let isLastFrag = prePayloadFragReg.isLast; + if (prePayloadFragReg.isLast) begin + if (sgeMergedMetaDataPipeIn.notEmpty && sgeMergedPayloadPipeIn.notEmpty) begin + nextPayloadFrag <- preprocessNextSGL; + end + else begin + // Wait for a SGE of next SGL, if no next SGE metadata or payload + stateReg <= MERGE_SGL_PAYLOAD_INIT; + end + end + else begin + nextPayloadFrag = sgeMergedPayloadPipeIn.first; + sgeMergedPayloadPipeIn.deq; + + nextPayloadFrag.isFirst = False; + if (!sgeIsOnlyReg && hasLessFragReg && nextPayloadFrag.isLast) begin + // Has one less fragment + stateReg <= MERGE_SGL_PAYLOAD_INIT; + isLastFrag = True; + end + end + prePayloadFragReg <= nextPayloadFrag; + + outPayloadFrag = mergeFragData( + prePayloadFragReg, nextPayloadFrag, + curInvalidByteNumReg, curInvalidBitNumReg + ); + outPayloadFrag.isLast = isLastFrag; + pktPayloadOutQ.enq(outPayloadFrag); + // $display( + // "time=%0t: mergeLastOrOnlySGE", $time, + // ", sgeIsOnlyReg=", fshow(sgeIsOnlyReg), + // ", hasLessFragReg=", fshow(hasLessFragReg), + // ", prePayloadFragReg.isFirst=", fshow(prePayloadFragReg.isFirst), + // ", prePayloadFragReg.isLast=", fshow(prePayloadFragReg.isLast), + // ", prePayloadFragReg.byteEn=%h", prePayloadFragReg.byteEn, + // ", nextPayloadFrag.isFirst=", fshow(nextPayloadFrag.isFirst), + // ", nextPayloadFrag.isLast=", fshow(nextPayloadFrag.isLast), + // ", nextPayloadFrag.byteEn=%h", nextPayloadFrag.byteEn, + // ", outPayloadFrag.isFirst=", fshow(outPayloadFrag.isFirst), + // ", outPayloadFrag.isLast=", fshow(outPayloadFrag.isLast), + // ", outPayloadFrag.byteEn=%h", outPayloadFrag.byteEn + // ); + endrule + + return toPipeOut(pktPayloadOutQ); +endmodule + +interface PayloadSegment; + interface PipeOut#(PktLen) pktLenPipeOut; + interface DataStreamPipeOut pktPayloadPipeOut; +endinterface + +typedef enum { + ADJUST_PAYLOAD_SEGMENT_INIT, + ADJUST_PAYLOAD_SEGMENT_FIRST_OR_MID_PKT, + ADJUST_PAYLOAD_SEGMENT_LAST_OR_ONLY_PKT + // ADJUST_PAYLOAD_SEGMENT_FIRST_PKT, + // ADJUST_PAYLOAD_SEGMENT_MID_PKT, + // ADJUST_PAYLOAD_SEGMENT_LAST_PKT, + // ADJUST_PAYLOAD_SEGMENT_ONLY_PKT, + // ADJUST_PAYLOAD_SEGMENT_LAST_PKT_EXTRA_FRAG +} AdjustPayloadSegmentState deriving(Bits, Eq, FShow); + +module mkAdjustPayloadSegment#( + Bool clearAll, + PipeOut#(AdjustedTotalPayloadMetaData) adjustedTotalPayloadMetaDataPipeIn, + PipeOut#(DataStream) sglAllPayloadPipeIn +)(DataStreamPipeOut); + FIFOF#(DataStream) pktPayloadOutQ <- mkFIFOF; + + Reg#(DataStream) prePayloadFragReg <- mkRegU; + Reg#(ByteEn) firstPktLastFragByteEnReg <- mkRegU; + + Reg#(ByteEnBitNum) firstPktLastFragValidByteNumReg <- mkRegU; + Reg#(BusBitNum) firstPktLastFragValidBitNumReg <- mkRegU; + // Reg#(ByteEnBitNum) firstPktLastFragInvalidByteNumReg <- mkRegU; + // Reg#(BusBitNum) firstPktLastFragInvalidBitNumReg <- mkRegU; + // Reg#(ByteEnBitNum) lastPktLastFragInvalidByteNumReg <- mkRegU; + // Reg#(BusBitNum) lastPktLastFragInvalidBitNumReg <- mkRegU; + + // Reg#(Bool) isFirstFragReg <- mkRegU; + Reg#(Bool) isFirstPktReg <- mkRegU; + Reg#(Bool) hasExtraFragReg <- mkRegU; + Reg#(Bool) sglHasOnlyPktReg <- mkRegU; + Reg#(Bool) sglHasOnlyFragOnlyPktReg <- mkRegU; + + // Reg#(PktLen) pmtuPktLenReg <- mkRegU; + // Reg#(PktFragNum) lastPktFragNumReg <- mkRegU; + Reg#(PktFragNum) pmtuFragNumReg <- mkRegU; + Reg#(PktFragNum) pktRemainingFragNumReg <- mkRegU; + Reg#(PktNum) sglRemainingPktNumReg <- mkRegU; + Reg#(AdjustPayloadSegmentState) stateReg <- mkReg(ADJUST_PAYLOAD_SEGMENT_INIT); + + let emptyDataStream = DataStream { + data: 0, + byteEn: 0, + isFirst: False, + isLast: False + }; + + rule resetAndClear if (clearAll); + // pktLenOutQ.clear; + pktPayloadOutQ.clear; + stateReg <= ADJUST_PAYLOAD_SEGMENT_INIT; + endrule + + function ActionValue#(DataStream) prepareNextSGL(); + actionvalue + let adjustedTotalPayloadMeta = adjustedTotalPayloadMetaDataPipeIn.first; + adjustedTotalPayloadMetaDataPipeIn.deq; + + // pmtuPktLenReg <= calcPmtuLen(adjustedTotalPayloadMeta.pmtu); + pmtuFragNumReg <= calcFragNumByPMTU(adjustedTotalPayloadMeta.pmtu); + + pktRemainingFragNumReg <= adjustedTotalPayloadMeta.firstPktFragNum; + sglRemainingPktNumReg <= adjustedTotalPayloadMeta.adjustedPktNum; + // lastPktFragNumReg <= adjustedTotalPayloadMeta.lastPktFragNum; + + let origLastFragValidByteNum = adjustedTotalPayloadMeta.origLastFragValidByteNum; + let firstPktLastFragValidByteNum = adjustedTotalPayloadMeta.firstPktLastFragValidByteNum; + // let lastPktLastFragValidByteNum = adjustedTotalPayloadMeta.lastPktLastFragValidByteNum; + let { + firstPktLastFragValidBitNum, + firstPktLastFragInvalidByteNum, + firstPktLastFragInvalidBitNum + } = calcFragBitNumAndByteNum(firstPktLastFragValidByteNum); + + firstPktLastFragValidByteNumReg <= firstPktLastFragValidByteNum; + firstPktLastFragValidBitNumReg <= firstPktLastFragValidBitNum; + // firstPktLastFragInvalidByteNumReg <= firstPktLastFragInvalidByteNum; + // firstPktLastFragInvalidBitNumReg <= firstPktLastFragInvalidBitNum; + firstPktLastFragByteEnReg <= genByteEn(firstPktLastFragValidByteNum); + + // let { + // lastPktLastFragValidBitNum, + // lastPktLastFragInvalidByteNum, + // lastPktLastFragInvalidBitNum + // } = calcFragBitNumAndByteNum(lastPktLastFragValidByteNum); + // lastPktLastFragInvalidByteNumReg <= lastPktLastFragInvalidByteNum; + // lastPktLastFragInvalidBitNumReg <= lastPktLastFragInvalidBitNum; + let sglHasOnlyPkt = isOneR(adjustedTotalPayloadMeta.adjustedPktNum); + sglHasOnlyPktReg <= sglHasOnlyPkt; + let hasExtraFrag = !sglHasOnlyPkt && (( + { 1'b0, firstPktLastFragInvalidByteNum } + + { 1'b0, origLastFragValidByteNum } + ) > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); + hasExtraFragReg <= hasExtraFrag; + + if (sglHasOnlyPkt) begin + stateReg <= ADJUST_PAYLOAD_SEGMENT_LAST_OR_ONLY_PKT; + end + else begin + stateReg <= ADJUST_PAYLOAD_SEGMENT_FIRST_OR_MID_PKT; + end + + let curPayloadFrag = sglAllPayloadPipeIn.first; + sglAllPayloadPipeIn.deq; + + let isOrigOnlyPkt = isOneR(adjustedTotalPayloadMeta.origPktNum); + let sglHasOnlyFragOnlyPkt = isOrigOnlyPkt && curPayloadFrag.isLast; + sglHasOnlyFragOnlyPktReg <= sglHasOnlyFragOnlyPkt; + + // isFirstFragReg <= True; + isFirstPktReg <= True; + // $display( + // "time=%0t: prepareNextSGL", $time, + // ", sglHasOnlyPkt=", fshow(sglHasOnlyPkt), + // ", hasExtraFrag=", fshow(hasExtraFrag), + // ", hasExtraFragReg=", fshow(hasExtraFragReg), + // // ", sgeMergedMetaData.isFirst=", fshow(sgeMergedMetaData.isFirst), + // // ", sgeMergedMetaData.isLast=", fshow(sgeMergedMetaData.isLast), + // ", firstPktLastFragValidByteNum=%0d", firstPktLastFragValidByteNum, + // ", firstPktLastFragValidBitNum=%0d", firstPktLastFragValidBitNum, + // ", firstPktLastFragInvalidByteNum=%0d", firstPktLastFragInvalidByteNum, + // ", firstPktLastFragInvalidBitNum=%0d", firstPktLastFragInvalidBitNum, + // // ", lastPktLastFragValidByteNum=%0d", lastPktLastFragValidByteNum, + // ", adjustedTotalPayloadMeta.pktNum=%0d", adjustedTotalPayloadMeta.adjustedPktNum, + // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), + // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), + // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn + // // ", nextPrePayloadFrag.isFirst=", fshow(nextPrePayloadFrag.isFirst), + // // ", nextPrePayloadFrag.isLast=", fshow(nextPrePayloadFrag.isLast), + // // ", nextPrePayloadFrag.byteEn=%h", nextPrePayloadFrag.byteEn + // ); + return curPayloadFrag; + endactionvalue + endfunction + + rule init if (!clearAll && stateReg == ADJUST_PAYLOAD_SEGMENT_INIT); + let nextPrePayloadFrag <- prepareNextSGL; + prePayloadFragReg <= nextPrePayloadFrag; + endrule + + rule adjustFirstOrMidPkt if (!clearAll && stateReg == ADJUST_PAYLOAD_SEGMENT_FIRST_OR_MID_PKT); + let isLastFrag = isOneR(pktRemainingFragNumReg); + let outPayloadFrag = prePayloadFragReg; + + if (!sglHasOnlyFragOnlyPktReg) begin + let curPayloadFrag = sglAllPayloadPipeIn.first; + let nextPayloadFrag = curPayloadFrag; + + let mergedFrag = mergeFragData( + prePayloadFragReg, curPayloadFrag, + firstPktLastFragValidByteNumReg, firstPktLastFragValidBitNumReg + ); + + if (!isFirstPktReg) begin + outPayloadFrag = mergedFrag; + end + + if (!(isFirstPktReg && isLastFrag)) begin + sglAllPayloadPipeIn.deq; + end + else begin + // Do not dequeue when the last fragment of the first packet, + // Since the queue head is the first fragment of the next packet, + // So keep the last fragment of the first packet in prePayloadFragReg. + nextPayloadFrag = prePayloadFragReg; + end + nextPayloadFrag.isFirst = isLastFrag; + prePayloadFragReg <= nextPayloadFrag; + end + + if (isLastFrag) begin + if (isFirstPktReg) begin + outPayloadFrag.byteEn = firstPktLastFragByteEnReg; + isFirstPktReg <= False; + end + + // isFirstFragReg <= True; + sglRemainingPktNumReg <= sglRemainingPktNumReg - 1; + + if (isTwoR(sglRemainingPktNumReg)) begin + // pktRemainingFragNumReg <= lastPktFragNumReg; + stateReg <= ADJUST_PAYLOAD_SEGMENT_LAST_OR_ONLY_PKT; + end + else begin + pktRemainingFragNumReg <= pmtuFragNumReg; + stateReg <= ADJUST_PAYLOAD_SEGMENT_FIRST_OR_MID_PKT; + end + end + else begin + // isFirstFragReg <= False; + pktRemainingFragNumReg <= pktRemainingFragNumReg - 1; + end + + // outPayloadFrag.isFirst = isFirstFragReg; + outPayloadFrag.isLast = isLastFrag; + pktPayloadOutQ.enq(outPayloadFrag); + // $display( + // "time=%0t: adjustFirstOrMidPkt", $time, + // ", sglHasOnlyFragOnlyPktReg=", fshow(sglHasOnlyFragOnlyPktReg), + // // ", isFirstFragReg=", fshow(isFirstFragReg), + // ", hasExtraFragReg=", fshow(hasExtraFragReg), + // ", isLastFrag=", fshow(isLastFrag), + // ", pktRemainingFragNumReg=%0d", pktRemainingFragNumReg, + // ", sglRemainingPktNumReg=%0d", sglRemainingPktNumReg, + // ", firstPktLastFragValidByteNumReg=%0d", firstPktLastFragValidByteNumReg, + // ", firstPktLastFragValidBitNumReg=%0d", firstPktLastFragValidBitNumReg, + // ", prePayloadFragReg.isFirst=", fshow(prePayloadFragReg.isFirst), + // ", prePayloadFragReg.isLast=", fshow(prePayloadFragReg.isLast), + // ", prePayloadFragReg.byteEn=%h", prePayloadFragReg.byteEn, + // // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), + // // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), + // // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn, + // // ", mergedFrag.isFirst=", fshow(mergedFrag.isFirst), + // // ", mergedFrag.isLast=", fshow(mergedFrag.isLast), + // // ", mergedFrag.byteEn=%h", mergedFrag.byteEn, + // ", outPayloadFrag.isFirst=", fshow(outPayloadFrag.isFirst), + // ", outPayloadFrag.isLast=", fshow(outPayloadFrag.isLast), + // ", outPayloadFrag.byteEn=%h", outPayloadFrag.byteEn + // ); + endrule + + rule adjustLastOrOnlyPkt if (!clearAll && stateReg == ADJUST_PAYLOAD_SEGMENT_LAST_OR_ONLY_PKT); + let nextPayloadFrag = emptyDataStream; + + let isLastFrag = prePayloadFragReg.isLast; + if (prePayloadFragReg.isLast) begin + if (adjustedTotalPayloadMetaDataPipeIn.notEmpty && sglAllPayloadPipeIn.notEmpty) begin + nextPayloadFrag <- prepareNextSGL; + end + else begin + // Wait for a packet of next SGE, if no next SGE packet metadata or payload + stateReg <= ADJUST_PAYLOAD_SEGMENT_INIT; + end + end + else begin + nextPayloadFrag = sglAllPayloadPipeIn.first; + sglAllPayloadPipeIn.deq; + + // nextPayloadFrag.isFirst = False; + if (!sglHasOnlyPktReg && !hasExtraFragReg && nextPayloadFrag.isLast) begin + // No extra fragment + stateReg <= ADJUST_PAYLOAD_SEGMENT_INIT; + isLastFrag = True; + end + end + + let mergedFrag = mergeFragData( + prePayloadFragReg, nextPayloadFrag, + firstPktLastFragValidByteNumReg, firstPktLastFragValidBitNumReg + ); + prePayloadFragReg <= nextPayloadFrag; + + immAssert( + isOne(sglRemainingPktNumReg), + "sglRemainingPktNumReg assertion @ mkAdjustPayloadSegment", + $format( + "sglRemainingPktNumReg=%0d", fshow(sglRemainingPktNumReg), + " should be one when stateReg=", fshow(stateReg) + ) + ); + + let outPayloadFrag = prePayloadFragReg; + if (!sglHasOnlyPktReg) begin + // isFirstFragReg <= False; + // outPayloadFrag.isFirst = isFirstFragReg; + outPayloadFrag = mergedFrag; + outPayloadFrag.isLast = isLastFrag; + end + pktPayloadOutQ.enq(outPayloadFrag); + // $display( + // "time=%0t: adjustLastOrOnlyPkt", $time, + // ", sglHasOnlyPktReg=", fshow(sglHasOnlyPktReg), + // ", hasExtraFragReg=", fshow(hasExtraFragReg), + // ", prePayloadFragReg.isFirst=", fshow(prePayloadFragReg.isFirst), + // ", prePayloadFragReg.isLast=", fshow(prePayloadFragReg.isLast), + // ", prePayloadFragReg.byteEn=%h", prePayloadFragReg.byteEn, + // ", nextPayloadFrag.isFirst=", fshow(nextPayloadFrag.isFirst), + // ", nextPayloadFrag.isLast=", fshow(nextPayloadFrag.isLast), + // ", nextPayloadFrag.byteEn=%h", nextPayloadFrag.byteEn, + // ", outPayloadFrag.isFirst=", fshow(outPayloadFrag.isFirst), + // ", outPayloadFrag.isLast=", fshow(outPayloadFrag.isLast), + // ", outPayloadFrag.byteEn=%h", outPayloadFrag.byteEn + // ); + endrule + + return toPipeOut(pktPayloadOutQ); +endmodule + +interface BramPipe#(type anytype); + interface PipeOut#(anytype) pipeOut; + method Action clear(); + method Bool notEmpty(); +endinterface + +module mkConnectBramQ2PipeOut#(FIFOF#(anytype) bramQ)( + BramPipe#(anytype) +) provisos(Bits#(anytype, tSz)); + FIFOF#(anytype) postBramQ <- mkFIFOF; + + mkConnection(toPut(postBramQ), toGet(bramQ)); + + interface pipeOut = toPipeOut(postBramQ); + method Action clear(); + postBramQ.clear; + endmethod + method Bool notEmpty() = bramQ.notEmpty && postBramQ.notEmpty; +endmodule + +module mkConnectPipeOut2BramQ#( + PipeOut#(anytype) pipeIn, FIFOF#(anytype) bramQ +)(BramPipe#(anytype)) provisos(Bits#(anytype, tSz)); + FIFOF#(anytype) postBramQ <- mkFIFOF; + + mkConnection(toPut(bramQ), toGet(pipeIn)); + mkConnection(toPut(postBramQ), toGet(bramQ)); + + interface pipeOut = toPipeOut(postBramQ); + method Action clear(); + postBramQ.clear; + endmethod + method Bool notEmpty() = bramQ.notEmpty && postBramQ.notEmpty; +endmodule + +typedef struct { + WorkQueueElem wqe; + // DmaReadMetaDataSGL sglDmaReadMetaData; + // PMTU pmtu; + // ADDR raddr; + // Bool segment; + // Bool addPadding; +} PayloadGenReqSG deriving(Bits, FShow); + +typedef struct { + // Bool segment; + // Bool addPadding; + Bool isRespErr; +} PayloadGenRespSG deriving(Bits, FShow); + +interface PayloadGenerator; + interface Server#(PayloadGenReqSG, PayloadGenRespSG) srvPort; + interface DataStreamPipeOut payloadDataStreamPipeOut; + method Bool payloadNotEmpty(); +endinterface + +module mkPayloadGenerator#( + Bool clearAll, DmaReadCntrl dmaReadCntrl +)(PayloadGenerator); + FIFOF#(PayloadGenReqSG) payloadGenReqQ <- mkFIFOF; + FIFOF#(PayloadGenRespSG) payloadGenRespQ <- mkFIFOF; + + // Pipeline FIFO + FIFOF#(Tuple2#(PayloadGenReqSG, Bool)) adjustReqPktLenQ <- mkFIFOF; + FIFOF#(Tuple8#(PayloadGenReqSG, Length, PktNum, ResiduePMTU, PMTU, PktLen, PktNum, Bool)) adjustedFirstAndLastPktLenQ <- mkFIFOF; + FIFOF#(Tuple2#(PayloadGenReqSG, AdjustedTotalPayloadMetaData)) adjustedTotalPayloadMetaDataQ <- mkFIFOF; + // FIFOF#(Tuple3#(PayloadGenReqSG, ByteEn, PktFragNum)) pendingGenReqQ <- mkFIFOF; + // FIFOF#(Tuple3#(Bool, Bool, PktFragNum)) pendingGenRespQ <- mkFIFOF; + // FIFOF#(Tuple3#(DataStream, Bool, Bool)) payloadSegmentQ <- mkFIFOF; + + // TODO: check payloadOutQ buffer size is enough for DMA read delay? + FIFOF#(DataStream) payloadBufQ <- mkSizedBRAMFIFOF(valueOf(DATA_STREAM_FRAG_BUF_SIZE)); + let bramQ2PipeOut <- mkConnectBramQ2PipeOut(payloadBufQ); + let payloadBufPipeOut = bramQ2PipeOut.pipeOut; + // let payloadBufPipeOut <- mkConnectBramQ2PipeOut(payloadBufQ); + + Reg#(PktFragNum) pmtuFragCntReg <- mkRegU; + Reg#(Bool) shouldSetFirstReg <- mkReg(False); + Reg#(Bool) isFragCntZeroReg <- mkReg(False); + Reg#(Bool) isNormalStateReg <- mkReg(True); + + rule resetAndClear if (clearAll); + payloadGenReqQ.clear; + payloadGenRespQ.clear; + + adjustReqPktLenQ.clear; + adjustedFirstAndLastPktLenQ.clear; + adjustedTotalPayloadMetaDataQ.clear; + // pendingGenReqQ.clear; + // pendingGenRespQ.clear; + // payloadSegmentQ.clear; + payloadBufQ.clear; + + bramQ2PipeOut.clear; + + shouldSetFirstReg <= False; + isFragCntZeroReg <= False; + isNormalStateReg <= True; + + // $display( + // "time=%0t: reset and clear mkPayloadGenerator", $time, + // ", pendingGenReqQ.notEmpty=", fshow(pendingGenReqQ.notEmpty) + // ); + endrule + + // rule debugNotFull if (!( + // payloadGenRespQ.notFull && + // pendingGenReqQ.notFull && + // // pendingGenRespQ.notFull && + // // payloadSegmentQ.notFull && + // payloadBufQ.notFull + // )); + // $display( + // "time=%0t: mkPayloadGenerator debugNotFull", $time, + // ", qpn=%h", cntrlStatus.comm.getSQPN, + // ", isSQ=", fshow(cntrlStatus.isSQ), + // ", payloadGenReqQ.notEmpty=", fshow(payloadGenReqQ.notEmpty), + // ", payloadGenRespQ.notFull=", fshow(payloadGenRespQ.notFull), + // ", pendingGenReqQ.notFull=", fshow(pendingGenReqQ.notFull), + // // ", pendingGenRespQ.notFull=", fshow(pendingGenRespQ.notFull), + // // ", payloadSegmentQ.notFull=", fshow(payloadSegmentQ.notFull), + // ", payloadBufQ.notFull=", fshow(payloadBufQ.notFull) + // ); + // endrule + + // rule debugNotEmpty if (!( + // payloadGenRespQ.notEmpty && + // pendingGenReqQ.notEmpty && + // // pendingGenRespQ.notEmpty && + // // payloadSegmentQ.notEmpty && + // payloadBufQ.notEmpty + // )); + // $display( + // "time=%0t: mkPayloadGenerator debugNotEmpty", $time, + // ", qpn=%h", cntrlStatus.comm.getSQPN, + // ", isSQ=", fshow(cntrlStatus.isSQ), + // ", payloadGenReqQ.notEmpty=", fshow(payloadGenReqQ.notEmpty), + // ", payloadGenRespQ.notEmpty=", fshow(payloadGenRespQ.notEmpty), + // ", pendingGenReqQ.notEmpty=", fshow(pendingGenReqQ.notEmpty), + // // ", pendingGenRespQ.notEmpty=", fshow(pendingGenRespQ.notEmpty), + // // ", payloadSegmentQ.notEmpty=", fshow(payloadSegmentQ.notEmpty), + // ", payloadBufQ.notEmpty=", fshow(payloadBufQ.notEmpty) + // ); + // endrule + + rule recvReq if (!clearAll); + let payloadGenReq = payloadGenReqQ.first; + payloadGenReqQ.deq; + + let wqe = payloadGenReq.wqe; + let sglIdx = 0; + let sge = wqe.sgl[sglIdx]; + // If first SGE has zero length, then whole WQE has no payload + let isZeroPayloadLen = isZeroR(sge.len); + if (isZeroPayloadLen) begin + immAssert( + sge.isLast, + "last SGE assertion @ mkDmaReadCntrl", + $format( + "sge.isLast=", fshow(sge.isLast), + " should be true when sglIdx=%d", sglIdx, + ", and sge.len=%d", sge.len + ) + ); + end + + if (!isZeroPayloadLen) begin + let dmaReadCntrlReq = DmaReadCntrlReq { + sglDmaReadMetaData: DmaReadMetaDataSGL { + sgl : wqe.sgl, + sqpn : wqe.sqpn, + wrID : wqe.id + }, + pmtu : wqe.pmtu + }; + dmaReadCntrl.srvPort.request.put(dmaReadCntrlReq); + end + adjustReqPktLenQ.enq(tuple2(payloadGenReq, isZeroPayloadLen)); + $display( + "time=%0t: mkPayloadGenerator recvReq", $time, + ", payloadGenReq=", fshow(payloadGenReq) + ); + endrule + + rule adjustFirstAndLastPktLen if (!clearAll); + let { payloadGenReq, isZeroPayloadLen } = adjustReqPktLenQ.first; + adjustReqPktLenQ.deq; + let wqe = payloadGenReq.wqe; + + let totalLen = 0; + if (!isZeroPayloadLen) begin + let totalPayloadMetaData = dmaReadCntrl.totalPayloadMetaDataPipeOut.first; + dmaReadCntrl.totalPayloadMetaDataPipeOut.deq; + totalLen = totalPayloadMetaData.totalLen; + end + + let { truncatedPktNum, residue } = truncateLenByPMTU(totalLen, wqe.pmtu); + let { + pmtuLen, firstPktLen, lastPktLen, totalPktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU( + wqe.raddr, totalLen, wqe.pmtu + ); + + adjustedFirstAndLastPktLenQ.enq(tuple8( + payloadGenReq, totalLen, truncatedPktNum, residue, + wqe.pmtu, firstPktLen, totalPktNum, isZeroPayloadLen + )); + $display( + "time=%0t: mkPayloadGenerator adjustFirstAndLastPktLen", $time, + ", payloadGenReq=", fshow(payloadGenReq) + ); + endrule + + rule calcDetailTotalPayloadMetaData if (!clearAll); + let { + payloadGenReq, totalLen, truncatedPktNum, residue, + pmtu, firstPktLen, totalPktNum, isZeroPayloadLen + } = adjustedFirstAndLastPktLenQ.first; + adjustedFirstAndLastPktLenQ.deq; + + let origLastFragValidByteNum = calcLastFragValidByteNum(totalLen); + let firstPktLastFragValidByteNum = calcLastFragValidByteNum(firstPktLen); + // let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLen); + let firstPktFragNum = calcFragNumByPktLen(firstPktLen); + // let lastPktFragNum = calcFragNumByPktLen(lastPktLen); + // let firstPktPadCnt = calcPadCnt(firstPktLen); + // let lastPktPadCnt = calcPadCnt(lastPktLen); + let origPktNum = truncatedPktNum + (isZeroR(residue) ? 0 : 1); + + let adjustedTotalPayloadMetaData = AdjustedTotalPayloadMetaData { + firstPktLen : firstPktLen, + firstPktFragNum : firstPktFragNum, + firstPktLastFragValidByteNum : firstPktLastFragValidByteNum, + // firstPktPadCnt : firstPktPadCnt, + // lastPktLen : lastPktLen, + // lastPktFragNum : lastPktFragNum, + // lastPktLastFragValidByteNum : lastPktLastFragValidByteNum, + // lastPktPadCnt : lastPktPadCnt, + origLastFragValidByteNum : origLastFragValidByteNum, + adjustedPktNum : totalPktNum, + origPktNum : origPktNum, + pmtu : pmtu + // totalLen : totalLen + }; + if (!isZeroPayloadLen) begin + adjustedTotalPayloadMetaDataQ.enq(tuple2( + payloadGenReq, adjustedTotalPayloadMetaData + )); + end + $display( + "time=%0t: mkPayloadGenerator calcDetailTotalPayloadMetaData", $time, + ", payloadGenReq=", fshow(payloadGenReq) + ); + endrule + + interface srvPort = toGPServer(payloadGenReqQ, payloadGenRespQ); + interface payloadDataStreamPipeOut = payloadBufPipeOut; + method Bool payloadNotEmpty() = bramQ2PipeOut.notEmpty; +endmodule diff --git a/src/Utils.bsv b/src/Utils.bsv index db52629..6dece30 100644 --- a/src/Utils.bsv +++ b/src/Utils.bsv @@ -170,7 +170,7 @@ function ByteEnBitNum calcLastFragValidByteNum(Bit#(nSz) len) provisos( Bit#(TSub#(nSz, DATA_BUS_BYTE_NUM_WIDTH)) truncatedLen = truncateLSB(len); ByteEnBitNum lastFragValidByteNum = zeroExtend(lastFragByteNumResidue); - if (isZero(lastFragByteNumResidue) && !isZero(truncatedLen)) begin + if (isZeroR(lastFragByteNumResidue) && !isZeroR(truncatedLen)) begin // BusByteWidthMask busByteWidthMask = maxBound; // ByteEnBitNum lastFragValidByteNum = zeroExtend(truncate(len) & busByteWidthMask); // if (isZero(lastFragValidByteNum) && !isZero(len)) begin diff --git a/test/TestPayloadGen.bsv b/test/TestPayloadGen.bsv new file mode 100644 index 0000000..956bdfc --- /dev/null +++ b/test/TestPayloadGen.bsv @@ -0,0 +1,1401 @@ +import BuildVector :: *; +import ClientServer :: *; +import FIFOF :: *; +import GetPut :: *; +import PAClib :: *; +import Vector :: *; + +import Headers :: *; +import Controller :: *; +import DataTypes :: *; +import MetaData :: *; +import PayloadGen :: *; +import PrimUtils :: *; +import Settings :: *; +import SimDma :: *; +import Utils :: *; +import Utils4Test :: *; + +(* doc = "testcase" *) +module mkTestAddrChunkSrv(Empty); + let minPayloadLen = 2048; + let maxPayloadLen = 8192; + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + FIFOF#(AddrChunkReq) reqQ <- mkFIFOF; + FIFOF#(AddrChunkResp) respQ <- mkFIFOF; + FIFOF#(Length) totalLenQ <- mkFIFOF; + FIFOF#(PktMetaDataSGE) sgePktMetaDataRefQ <- mkFIFOF; + + Reg#(Bool) clearReg <- mkReg(True); + let dut <- mkAddrChunkSrv(clearReg); + + PipeOut#(Tuple2#(Bool, Bool)) firstLastPipeOut <- mkGenericRandomPipeOut; + PipeOut#(ADDR) startAddrPipeOut <- mkGenericRandomPipeOut; + let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); + let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); + + Reg#(PktLen) firstPktLenReg <- mkRegU; + Reg#(PktLen) lastPktLenReg <- mkRegU; + Reg#(PktLen) pmtuLenReg <- mkRegU; + Reg#(PMTU) pmtuReg <- mkRegU; + Reg#(Bool) isFirstReg <- mkRegU; + Reg#(PktNum) pktNumReg <- mkRegU; + Reg#(ADDR) nextAddrReg <- mkRegU; + Reg#(Length) totalLenReg <- mkRegU; + + Reg#(PktNum) remainingPktNumReg <- mkReg(0); + + Reg#(Bool) busyReg <- mkReg(False); + PSN oneAsPSN = 1; + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + // mkSink(dut.sgePktMetaDataPipeOut); + + rule clearAll if (clearReg); + clearReg <= False; + endrule + + rule issueReq if (!clearReg && !busyReg); + let startAddr = startAddrPipeOut.first; + startAddrPipeOut.deq; + + let payloadLen = payloadLenPipeOut.first; + payloadLenPipeOut.deq; + + let pmtu = pmtuPipeOut.first; + pmtuPipeOut.deq; + + let { isOrigFirst, isOrigLast } = firstLastPipeOut.first; + firstLastPipeOut.deq; + + let pmtuLen = calcPmtuLen(pmtu); + let alignedAddr = alignAddrByPMTU(startAddr, pmtu); + let invalidLen = startAddr - alignedAddr; + let tmpFirstPktLen = pmtuLen - truncate(invalidLen); + let totalLen = payloadLen + truncate(invalidLen); + + let totalPktNum = calcPktNumByLenOnly(totalLen, pmtu); + let { tmpPktNum, pmtuResidue } = truncateLenByPMTU( + totalLen, pmtu + ); + let isOnlyPkt = isLessOrEqOne(totalPktNum); + let isZeroResidue = isZero(pmtuResidue); + // let totalPktNum = tmpPktNum + (isZero(pmtuResidue) ? 0 : 1); + // residueReg <= pmtuResidue; + pktNumReg <= totalPktNum; + pmtuReg <= pmtu; + pmtuLenReg <= pmtuLen; + firstPktLenReg <= isOnlyPkt ? truncate(payloadLen) : tmpFirstPktLen; + lastPktLenReg <= isZeroResidue ? pmtuLen : zeroExtend(pmtuResidue); + nextAddrReg <= alignedAddr; + isFirstReg <= True; + busyReg <= True; + + let addrChunkReq = AddrChunkReq { + startAddr: startAddr, + len : payloadLen, + pmtu : pmtu, + isFirst : isOrigFirst, + isLast : isOrigLast + }; + dut.srvPort.request.put(addrChunkReq); + reqQ.enq(addrChunkReq); + + countDown.decr; + // $display( + // "time=%0t: issueReq", $time, + // ", startAddr=%h", startAddr, + // ", payloadLen=%0d", payloadLen, + // ", invalidLen=%0d", invalidLen, + // ", totalLen=%0d", totalLen, + // ", pmtuResidue=%0d", pmtuResidue, + // ", tmpFirstPktLen=%0d", tmpFirstPktLen, + // ", totalPktNum=%0d", totalPktNum, + // ", pmtuLen=%0d", pmtuLen, + // ", pmtu=", fshow(pmtu), + // ", isOnlyPkt=", fshow(isOnlyPkt) + // ); + endrule + + rule expectedRespGen if (busyReg); + let req = reqQ.first; + + isFirstReg <= False; + pktNumReg <= pktNumReg - 1; + // if (!isFirstReg) begin + nextAddrReg <= addrAddPsnMultiplyPMTU(nextAddrReg, oneAsPSN, pmtuReg); + // end + + // let isZeroResidue = isZero(residueReg); + // let isFirst = nextAddrReg == req.startAddr; + let isLast = isLessOrEqOne(pktNumReg); + if (isLast) begin + reqQ.deq; + totalLenQ.enq(req.len); + busyReg <= False; + end + + let chunkLen = isFirstReg ? firstPktLenReg : (isLast ? lastPktLenReg : pmtuLenReg); + let curPktLastFragValidByteNum = calcLastFragValidByteNum(chunkLen); + + let addrChunkResp = AddrChunkResp { + chunkAddr : isFirstReg ? req.startAddr : nextAddrReg, + chunkLen : chunkLen, + isFirst : isFirstReg, + isLast : isLast, + isOrigFirst: req.isFirst, + isOrigLast : req.isLast + }; + respQ.enq(addrChunkResp); + + let sgePktMetaDataRef = PktMetaDataSGE { + curPktLastFragValidByteNum: curPktLastFragValidByteNum, + pktLen : chunkLen, + pmtu : pmtuReg, + isFirst : isFirstReg, + isLast : isLast, + isFirstSGE : req.isFirst, + isLastSGE : req.isLast + }; + sgePktMetaDataRefQ.enq(sgePktMetaDataRef); + endrule + + rule checkResp; + let addrChunkResp <- dut.srvPort.response.get; + let expectedResp = respQ.first; + respQ.deq; + + let expectedTotalLen = totalLenReg + zeroExtend(addrChunkResp.chunkLen); + if (addrChunkResp.isFirst) begin + expectedTotalLen = zeroExtend(addrChunkResp.chunkLen); + end + totalLenReg <= expectedTotalLen; + + immAssert( + addrChunkResp.chunkAddr == expectedResp.chunkAddr && + addrChunkResp.chunkLen == expectedResp.chunkLen && + addrChunkResp.isFirst == expectedResp.isFirst && + addrChunkResp.isLast == expectedResp.isLast, + "expected addrChunkResp assertion @ mkTestAddrChunkSrv", + $format( + "addrChunkResp=", fshow(addrChunkResp), + " should match expectedResp=", fshow(expectedResp) + ) + ); + + if (addrChunkResp.isLast) begin + let totalLen = totalLenQ.first; + totalLenQ.deq; + + immAssert( + totalLen == expectedTotalLen, + "totalLen assertion @ mkTestAddrChunkSrv", + $format( + "totalLen=%0d should == expectedTotalLen=%0d", + totalLen, expectedTotalLen + ) + ); + end + // $display("time=%0t: checkResp", $time); + endrule + + rule checkPktMetaDataSGE; + let sgePktMetaData = dut.sgePktMetaDataPipeOut.first; + dut.sgePktMetaDataPipeOut.deq; + let sgePktMetaDataRef = sgePktMetaDataRefQ.first; + sgePktMetaDataRefQ.deq; + + immAssert( + sgePktMetaData.curPktLastFragValidByteNum == sgePktMetaDataRef.curPktLastFragValidByteNum && + sgePktMetaData.pktLen == sgePktMetaDataRef.pktLen && + sgePktMetaData.pmtu == sgePktMetaDataRef.pmtu && + sgePktMetaData.isFirst == sgePktMetaDataRef.isFirst && + sgePktMetaData.isLast == sgePktMetaDataRef.isLast && + sgePktMetaData.isFirstSGE == sgePktMetaDataRef.isFirstSGE && + sgePktMetaData.isLastSGE == sgePktMetaDataRef.isLastSGE, + "sgePktMetaData assertion @ mkTestAddrChunkSrv", + $format( + "sgePktMetaData.curPktLastFragValidByteNum=%h should == sgePktMetaDataRef.curPktLastFragValidByteNum=%h", + sgePktMetaData.curPktLastFragValidByteNum, sgePktMetaDataRef.curPktLastFragValidByteNum, + "sgePktMetaData.pktLen=%0d should == sgePktMetaDataRef.pktLen=%0d", + sgePktMetaData.pktLen, sgePktMetaDataRef.pktLen, + "sgePktMetaData.pmtu=", fshow(sgePktMetaData.pmtu), + " should == sgePktMetaData.pmtu=", fshow(sgePktMetaData.pmtu), + "sgePktMetaData.isFirst=", fshow(sgePktMetaData.isFirst), + " should == sgePktMetaData.isFirst=", fshow(sgePktMetaData.isFirst), + "sgePktMetaData.isLast=", fshow(sgePktMetaData.isLast), + " should == sgePktMetaData.isLast=", fshow(sgePktMetaData.isLast), + "sgePktMetaData.isFirstSGE=", fshow(sgePktMetaData.isFirstSGE), + " should == sgePktMetaData.isFirstSGE=", fshow(sgePktMetaData.isFirstSGE), + "sgePktMetaData.isLastSGE=", fshow(sgePktMetaData.isLastSGE), + " should == sgePktMetaData.isLastSGE=", fshow(sgePktMetaData.isLastSGE) + ) + ); + // $display("time=%0t: checkPktMetaDataSGE", $time); + endrule +endmodule + +(* doc = "testcase" *) +module mkTestDmaReadCntrlNormalCase(Empty); + let normalOrCancelCase = True; + let result <- mkTestDmaReadCntrlNormalOrCancelCase(normalOrCancelCase); +endmodule + +(* doc = "testcase" *) +module mkTestDmaReadCntrlCancelCase(Empty); + let normalOrCancelCase = False; + let result <- mkTestDmaReadCntrlNormalOrCancelCase(normalOrCancelCase); +endmodule + +typedef enum { + TEST_DMA_CNTRL_INIT, + TEST_DMA_CNTRL_RUN, + TEST_DMA_CNTRL_RESET +} TestDmaCntrlState deriving(Bits, Eq, FShow); + +module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); + let minPayloadLen = 2048; + let maxPayloadLen = 8192; + let qpType = IBV_QPT_XRC_SEND; + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + + Reg#(Bool) clearReg <- mkReg(True); + Reg#(Bool) canceledReg <- mkReg(False); + Reg#(TotalFragNum) remainingFragNumReg <- mkReg(0); + FIFOF#(TotalFragNum) totalFragNumQ <- mkFIFOF; + + PipeOut#(Bool) randomCancelPipeOut <- mkGenericRandomPipeOut; + Reg#(Bool) isFinalRespLastFragReg <- mkReg(False); + + PipeOut#(ADDR) startAddrPipeOut <- mkGenericRandomPipeOut; + let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); + let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); + + let simDmaReadSrv <- mkSimDmaReadSrv; + let dut <- mkDmaReadCntrl(clearReg, simDmaReadSrv); + + Reg#(TestDmaCntrlState) stateReg <- mkReg(TEST_DMA_CNTRL_INIT); + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + mkSink(dut.sgePktMetaDataPipeOut); + mkSink(dut.totalPayloadMetaDataPipeOut); + mkSink(dut.sgeMergedMetaDataPipeOut); + + rule clearAll if (stateReg == TEST_DMA_CNTRL_INIT); + clearReg <= False; + canceledReg <= False; + remainingFragNumReg <= 0; + isFinalRespLastFragReg <= False; + totalFragNumQ.clear; + + stateReg <= TEST_DMA_CNTRL_RUN; + // $display("time=%0t: clearAll", $time); + endrule + + rule issueReq if (stateReg == TEST_DMA_CNTRL_RUN); + let startAddr = startAddrPipeOut.first; + startAddrPipeOut.deq; + + let payloadLen = payloadLenPipeOut.first; + payloadLenPipeOut.deq; + + let pmtu = pmtuPipeOut.first; + pmtuPipeOut.deq; +/* + let len = payloadLen; + let oneAsPSN = 1; + let pmtuAlignedStartAddr = alignAddrByPMTU(startAddr, pmtu); + let secondChunkStartAddr = addrAddPsnMultiplyPMTU(pmtuAlignedStartAddr, oneAsPSN, pmtu); + let pmtuLen = calcPmtuLen(pmtu); + + Tuple4#(PktLen, PktNum, PktLen, PktLen) tmpTuple = case (pmtu) + IBV_MTU_256 : begin + Bit#(8) addrLowPart = truncate(startAddr); // [7 : 0] + Bit#(8) lenLowPart = truncate(len); + Bit#(8) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 8)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + IBV_MTU_512 : begin + Bit#(9) addrLowPart = truncate(startAddr); // [8 : 0] + Bit#(9) lenLowPart = truncate(len); + Bit#(9) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 9)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + IBV_MTU_1024: begin + Bit#(10) addrLowPart = truncate(startAddr); // [9 : 0] + Bit#(10) lenLowPart = truncate(len); + Bit#(10) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 10)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + IBV_MTU_2048: begin + Bit#(11) addrLowPart = truncate(startAddr); // [10 : 0] + Bit#(11) lenLowPart = truncate(len); + Bit#(11) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 11)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + IBV_MTU_4096: begin + Bit#(12) addrLowPart = truncate(startAddr); // [11 : 0] + Bit#(12) lenLowPart = truncate(len); + Bit#(12) pmtuMask = maxBound; + Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 12)) truncatedLen = truncateLSB(len); + tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); + end + endcase; + + let { pmtuMask, truncatedPktNum, addrLowPart, lenLowPart } = tmpTuple; + let maxFirstPktLen = pmtuLen - addrLowPart; + let tmpSum = addrLowPart + lenLowPart; + ResiduePMTU residue = truncateByPMTU(tmpSum, pmtu); + PktLen tmpLastPktLen = zeroExtend(residue); + + let pmtuInvMask = ~pmtuMask; + let residuePktNum = |(pmtuMask & tmpSum); + let extraPktNum = |(pmtuInvMask & tmpSum); + Bool hasResidue = unpack(residuePktNum); + Bool hasExtraPkt = unpack(extraPktNum); + let notFullPkt = isZeroR(truncatedPktNum); + + let sgePktNum = truncatedPktNum + zeroExtend(residuePktNum) + zeroExtend(extraPktNum); + // let firstPktLen = notFullPkt ? (hasExtraPkt ? maxFirstPktLen : lenLowPart) : maxFirstPktLen; + let firstPktLen = (notFullPkt && !hasExtraPkt) ? lenLowPart : maxFirstPktLen; + // let lastPktLen = notFullPkt ? (hasResidue ? tmpLastPktLen : 0) : (hasResidue ? tmpLastPktLen : pmtuLen); + let lastPktLen = notFullPkt ? (hasExtraPkt ? tmpLastPktLen : lenLowPart) : (hasResidue ? tmpLastPktLen : pmtuLen); + // let isSinglePkt = isLessOrEqOneR(totalPktNum); +*/ + let { + pmtuLen, firstPktLen, lastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU(startAddr, payloadLen, pmtu); + let { + firstPktFragNum, firstPktByteEn, firstPktFragValidByteNum + } = calcTotalFragNumByLength(zeroExtend(firstPktLen)); + let midPktLen = payloadLen - zeroExtend(firstPktLen); + if (isLessOrEqOne(sgePktNum)) begin + lastPktLen = 0; + end + else begin + midPktLen = payloadLen - zeroExtend(firstPktLen) - zeroExtend(lastPktLen); + end + let { + lastPktFragNum, lastPktByteEn, lastPktFragValidByteNum + } = calcTotalFragNumByLength(zeroExtend(lastPktLen)); + let { + midPktFragNum, midPktByteEn, midPktFragValidByteNum + } = calcTotalFragNumByLength(midPktLen); + let totalFragNum = firstPktFragNum + midPktFragNum + lastPktFragNum; + totalFragNumQ.enq(totalFragNum); + + let sge = ScatterGatherElem { + laddr : startAddr, + len : payloadLen, + lkey : dontCareValue, + isFirst: True, + isLast : True + }; + let dummySGE = sge; + // dummySGE.isFirst = False; + // dummySGE.isLast = False; + + let sgl = vec(sge, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE); + + let dmaReadCntrlReq = DmaReadCntrlReq { + pmtu : pmtu, + sglDmaReadMetaData: DmaReadMetaDataSGL { + sgl : sgl, + sqpn : getDefaultQPN, + wrID : dontCareValue + // initiator: DMA_SRC_RQ_RD, + // startAddr: startAddr, + // len : payloadLen, + } + }; + + dut.srvPort.request.put(dmaReadCntrlReq); + countDown.decr; + // $display( + // "time=%0t: issueReq", $time, + // ", startAddr=%h", startAddr, + // ", secondChunkStartAddr=%h", secondChunkStartAddr, + // ", payloadLen=%0d", payloadLen, + // ", pmtu=", fshow(pmtu), + // ", sgePktNum=%0d", sgePktNum, + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen, + // ", pmtuLen=%0d", pmtuLen, + // ", totalFragNum=%0d", totalFragNum, + // ", firstPktFragNum=%0d", firstPktFragNum, + // ", lastPktFragNum=%0d", lastPktFragNum, + // ", midPktLen=%0d", midPktLen + // // ", pmtuMask=%h", pmtuMask, + // // ", truncatedPktNum=%0d", truncatedPktNum, + // // ", addrLowPart=%0d", addrLowPart, + // // ", lenLowPart=%0d", lenLowPart, + // // ", tmpLastPktLen=%0d", tmpLastPktLen, + // // ", hasResidue=", fshow(hasResidue), + // // ", hasExtraPkt=", fshow(hasExtraPkt), + // // ", notFullPkt=", fshow(notFullPkt) + // ); + endrule + + rule checkResp if (stateReg == TEST_DMA_CNTRL_RUN && !dut.dmaCntrl.isIdle); + let dmaReadCntrlResp <- dut.srvPort.response.get; + // let isOrigFirst = dmaReadCntrlResp.isOrigFirst; + // let isOrigLast = dmaReadCntrlResp.isOrigLast; + isFinalRespLastFragReg <= dmaReadCntrlResp.dmaReadResp.dataStream.isLast; + + let isFirstFrag = dmaReadCntrlResp.isOrigFirst && dmaReadCntrlResp.dmaReadResp.dataStream.isFirst; + let isLastFrag = dmaReadCntrlResp.isOrigLast && dmaReadCntrlResp.dmaReadResp.dataStream.isLast; + + let totalFragNum = totalFragNumQ.first; + let remainingFragNum = remainingFragNumReg; + if (!isLessOrEqOne(totalFragNum)) begin + if (isFirstFrag) begin + remainingFragNum = totalFragNum - 2; + end + else begin + remainingFragNum = remainingFragNumReg - 1; + end + end + remainingFragNumReg <= remainingFragNum; + + if (isLastFrag) begin + totalFragNumQ.deq; + + immAssert( + isZero(remainingFragNumReg), + "remaining fragNum assertion @ mkTestDmaReadCntrlNormalOrCancelCase", + $format( + "remainingFragNumReg=%0d should be zero when", remainingFragNumReg, + " isOrigFirst=", fshow(isFirstFrag), + " and isLastFrag=", fshow(isLastFrag) + ) + ); + end + + if (!normalOrCancelCase) begin + let shouldCancel = randomCancelPipeOut.first; + randomCancelPipeOut.deq; + + if (shouldCancel && !canceledReg) begin + dut.dmaCntrl.cancel; + canceledReg <= True; + // $display("time=%0t: dmaCntrl.cancel", $time); + end + end + // $display( + // "time=%0t: checkResp", $time, + // ", remainingFragNumReg=%0d", remainingFragNumReg, + // ", isFirst=", fshow(dmaReadCntrlResp.dmaReadResp.dataStream.isFirst), + // ", isLast=", fshow(dmaReadCntrlResp.dmaReadResp.dataStream.isLast), + // ", isOrigFirst=", fshow(dmaReadCntrlResp.isOrigFirst), + // ", isOrigLast=", fshow(dmaReadCntrlResp.isOrigLast) + // ); + endrule + + rule waitUntilGracefulStop if (stateReg == TEST_DMA_CNTRL_RUN && dut.dmaCntrl.isIdle); + immAssert( + isFinalRespLastFragReg, + "isFinalRespLastFragReg assertion @ mkTestDmaReadCntrlNormalOrCancelCase", + $format( + "isFinalRespLastFragReg=", fshow(isFinalRespLastFragReg), + " should be true, when dut.dmaCntrl.isIdle=", fshow(dut.dmaCntrl.isIdle) + ) + ); + + stateReg <= TEST_DMA_CNTRL_RESET; + // $display("time=%0t: waitUntilGracefulStop, dmaCntrl.isIdle", $time); + endrule + + rule loop if (stateReg == TEST_DMA_CNTRL_RESET); + clearReg <= True; + stateReg <= TEST_DMA_CNTRL_INIT; + endrule +endmodule + +(* doc = "testcase" *) +module mkTestMergeNormalPayloadEachSGE(Empty); + let minPayloadLen = 2048; + let maxPayloadLen = 8192; + let result <- mkTestMergeNormalOrSmallPayloadEachSGE(minPayloadLen, maxPayloadLen); +endmodule + +(* doc = "testcase" *) +module mkTestMergeSmallPayloadEachSGE(Empty); + let minPayloadLen = 1; + let maxPayloadLen = 7; + let result <- mkTestMergeNormalOrSmallPayloadEachSGE(minPayloadLen, maxPayloadLen); +endmodule + +typedef enum { + TEST_MERGE_EACH_SGE_INIT, + TEST_MERGE_EACH_SGE_PREPARE, + TEST_MERGE_EACH_SGE_ISSUE, + TEST_MERGE_EACH_SGE_RUN +} TestMergePayloadStateSGE deriving(Bits, Eq, FShow); + +module mkTestMergeNormalOrSmallPayloadEachSGE#( + // Both min and max are inclusive + Length minPayloadLen, Length maxPayloadLen +)(Empty); + let qpType = IBV_QPT_XRC_SEND; + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + + Reg#(Bool) clearReg <- mkReg(True); + Reg#(TotalFragNum) remainingFragNumReg <- mkReg(0); + Reg#(IdxSGL) sglIdxReg <- mkReg(0); + Reg#(Length) totalLenReg <- mkRegU; + + Vector#(MAX_SGE, Reg#(ScatterGatherElem)) sglRegVec <- replicateM(mkRegU); + FIFOF#(DataStream) sgePayloadOutQ <- mkFIFOF; + FIFOF#(Tuple2#(TotalFragNum, ByteEn)) sgeRefQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + + PipeOut#(ADDR) startAddrPipeOut <- mkGenericRandomPipeOut; + let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); + let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); + + let simDmaReadSrv <- mkSimDmaReadSrv; + let dmaReadCntrl <- mkDmaReadCntrl(clearReg, simDmaReadSrv); + + let dut <- mkMergePayloadEachSGE( + clearReg, dmaReadCntrl.sgePktMetaDataPipeOut, toPipeOut(sgePayloadOutQ) + ); + + Reg#(TestMergePayloadStateSGE) stateReg <- mkReg(TEST_MERGE_EACH_SGE_INIT); + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + // mkSink(dut.sgePktMetaDataPipeOut); + mkSink(dmaReadCntrl.totalPayloadMetaDataPipeOut); + mkSink(dmaReadCntrl.sgeMergedMetaDataPipeOut); + + rule clearAll if (stateReg == TEST_MERGE_EACH_SGE_INIT); + clearReg <= False; + + stateReg <= TEST_MERGE_EACH_SGE_PREPARE; + // $display("time=%0t: clearAll", $time); + endrule + + rule genSGE if (stateReg == TEST_MERGE_EACH_SGE_PREPARE); + let startAddr = startAddrPipeOut.first; + startAddrPipeOut.deq; + + let payloadLen = payloadLenPipeOut.first; + payloadLenPipeOut.deq; + + let isFirst = isZero(sglIdxReg); + let isLast = isAllOnesR(sglIdxReg); + let sge = ScatterGatherElem { + laddr : startAddr, + len : payloadLen, + lkey : dontCareValue, + isFirst: isFirst, + isLast : isLast + }; + sglRegVec[sglIdxReg] <= sge; + + let totalLen = totalLenReg; + if (isFirst) begin + totalLen = payloadLen; + end + else begin + totalLen = totalLenReg + payloadLen; + end + totalLenReg <= totalLen; + + if (isLast) begin + sglIdxReg <= 0; + stateReg <= TEST_MERGE_EACH_SGE_ISSUE; + end + else begin + sglIdxReg <= sglIdxReg + 1; + end + + let { + sgeFragNum, lastFragByteEn, lastFragValidByteNum + } = calcTotalFragNumByLength(payloadLen); + sgeRefQ.enq(tuple2(sgeFragNum, lastFragByteEn)); + + countDown.decr; + // $display( + // "time=%0t: genSGE", $time, + // ", sglIdxReg=%0d", sglIdxReg, + // ", startAddr=%h", startAddr, + // ", payloadLen=%0d", payloadLen, + // ", totalLen=%0d", totalLen, + // ", sgeFragNum=%0d", sgeFragNum, + // ", lastFragValidByteNum=%0d", lastFragValidByteNum + // ); + endrule + + rule issueDmaReadCntrlReq if (stateReg == TEST_MERGE_EACH_SGE_ISSUE); + let sgl = readVReg(sglRegVec); + let pmtu = pmtuPipeOut.first; + pmtuPipeOut.deq; + + let dmaReadCntrlReq = DmaReadCntrlReq { + pmtu : pmtu, + sglDmaReadMetaData: DmaReadMetaDataSGL { + sgl : sgl, + sqpn : getDefaultQPN, + wrID : dontCareValue + // initiator: DMA_SRC_RQ_RD, + // startAddr: startAddr, + // len : payloadLen, + } + }; + + dmaReadCntrl.srvPort.request.put(dmaReadCntrlReq); + stateReg <= TEST_MERGE_EACH_SGE_RUN; + // $display( + // "time=%0t: issueDmaReadCntrlReq", $time, + // ", pmtu=", fshow(pmtu) + // ); + // for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin + // let sge = sglRegVec[idx]; + // let { + // pmtuLen, firstPktLen, lastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + // } = calcPktNumAndPktLenByAddrAndPMTU(sge.laddr, sge.len, pmtu); + // let firstPktFragNum = calcFragNumByPktLen(firstPktLen); + // let lastPktFragNum = calcFragNumByPktLen(lastPktLen); + + // $display( + // "time=%0t: issueDmaReadCntrlReq", $time, + // ", SGE idx=%0d", idx, + // ", sge.laddr=%h", sge.laddr, + // ", sgePktNum=%0d", sgePktNum, + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen, + // ", pmtuLen=%0d", pmtuLen, + // ", firstPktFragNum=%0d", firstPktFragNum, + // ", lastPktFragNum=%0d", lastPktFragNum + // ); + // end + endrule + + rule recvDmaReadCntrlResp; + let dmaReadCntrlResp <- dmaReadCntrl.srvPort.response.get; + sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); + endrule + + rule checkMergedPayloadSGE if (stateReg == TEST_MERGE_EACH_SGE_RUN); + let { sgeFragNum, lastFragByteEn } = sgeRefQ.first; + // let { + // sgeFragNum, lastFragByteEn, lastFragValidByteNum + // } = calcTotalFragNumByLength(refSGE.len); + + let payloadFrag = dut.first; + dut.deq; + if (!payloadFrag.isLast) begin + if (payloadFrag.isFirst) begin + remainingFragNumReg <= sgeFragNum - 2; + end + else begin + remainingFragNumReg <= remainingFragNumReg - 1; + end + end + + if (payloadFrag.isFirst) begin + immAssert( + isZero(remainingFragNumReg), + "remainingFragNumReg assertion @ mkTestMergeNormalOrSmallPayloadEachSGE", + $format( + "remainingFragNumReg=%0d", remainingFragNumReg, + " should be zero when payloadFrag.isFirst=", + fshow(payloadFrag.isFirst) + ) + ); + end + + if (payloadFrag.isLast) begin + sgeRefQ.deq; + + if (isAllOnesR(sglIdxReg)) begin + sglIdxReg <= 0; + stateReg <= TEST_MERGE_EACH_SGE_INIT; + end + else begin + sglIdxReg <= sglIdxReg + 1; + end + + immAssert( + isZero(remainingFragNumReg), + "remainingFragNumReg assertion @ mkTestMergeNormalOrSmallPayloadEachSGE", + $format( + "remainingFragNumReg=%0d", remainingFragNumReg, + " should be zero when payloadFrag.isLast=", + fshow(payloadFrag.isLast) + ) + ); + immAssert( + payloadFrag.byteEn == lastFragByteEn, + "lastFragByteEn assertion @ mkTestMergeNormalOrSmallPayloadEachSGE", + $format( + "payloadFrag.byteEn=%h", payloadFrag.byteEn, + " should == lastFragByteEn=%h", lastFragByteEn, + ", when payloadFrag.isLast=", fshow(payloadFrag.isLast) + ) + ); + end + else begin + immAssert( + isAllOnesR(payloadFrag.byteEn), + "payloadFrag.byteEn assertion @ mkTestMergeNormalOrSmallPayloadEachSGE", + $format( + "payloadFrag.byteEn=%h", payloadFrag.byteEn, + " should be all ones when payloadFrag.isLast=", + fshow(payloadFrag.isLast) + ) + ); + end + // $display( + // "time=%0t: checkMergedPayloadSGE", $time, + // ", refSGE.len=%0d", refSGE.len, + // ", sgeFragNum=%0d", sgeFragNum, + // ", lastFragValidByteNum=%0d", lastFragValidByteNum, + // ", lastFragByteEn=%h", lastFragByteEn, + // ", remainingFragNumReg=%0d", remainingFragNumReg, + // ", payloadFrag.isFirst=", fshow(payloadFrag.isFirst), + // ", payloadFrag.isLast=", fshow(payloadFrag.isLast), + // ", payloadFrag.byteEn=%h", payloadFrag.byteEn + // ); + endrule +endmodule + +(* doc = "testcase" *) +module mkTestMergeNormalPayloadAllSGE(Empty); + let minPayloadLen = 2048; + let maxPayloadLen = 8192; + let result <- mkTestMergeNormalOrSmallPayloadAllSGE(minPayloadLen, maxPayloadLen); +endmodule + +(* doc = "testcase" *) +module mkTestMergeSmallPayloadAllSGE(Empty); + let minPayloadLen = 1; + let maxPayloadLen = 7; + let result <- mkTestMergeNormalOrSmallPayloadAllSGE(minPayloadLen, maxPayloadLen); +endmodule + +typedef enum { + TEST_MERGE_ALL_SGE_INIT, + TEST_MERGE_ALL_SGE_PREPARE, + TEST_MERGE_ALL_SGE_ISSUE, + TEST_MERGE_ALL_SGE_RUN +} TestMergePayloadStateSGL deriving(Bits, Eq, FShow); + +module mkTestMergeNormalOrSmallPayloadAllSGE#( + // Both min and max are inclusive + Length minPayloadLen, Length maxPayloadLen +)(Empty); + let qpType = IBV_QPT_XRC_SEND; + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + + Reg#(Bool) clearReg <- mkReg(True); + Reg#(TotalFragNum) remainingFragNumReg <- mkReg(0); + Reg#(IdxSGL) sglIdxReg <- mkReg(0); + Reg#(Length) totalLenReg <- mkRegU; + + Vector#(MAX_SGE, Reg#(ScatterGatherElem)) sglRegVec <- replicateM(mkRegU); + FIFOF#(DataStream) sgePayloadOutQ <- mkFIFOF; + FIFOF#(Tuple2#(TotalFragNum, ByteEn)) sglRefQ <- mkFIFOF; + + PipeOut#(ADDR) startAddrPipeOut <- mkGenericRandomPipeOut; + let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); + let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); + + let simDmaReadSrv <- mkSimDmaReadSrv; + let dmaReadCntrl <- mkDmaReadCntrl(clearReg, simDmaReadSrv); + + let sgeMergedPayloadPipeOut <- mkMergePayloadEachSGE( + clearReg, dmaReadCntrl.sgePktMetaDataPipeOut, toPipeOut(sgePayloadOutQ) + ); + + let dut <- mkMergePayloadAllSGE( + clearReg, dmaReadCntrl.sgeMergedMetaDataPipeOut, sgeMergedPayloadPipeOut + ); + + Reg#(TestMergePayloadStateSGL) stateReg <- mkReg(TEST_MERGE_ALL_SGE_INIT); + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + // mkSink(dmaReadCntrl.sgePktMetaDataPipeOut); + // mkSink(dmaReadCntrl.sgeMergedMetaDataPipeOut); + mkSink(dmaReadCntrl.totalPayloadMetaDataPipeOut); + + rule clearAll if (stateReg == TEST_MERGE_ALL_SGE_INIT); + clearReg <= False; + + stateReg <= TEST_MERGE_ALL_SGE_PREPARE; + // $display("time=%0t: clearAll", $time); + endrule + + rule genSGE if (stateReg == TEST_MERGE_ALL_SGE_PREPARE); + let startAddr = startAddrPipeOut.first; + startAddrPipeOut.deq; + + let payloadLen = payloadLenPipeOut.first; + payloadLenPipeOut.deq; + + let isFirst = isZero(sglIdxReg); + let isLast = isAllOnesR(sglIdxReg); + let sge = ScatterGatherElem { + laddr : startAddr, + len : payloadLen, + lkey : dontCareValue, + isFirst: isFirst, + isLast : isLast + }; + sglRegVec[sglIdxReg] <= sge; + + let totalLen = totalLenReg; + if (isFirst) begin + totalLen = payloadLen; + end + else begin + totalLen = totalLenReg + payloadLen; + end + totalLenReg <= totalLen; + + if (isLast) begin + sglIdxReg <= 0; + stateReg <= TEST_MERGE_ALL_SGE_ISSUE; + end + else begin + sglIdxReg <= sglIdxReg + 1; + end + + let { + sgeFragNum, sgeLastFragByteEn, sgeLastFragValidByteNum + } = calcTotalFragNumByLength(payloadLen); + countDown.decr; + // $display( + // "time=%0t: genSGE", $time, + // ", sglIdxReg=%0d", sglIdxReg, + // ", startAddr=%h", startAddr, + // ", payloadLen=%0d", payloadLen, + // ", sgeFragNum=%0d", sgeFragNum, + // ", sgeLastFragByteEn=%h", sgeLastFragByteEn, + // ", sgeLastFragValidByteNum=%0d", sgeLastFragValidByteNum + // ); + endrule + + rule issueDmaReadCntrlReq if (stateReg == TEST_MERGE_ALL_SGE_ISSUE); + let sgl = readVReg(sglRegVec); + let pmtu = pmtuPipeOut.first; + pmtuPipeOut.deq; + + let dmaReadCntrlReq = DmaReadCntrlReq { + pmtu : pmtu, + sglDmaReadMetaData: DmaReadMetaDataSGL { + sgl : sgl, + sqpn : getDefaultQPN, + wrID : dontCareValue + // initiator: DMA_SRC_RQ_RD, + // startAddr: startAddr, + // len : payloadLen, + } + }; + + dmaReadCntrl.srvPort.request.put(dmaReadCntrlReq); + + stateReg <= TEST_MERGE_ALL_SGE_RUN; + let { + sglFragNum, sglLastFragByteEn, sglLastFragValidByteNum + } = calcTotalFragNumByLength(totalLenReg); + sglRefQ.enq(tuple2(sglFragNum, sglLastFragByteEn)); + // $display( + // "time=%0t: issueDmaReadCntrlReq", $time, + // ", pmtu=", fshow(pmtu), + // ", totalLenReg=%0d", totalLenReg, + // ", sglFragNum=%0d", sglFragNum, + // ", sglLastFragByteEn=%h", sglLastFragByteEn, + // ", sglLastFragValidByteNum=%0d", sglLastFragValidByteNum + // ); + // for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin + // let sge = sglRegVec[idx]; + // let { + // pmtuLen, sgeFirstPktLen, sgeLastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + // } = calcPktNumAndPktLenByAddrAndPMTU(sge.laddr, sge.len, pmtu); + // let sgeFirstPktFragNum = calcFragNumByPktLen(sgeFirstPktLen); + // let sgeLastPktFragNum = calcFragNumByPktLen(sgeLastPktLen); + + // $display( + // "time=%0t: issueDmaReadCntrlReq", $time, + // ", SGE idx=%0d", idx, + // ", sge.laddr=%h", sge.laddr, + // ", sgePktNum=%0d", sgePktNum, + // ", sgeFirstPktLen=%0d", sgeFirstPktLen, + // ", sgeLastPktLen=%0d", sgeLastPktLen, + // ", pmtuLen=%0d", pmtuLen, + // ", sgeFirstPktFragNum=%0d", sgeFirstPktFragNum, + // ", sgeLastPktFragNum=%0d", sgeLastPktFragNum + // ); + // end + endrule + + rule recvDmaReadCntrlResp; + let dmaReadCntrlResp <- dmaReadCntrl.srvPort.response.get; + sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); + $display( + "time=%0t: recvDmaReadCntrlResp", $time, + ", dmaReadCntrlResp=", fshow(dmaReadCntrlResp) + ); + endrule + + rule checkMergedPayloadSGL if (stateReg == TEST_MERGE_ALL_SGE_RUN); + let { sglFragNum, sglLastFragByteEn } = sglRefQ.first; + + let payloadFrag = dut.first; + dut.deq; + if (!payloadFrag.isLast) begin + if (payloadFrag.isFirst) begin + remainingFragNumReg <= sglFragNum - 2; + end + else begin + remainingFragNumReg <= remainingFragNumReg - 1; + end + end + + if (payloadFrag.isFirst) begin + immAssert( + isZero(remainingFragNumReg), + "remainingFragNumReg assertion @ mkTestMergeNormalOrSmallPayloadAllSGE", + $format( + "remainingFragNumReg=%0d", remainingFragNumReg, + " should be zero when payloadFrag.isFirst=", + fshow(payloadFrag.isFirst) + ) + ); + end + + if (payloadFrag.isLast) begin + sglRefQ.deq; + stateReg <= TEST_MERGE_ALL_SGE_INIT; + + immAssert( + isZero(remainingFragNumReg), + "remainingFragNumReg assertion @ mkTestMergeNormalOrSmallPayloadAllSGE", + $format( + "remainingFragNumReg=%0d", remainingFragNumReg, + " should be zero when payloadFrag.isLast=", + fshow(payloadFrag.isLast) + ) + ); + immAssert( + payloadFrag.byteEn == sglLastFragByteEn, + "sglLastFragByteEn assertion @ mkTestMergeNormalOrSmallPayloadAllSGE", + $format( + "payloadFrag.byteEn=%h", payloadFrag.byteEn, + " should == sglLastFragByteEn=%h", sglLastFragByteEn, + ", when payloadFrag.isLast=", fshow(payloadFrag.isLast) + ) + ); + end + else begin + immAssert( + isAllOnesR(payloadFrag.byteEn), + "payloadFrag.byteEn assertion @ mkTestMergeNormalOrSmallPayloadAllSGE", + $format( + "payloadFrag.byteEn=%h", payloadFrag.byteEn, + " should be all ones when payloadFrag.isLast=", + fshow(payloadFrag.isLast) + ) + ); + end + // $display( + // "time=%0t: checkMergedPayloadSGL", $time, + // ", sglFragNum=%0d", sglFragNum, + // // ", lastFragValidByteNum=%0d", lastFragValidByteNum, + // ", sglLastFragByteEn=%h", sglLastFragByteEn, + // ", remainingFragNumReg=%0d", remainingFragNumReg, + // ", payloadFrag.isFirst=", fshow(payloadFrag.isFirst), + // ", payloadFrag.isLast=", fshow(payloadFrag.isLast), + // ", payloadFrag.byteEn=%h", payloadFrag.byteEn + // ); + endrule +endmodule + +(* doc = "testcase" *) +module mkTestAdjustNormalPayloadSegmentCase(Empty); + let minPayloadLen = 2048; + let maxPayloadLen = 8192; + let result <- mkTestAdjustNormalOrSmallPayloadSegment(minPayloadLen, maxPayloadLen); +endmodule + +(* doc = "testcase" *) +module mkTestAdjustSmallPayloadSegmentCase(Empty); + let minPayloadLen = 1; + let maxPayloadLen = 7; + let result <- mkTestAdjustNormalOrSmallPayloadSegment(minPayloadLen, maxPayloadLen); +endmodule + +typedef enum { + TEST_ADJUST_PAYLOAD_INIT, + TEST_ADJUST_PAYLOAD_PREPARE, + TEST_ADJUST_PAYLOAD_ISSUE, + TEST_ADJUST_PAYLOAD_RECV, + TEST_ADJUST_PAYLOAD_RUN +} TestAdjustPayloadState deriving(Bits, Eq, FShow); + +module mkTestAdjustNormalOrSmallPayloadSegment#( + // Both min and max are inclusive + Length minPayloadLen, Length maxPayloadLen +)(Empty); + let qpType = IBV_QPT_XRC_SEND; + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + + Reg#(Bool) clearReg <- mkReg(True); + Reg#(Bool) isFirstPktReg <- mkRegU; + Reg#(PktNum) remainingPktNumReg <- mkRegU; + Reg#(PktFragNum) remainingFragNumReg <- mkReg(0); + Reg#(IdxSGL) sglIdxReg <- mkReg(0); + + Vector#(MAX_SGE, Reg#(ScatterGatherElem)) sglRegVec <- replicateM(mkRegU); + FIFOF#(DataStream) sgePayloadOutQ <- mkFIFOF; + FIFOF#(PMTU) pmtuQ <- mkFIFOF; + FIFOF#(AdjustedTotalPayloadMetaData) adjustedTotalPayloadMetaDataQ <- mkFIFOF; + FIFOF#(Tuple3#(AdjustedTotalPayloadMetaData, PktFragNum, ByteEnBitNum)) adjustedTotalPayloadMetaDataRefQ <- mkFIFOF; + + PipeOut#(ADDR) remoteAddrPipeOut <- mkGenericRandomPipeOut; + PipeOut#(ADDR) localAddrPipeOut <- mkGenericRandomPipeOut; + let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); + let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); + + let simDmaReadSrv <- mkSimDmaReadSrv; + let dmaReadCntrl <- mkDmaReadCntrl(clearReg, simDmaReadSrv); + + let sgeMergedPayloadPipeOut <- mkMergePayloadEachSGE( + clearReg, dmaReadCntrl.sgePktMetaDataPipeOut, toPipeOut(sgePayloadOutQ) + ); + let sglMergedPayloadPipeOut <- mkMergePayloadAllSGE( + clearReg, dmaReadCntrl.sgeMergedMetaDataPipeOut, sgeMergedPayloadPipeOut + ); + let dut <- mkAdjustPayloadSegment( + clearReg, toPipeOut(adjustedTotalPayloadMetaDataQ), sglMergedPayloadPipeOut + ); + + Reg#(TestAdjustPayloadState) stateReg <- mkReg(TEST_ADJUST_PAYLOAD_INIT); + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + // mkSink(dmaReadCntrl.sgePktMetaDataPipeOut); + // mkSink(dmaReadCntrl.sgeMergedMetaDataPipeOut); + // mkSink(dmaReadCntrl.totalPayloadMetaDataPipeOut); + + rule clearAll if (stateReg == TEST_ADJUST_PAYLOAD_INIT); + clearReg <= False; + + stateReg <= TEST_ADJUST_PAYLOAD_PREPARE; + $display("time=%0t: clearAll", $time); + endrule + + rule genSGE if (stateReg == TEST_ADJUST_PAYLOAD_PREPARE); + let localAddr = localAddrPipeOut.first; + localAddrPipeOut.deq; + + let payloadLen = payloadLenPipeOut.first; + payloadLenPipeOut.deq; + + let isFirst = isZero(sglIdxReg); + let isLast = isAllOnesR(sglIdxReg); + let sge = ScatterGatherElem { + laddr : localAddr, + len : payloadLen, + lkey : dontCareValue, + isFirst: isFirst, + isLast : isLast + }; + sglRegVec[sglIdxReg] <= sge; + + if (isLast) begin + sglIdxReg <= 0; + stateReg <= TEST_ADJUST_PAYLOAD_ISSUE; + end + else begin + sglIdxReg <= sglIdxReg + 1; + end + + let { + sgeFragNum, sgeLastFragByteEn, sgeLastFragValidByteNum + } = calcTotalFragNumByLength(payloadLen); + countDown.decr; + // $display( + // "time=%0t: genSGE", $time, + // ", sglIdxReg=%0d", sglIdxReg, + // ", localAddr=%h", localAddr, + // ", payloadLen=%0d", payloadLen, + // ", sgeFragNum=%0d", sgeFragNum, + // ", sgeLastFragByteEn=%h", sgeLastFragByteEn, + // ", sgeLastFragValidByteNum=%0d", sgeLastFragValidByteNum + // ); + endrule + + rule issueDmaReadCntrlReq if (stateReg == TEST_ADJUST_PAYLOAD_ISSUE); + let sgl = readVReg(sglRegVec); + let pmtu = pmtuPipeOut.first; + pmtuPipeOut.deq; + pmtuQ.enq(pmtu); + + let dmaReadCntrlReq = DmaReadCntrlReq { + pmtu : pmtu, + sglDmaReadMetaData: DmaReadMetaDataSGL { + sgl : sgl, + sqpn : getDefaultQPN, + wrID : dontCareValue + // initiator: DMA_SRC_RQ_RD, + // startAddr: startAddr, + // len : payloadLen, + } + }; + dmaReadCntrl.srvPort.request.put(dmaReadCntrlReq); + + stateReg <= TEST_ADJUST_PAYLOAD_RECV; + for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin + let sge = sglRegVec[idx]; + let { + pmtuLen, sgeFirstPktLen, sgeLastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU(sge.laddr, sge.len, pmtu); + let sgeFirstPktFragNum = calcFragNumByPktLen(sgeFirstPktLen); + let sgeLastPktFragNum = calcFragNumByPktLen(sgeLastPktLen); + + // $display( + // "time=%0t: issueDmaReadCntrlReq", $time, + // ", SGE idx=%0d", idx, + // ", sge.laddr=%h", sge.laddr, + // ", sgePktNum=%0d", sgePktNum, + // ", sgeFirstPktLen=%0d", sgeFirstPktLen, + // ", sgeLastPktLen=%0d", sgeLastPktLen, + // ", pmtuLen=%0d", pmtuLen, + // ", sgeFirstPktFragNum=%0d", sgeFirstPktFragNum, + // ", sgeLastPktFragNum=%0d", sgeLastPktFragNum + // ); + end + endrule + + rule recvDmaReadCntrlResp; + let dmaReadCntrlResp <- dmaReadCntrl.srvPort.response.get; + sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); + endrule + + rule adjustTotalPayloadMetaData if (stateReg == TEST_ADJUST_PAYLOAD_RECV); + let remoteAddr = remoteAddrPipeOut.first; + remoteAddrPipeOut.deq; + + let pmtu = pmtuQ.first; + pmtuQ.deq; + + let totalPayloadMetaData = dmaReadCntrl.totalPayloadMetaDataPipeOut.first; + dmaReadCntrl.totalPayloadMetaDataPipeOut.deq; + let totalLen = totalPayloadMetaData.totalLen; + + let { + pmtuLen, firstPktLen, lastPktLen, totalPktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU( + remoteAddr, totalLen, pmtu + ); + remainingPktNumReg <= totalPktNum - 1; + isFirstPktReg <= True; + + let origLastFragValidByteNum = calcLastFragValidByteNum(totalLen); + let origPktNum = calcPktNumByLenOnly(totalLen, pmtu); + + let firstPktLastFragValidByteNum = calcLastFragValidByteNum(firstPktLen); + let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLen); + let firstPktFragNum = calcFragNumByPktLen(firstPktLen); + let lastPktFragNum = calcFragNumByPktLen(lastPktLen); + + let adjustedTotalPayloadMetaData = AdjustedTotalPayloadMetaData { + firstPktLen : firstPktLen, + firstPktFragNum : firstPktFragNum, + firstPktLastFragValidByteNum : firstPktLastFragValidByteNum, + // firstPktPadCnt : firstPktPadCnt, + // lastPktLen : lastPktLen, + // lastPktFragNum : lastPktFragNum, + // lastPktLastFragValidByteNum : lastPktLastFragValidByteNum, + // lastPktPadCnt : lastPktPadCnt, + origLastFragValidByteNum : origLastFragValidByteNum, + adjustedPktNum : totalPktNum, + origPktNum : origPktNum, + pmtu : pmtu + // totalLen : totalLen + }; + adjustedTotalPayloadMetaDataQ.enq(adjustedTotalPayloadMetaData); + adjustedTotalPayloadMetaDataRefQ.enq(tuple3( + adjustedTotalPayloadMetaData, lastPktFragNum, lastPktLastFragValidByteNum + )); + + stateReg <= TEST_ADJUST_PAYLOAD_RUN; + // $display( + // "time=%0t: adjustTotalPayloadMetaData", $time, + // ", pmtu=", fshow(pmtu), + // ", totalLen=%0d", totalLen, + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen, + // ", pmtuLen=%0d", pmtuLen, + // ", totalPktNum=%0d", totalPktNum, + // ", firstPktFragNum=%0d", firstPktFragNum, + // ", lastPktFragNum=%0d", lastPktFragNum, + // ", firstPktLastFragValidByteNum=%0d", firstPktLastFragValidByteNum, + // ", lastPktLastFragValidByteNum=%0d", lastPktLastFragValidByteNum, + // ", remoteAddr=%h", remoteAddr, + // ", secondChunkStartAddr=%h", secondChunkStartAddr + // ); + endrule + + rule checkAdjustedPayloadSegment if (stateReg == TEST_ADJUST_PAYLOAD_RUN); + let { + adjustedTotalPayloadMetaData, lastPktFragNum, lastPktLastFragValidByteNum + } = adjustedTotalPayloadMetaDataRefQ.first; + + let firstPktLen = adjustedTotalPayloadMetaData.firstPktLen; + let firstPktFragNum = adjustedTotalPayloadMetaData.firstPktFragNum; + // let lastPktLen = adjustedTotalPayloadMetaData.lastPktLen; + // let lastPktFragNum = adjustedTotalPayloadMetaData.lastPktFragNum; + // let pktNum = adjustedTotalPayloadMetaData.pktNum; + let pmtu = adjustedTotalPayloadMetaData.pmtu; + let firstPktLastFragValidByteNum = adjustedTotalPayloadMetaData.firstPktLastFragValidByteNum; + // let lastPktLastFragValidByteNum = adjustedTotalPayloadMetaData.lastPktLastFragValidByteNum; + + let firstPktLastFragByteEn = genByteEn(firstPktLastFragValidByteNum); + let lastPktLastFragByteEn = genByteEn(lastPktLastFragValidByteNum); + + let pmtuFragNum = calcFragNumByPMTU(pmtu); + let payloadFrag = dut.first; + dut.deq; + + let isLastPkt = isZero(remainingPktNumReg); + if (payloadFrag.isLast) begin + isFirstPktReg <= isLastPkt; + + if (isLastPkt) begin + adjustedTotalPayloadMetaDataRefQ.deq; + stateReg <= TEST_ADJUST_PAYLOAD_INIT; + end + else begin + remainingPktNumReg <= remainingPktNumReg - 1; + end + end + else begin + let remainingFragNum = remainingFragNumReg; + if (payloadFrag.isFirst) begin + if (isFirstPktReg) begin + remainingFragNum = firstPktFragNum - 2; + end + else if (isLastPkt) begin + remainingFragNum = lastPktFragNum - 2; + end + else begin + remainingFragNum = pmtuFragNum - 2; + end + end + else begin + remainingFragNum = remainingFragNumReg - 1; + end + remainingFragNumReg <= remainingFragNum; + $display( + "time=%0t: checkAdjustedPayloadSegment", $time, + ", remainingFragNum=%0d", remainingFragNum + ); + end + + if (payloadFrag.isFirst) begin + immAssert( + isZero(remainingFragNumReg), + "remainingFragNumReg assertion @ mkTestAdjustNormalOrSmallPayloadSegment", + $format( + "remainingFragNumReg=%0d", remainingFragNumReg, + " should be zero when payloadFrag.isFirst=", + fshow(payloadFrag.isFirst) + ) + ); + end + + ByteEn expectedLastFragByteEn = maxBound; + if (payloadFrag.isLast) begin + if (isFirstPktReg) begin + expectedLastFragByteEn = firstPktLastFragByteEn; + end + else if (isLastPkt) begin + expectedLastFragByteEn = lastPktLastFragByteEn; + end + immAssert( + payloadFrag.byteEn == expectedLastFragByteEn, + "payloadFrag.byteEn assertion @ mkTestAdjustNormalOrSmallPayloadSegment", + $format( + "payloadFrag.byteEn=%h", payloadFrag.byteEn, + " should == expectedLastFragByteEn=%h", expectedLastFragByteEn, + ", when payloadFrag.isLast=", fshow(payloadFrag.isLast), + ", isFirstPktReg=", fshow(isFirstPktReg), + ", isLastPkt=", fshow(isLastPkt) + ) + ); + + immAssert( + isZero(remainingFragNumReg), + "remainingFragNumReg assertion @ mkTestAdjustNormalOrSmallPayloadSegment", + $format( + "remainingFragNumReg=%0d", remainingFragNumReg, + " should be zero when payloadFrag.isLast=", + fshow(payloadFrag.isLast) + ) + ); + end + else begin + immAssert( + isAllOnesR(payloadFrag.byteEn), + "payloadFrag.byteEn assertion @ mkTestAdjustNormalOrSmallPayloadSegment", + $format( + "payloadFrag.byteEn=%h", payloadFrag.byteEn, + " should be all ones when payloadFrag.isLast=", + fshow(payloadFrag.isLast) + ) + ); + end + // $display( + // "time=%0t: checkAdjustedPayloadSegment", $time, + // ", isFirstPktReg=", fshow(isFirstPktReg), + // ", isLastPkt=", fshow(isLastPkt), + // ", remainingPktNumReg=%0d", remainingPktNumReg, + // ", remainingFragNumReg=%0d", remainingFragNumReg, + // ", pmtuFragNum=%0d", pmtuFragNum, + // ", firstPktFragNum=%0d", firstPktFragNum, + // ", firstPktLastFragValidByteNum=%0d", firstPktLastFragValidByteNum, + // ", firstPktLastFragByteEn=%h", firstPktLastFragByteEn, + // ", lastPktFragNum=%0d", lastPktFragNum, + // ", lastPktLastFragValidByteNum=%0d", lastPktLastFragValidByteNum, + // ", lastPktLastFragByteEn=%h", lastPktLastFragByteEn, + // ", payloadFrag.isFirst=", fshow(payloadFrag.isFirst), + // ", payloadFrag.isLast=", fshow(payloadFrag.isLast), + // ", payloadFrag.byteEn=%h", payloadFrag.byteEn + // ); + endrule +endmodule From 3ff2cf767856ed12ba5f4a3faa2e1cbc1276cc2d Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Mon, 22 Jan 2024 02:53:54 +0800 Subject: [PATCH 07/16] refactor mkMergePayloadEachSGE --- src/DataTypes.bsv | 2 +- src/PayloadGen.bsv | 475 +++++++++++++--------------------------- test/TestPayloadGen.bsv | 86 +++++--- 3 files changed, 212 insertions(+), 351 deletions(-) diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index 2905a11..3424e1e 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -72,7 +72,7 @@ typedef TLog#(MAX_RNR_WAIT_CYCLES) RNR_WAIT_CYCLE_CNT_WIDTH; typedef TDiv#(MAX_TIMEOUT_NS, TARGET_CYCLE_NS) MAX_TIMEOUT_CYCLES; typedef TAdd#(1, TLog#(MAX_TIMEOUT_CYCLES)) TIMEOUT_CYCLE_CNT_WIDTH; -typedef 48 PHYSICAL_ADDR_WIDTH; // X86 physical address width +typedef 48 PHYSICAL_ADDR_WIDTH; // X86 physical address width typedef TExp#(14) TLB_CACHE_SIZE; // TLB cache size 16K typedef TLog#(PAGE_SIZE_CAP) PAGE_OFFSET_WIDTH; typedef TLog#(TLB_CACHE_SIZE) TLB_CACHE_INDEX_WIDTH; // 14 diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index 4c9b59e..1128b9e 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -104,43 +104,24 @@ typedef struct { typedef Vector#(MAX_SGE, ScatterGatherElem) ScatterGatherList; -typedef struct { - // The last fragment ByteEn for each packet of the SGE - ByteEnBitNum curPktLastFragValidByteNum; - PktLen pktLen; - PMTU pmtu; - Bool isFirst; - Bool isLast; - Bool isFirstSGE; - Bool isLastSGE; -} PktMetaDataSGE deriving(Bits, FShow); // typedef struct { -// PktLen firstPktLen; -// PktLen lastPktLen; -// PktNum sgePktNum; -// Length sgeLen; -// PMTU pmtu; -// Bool isFirst; -// Bool isLast; -// } DmaReadMetaDataSGE deriving(Bits, FShow); -// typedef struct { -// PktLen firstPktLen; -// PktFragNum firstPktFragNum; -// ByteEnBitNum firstPktLastFragValidByteNum; -// // PAD firstPktPadCnt; -// // ByteEn firstPktLastFragByteEn; -// PktLen lastPktLen; -// PktFragNum lastPktFragNum; -// ByteEnBitNum lastPktLastFragValidByteNum; -// // PAD lastPktPadCnt; -// // ByteEn lastPktLastFragByteEn; -// Length sgeLen; -// PktNum sgePktNum; +// // The last fragment ByteEn for each packet of the SGE +// ByteEnBitNum curPktLastFragValidByteNum; +// PktLen pktLen; // PMTU pmtu; -// Bool isSinglePktSGE; +// Bool sgeHasJustTwoPkts; // Bool isFirst; // Bool isLast; -// } DetailMetaDataSGE deriving(Bits, FShow); +// Bool isFirstSGE; +// Bool isLastSGE; +// } PktMetaDataSGE deriving(Bits, FShow); + +typedef struct { + PktLen firstPktLen; + PktLen lastPktLen; + PktNum sgePktNum; + PMTU pmtu; +} PktMetaDataSGE deriving(Bits, FShow); typedef struct { // PktLen lastPktLen; @@ -155,11 +136,11 @@ typedef struct { QPN sqpn; // TODO: remove it WorkReqID wrID; // TODO: remove it Length totalLen; - NumSGE sgeNum; PMTU pmtu; + // NumSGE sgeNum; // PktFragNum pmtuFragNum; // PktLen pmtuLen; -} TotalPayloadMetaData deriving(Bits, FShow); +} TotalPayloadLenMetaDataSGL deriving(Bits, FShow); typedef struct { // QPN sqpn; // TODO: remove it @@ -369,15 +350,14 @@ endfunction interface AddrChunkSrv; interface Server#(AddrChunkReq, AddrChunkResp) srvPort; - // interface PipeOut#(DmaReadMetaDataSGE) sgeDmaReadMetaDataPipeOut; interface PipeOut#(PktMetaDataSGE) sgePktMetaDataPipeOut; method Bool isIdle(); endinterface +// TODO: refactor AddrChunkSrv into fully pipeline module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); FIFOF#(AddrChunkReq) reqQ <- mkSizedFIFOF(valueOf(MAX_SGE)); FIFOF#(AddrChunkResp) respQ <- mkFIFOF; - // FIFOF#(DmaReadMetaDataSGE) sgeDmaReadMetaDataOutQ <- mkSizedFIFOF(valueOf(MAX_SGE)); FIFOF#(PktMetaDataSGE) sgePktMetaDataOutQ <- mkFIFOF; Reg#(PktLen) firstPktLenReg <- mkRegU; @@ -385,39 +365,23 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); Reg#(PktLen) lastPktLenReg <- mkRegU; Reg#(Bool) isOrigFirstReg <- mkRegU; Reg#(Bool) isOrigLastReg <- mkRegU; - // Reg#(ResiduePMTU) residueReg <- mkRegU; - // Reg#(Bool) isZeroResidueReg <- mkRegU; - Reg#(PktNum) pktNumReg <- mkRegU; - Reg#(ADDR) chunkAddrReg <- mkRegU; - Reg#(ADDR) nextAddrReg <- mkRegU; - Reg#(PMTU) pmtuReg <- mkRegU; - Reg#(Bool) isFirstReg <- mkRegU; - Reg#(Bool) busyReg <- mkReg(False); + Reg#(PktNum) remainingPktNumReg <- mkRegU; + + Reg#(ADDR) chunkAddrReg <- mkRegU; + Reg#(ADDR) nextAddrReg <- mkRegU; + Reg#(PMTU) pmtuReg <- mkRegU; + Reg#(Bool) isFirstReg <- mkRegU; + Reg#(Bool) busyReg <- mkReg(False); PSN oneAsPSN = 1; rule resetAndClear if (clearAll); reqQ.clear; respQ.clear; - busyReg <= False; - // isFirstReg <= False; + busyReg <= False; endrule - // rule debug; - // $display( - // "time=%0t: mkAddrChunkSrv debug", $time, - // ", isSQ=", fshow(isSQ), - // ", reqQ.notEmpty=", fshow(reqQ.notEmpty), - // ", respQ.notFull=", fshow(respQ.notFull), - // ", pktNumReg=%0d", pktNumReg, - // ", fullPktLenReg=%h", fullPktLenReg, - // ", busyReg=", fshow(busyReg), - // ", isFirstReg=", fshow(isFirstReg), - // ", clearAll=", fshow(clearAll) - // ); - // endrule - rule recvReq if (!clearAll && !busyReg); let addrChunkReq = reqQ.first; reqQ.deq; @@ -435,17 +399,11 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); } = calcPktNumAndPktLenByAddrAndPMTU( addrChunkReq.startAddr, addrChunkReq.len, addrChunkReq.pmtu ); - // let { tmpPktNum, pmtuResidue } = truncateLenByPMTU( - // addrChunkReq.len, addrChunkReq.pmtu - // ); - // let isZeroResidue = isZeroR(pmtuResidue); - // let totalPktNum = tmpPktNum + (isZeroResidue ? 0 : 1); - // let pmtuLen = calcPmtuLen(addrChunkReq.pmtu); + remainingPktNumReg <= sgePktNum - 1; firstPktLenReg <= firstPktLen; fullPktLenReg <= pmtuLen; lastPktLenReg <= lastPktLen; - pktNumReg <= sgePktNum; chunkAddrReg <= addrChunkReq.startAddr; nextAddrReg <= secondChunkStartAddr; pmtuReg <= addrChunkReq.pmtu; @@ -453,45 +411,31 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); isOrigLastReg <= addrChunkReq.isLast; isFirstReg <= True; busyReg <= True; - // residueReg <= pmtuResidue; - // isZeroResidueReg <= isZeroResidue; - - // pktNumReg <= totalPktNum - 1; - // chunkAddrReg <= addrAddPsnMultiplyPMTU(addrChunkReq.startAddr, oneAsPSN, addrChunkReq.pmtu); - // busyReg <= !isSinglePkt; - // let addrChunkResp = AddrChunkResp { - // chunkAddr: addrChunkReq.startAddr, - // chunkLen : firstPktLen, - // isFirst : True, - // isLast : isSinglePkt - // }; - // respQ.enq(addrChunkResp); - - // let sgeDmaReadMetaData = DmaReadMetaDataSGE { - // firstPktLen: firstPktLen, - // lastPktLen : lastPktLen, - // sgePktNum : sgePktNum, - // sgeLen : addrChunkReq.len, - // pmtu : addrChunkReq.pmtu, - // isFirst : addrChunkReq.isOrigFirst, - // isLast : addrChunkReq.isOrigLast - // }; - // sgeDmaReadMetaDataOutQ.enq(sgeDmaReadMetaData); + + let sgePktMetaData = PktMetaDataSGE { + firstPktLen: firstPktLen, + lastPktLen : lastPktLen, + sgePktNum : sgePktNum, + pmtu : addrChunkReq.pmtu + }; + sgePktMetaDataOutQ.enq(sgePktMetaData); // $display( // "time=%0t: mkAddrChunkSrv recvReq", $time, - // ", isSQ=", fshow(isSQ), + // ", addrChunkReq.len=%0d", addrChunkReq.len, // ", sgePktNum=%0d", sgePktNum, - // ", pmtuResidue=%h", pmtuResidue, - // ", addrChunkReq=", fshow(addrChunkReq) + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen // ); endrule rule genResp if (!clearAll && busyReg); - let isLast = isLessOrEqOneR(pktNumReg); + let isLast = isZeroR(remainingPktNumReg); + if (!isLast) begin + remainingPktNumReg <= remainingPktNumReg - 1; + end busyReg <= !isLast; - pktNumReg <= pktNumReg - 1; isFirstReg <= False; chunkAddrReg <= nextAddrReg; nextAddrReg <= addrAddPsnMultiplyPMTU(nextAddrReg, oneAsPSN, pmtuReg); @@ -500,14 +444,13 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); let addrChunkResp = AddrChunkResp { chunkAddr : chunkAddrReg, chunkLen : chunkLen, - // chunkLen : (isLast && !isZeroResidueReg) ? zeroExtend(residueReg) : fullPktLenReg, isFirst : isFirstReg, isLast : isLast, isOrigFirst: isOrigFirstReg, isOrigLast : isOrigLastReg }; respQ.enq(addrChunkResp); - +/* let firstPktLastFragValidByteNum = calcLastFragValidByteNum(firstPktLenReg); let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLenReg); let curPktLastFragValidByteNum = isFirstReg ? @@ -519,17 +462,18 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); curPktLastFragValidByteNum: curPktLastFragValidByteNum, pktLen : chunkLen, pmtu : pmtuReg, + sgeHasJustTwoPkts : isTwoR(sgePktNumReg), isFirst : isFirstReg, isLast : isLast, isFirstSGE : isOrigFirstReg, isLastSGE : isOrigLastReg }; sgePktMetaDataOutQ.enq(sgePktMetaData); - +*/ // $display( // "time=%0t: mkAddrChunkSrv genResp", $time, // ", isSQ=", fshow(isSQ), - // ", pktNumReg=%0d", pktNumReg, + // ", remainingPktNumReg=%0d", remainingPktNumReg, // ", chunkAddrReg=%h", chunkAddrReg, // ", nextAddrReg=%h", nextAddrReg, // ", addrChunkResp=", fshow(addrChunkResp) @@ -537,7 +481,6 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); endrule interface srvPort = toGPServer(reqQ, respQ); - // interface sgeDmaReadMetaDataPipeOut = toPipeOut(sgeDmaReadMetaDataOutQ); interface sgePktMetaDataPipeOut = toPipeOut(sgePktMetaDataOutQ); method Bool isIdle() = !busyReg && !reqQ.notEmpty && !sgePktMetaDataOutQ.notEmpty && !respQ.notEmpty; @@ -566,7 +509,7 @@ interface DmaReadCntrl; interface DmaCntrlReadSrv srvPort; interface DmaCntrl dmaCntrl; interface PipeOut#(PktMetaDataSGE) sgePktMetaDataPipeOut; - interface PipeOut#(TotalPayloadMetaData) totalPayloadMetaDataPipeOut; + interface PipeOut#(TotalPayloadLenMetaDataSGL) sglTotalPayloadLenMetaDataPipeOut; interface PipeOut#(MergedMetaDataSGE) sgeMergedMetaDataPipeOut; endinterface @@ -576,7 +519,7 @@ module mkDmaReadCntrl#( FIFOF#(DmaReadCntrlReq) reqQ <- mkFIFOF; FIFOF#(DmaReadCntrlResp) respQ <- mkFIFOF; FIFOF#(MergedMetaDataSGE) sgeMergedMetaDataOutQ <- mkSizedFIFOF(valueOf(MAX_SGE)); - FIFOF#(TotalPayloadMetaData) totalPayloadMetaDataOutQ <- mkFIFOF; + FIFOF#(TotalPayloadLenMetaDataSGL) sglTotalPayloadLenMetaDataOutQ <- mkFIFOF; FIFOF#(Tuple2#(QPN, WorkReqID)) pendingDmaCntrlReqQ <- mkFIFOF; // TODO: remove it FIFOF#(Tuple2#(Bool, Bool)) pendingDmaReadReqQ <- mkFIFOF; @@ -689,18 +632,18 @@ module mkDmaReadCntrl#( reqQ.deq; sglIdxReg <= 0; - let totalPayloadMetaData = TotalPayloadMetaData { + let sglTotalPayloadLenMetaData = TotalPayloadLenMetaDataSGL { sqpn : curSQPN, wrID : curWorkReqID, totalLen: totalLen, - sgeNum : sgeNum, + // sgeNum : sgeNum, pmtu : dmaReadCntrlReq.pmtu }; - totalPayloadMetaDataOutQ.enq(totalPayloadMetaData); + sglTotalPayloadLenMetaDataOutQ.enq(sglTotalPayloadLenMetaData); // $display( // "time=%0t: mkDmaReadCntrl recvReq", $time, // ", sqpn=%h", curSQPN, - // ", totalPayloadMetaData=", fshow(totalPayloadMetaData) + // ", sglTotalPayloadLenMetaData=", fshow(sglTotalPayloadLenMetaData) // ); end else begin @@ -817,22 +760,18 @@ module mkDmaReadCntrl#( method Bool isIdle() = gracefulStopReg[0]; endinterface; - interface totalPayloadMetaDataPipeOut = toPipeOut(totalPayloadMetaDataOutQ); + interface sglTotalPayloadLenMetaDataPipeOut = toPipeOut(sglTotalPayloadLenMetaDataOutQ); interface sgeMergedMetaDataPipeOut = toPipeOut(sgeMergedMetaDataOutQ); interface sgePktMetaDataPipeOut = addrChunkSrv.sgePktMetaDataPipeOut; endmodule -// interface MergePayloadEachSGE; -// interface PipeOut#(MergedMetaDataSGE) sgeMergedMetaDataPipeOut; -// interface DataStreamPipeOut sgeMergedPayloadPipeOut; -// endinterface - typedef enum { MERGE_SGE_PAYLOAD_INIT, // MERGE_SGE_PAYLOAD_ONLY_PKT, - MERGE_SGE_PAYLOAD_FIRST_PKT, - MERGE_SGE_PAYLOAD_MID_PKT, + // MERGE_SGE_PAYLOAD_FIRST_PKT, + // MERGE_SGE_PAYLOAD_MID_PKT, // MERGE_SGE_PAYLOAD_LAST_PKT, + MERGE_SGE_PAYLOAD_FIRST_OR_MID_PKT, MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT } MergePayloadStateEachSGE deriving(Bits, Eq, FShow); @@ -843,16 +782,15 @@ module mkMergePayloadEachSGE#( )(DataStreamPipeOut); FIFOF#(DataStream) pktPayloadOutQ <- mkFIFOF; - Reg#(ByteEnBitNum) sgeFirstPktLastFragValidByteNumReg <- mkRegU; + // Reg#(ByteEnBitNum) sgeFirstPktLastFragValidByteNumReg <- mkRegU; Reg#(ByteEnBitNum) sgeFirstPktLastFragInvalidByteNumReg <- mkRegU; Reg#(BusBitNum) sgeFirstPktLastFragInvalidBitNumReg <- mkRegU; - // Reg#(ByteEnBitNum) sgeLastPktLastFragInvalidByteNumReg <- mkRegU; - // Reg#(BusBitNum) sgeLastPktLastFragInvalidBitNumReg <- mkRegU; - Reg#(Bool) isPreFragValidReg <- mkRegU; + Reg#(Bool) isFirstFragReg <- mkRegU; + Reg#(Bool) isFirstPktReg <- mkRegU; Reg#(Bool) sgeHasOnlyPktReg <- mkRegU; - Reg#(Bool) sgePktIsLastReg <- mkRegU; Reg#(Bool) hasExtraFragReg <- mkRegU; + Reg#(PktNum) remainingPktNumReg <- mkRegU; Reg#(DataStream) prePayloadFragReg <- mkRegU; Reg#(MergePayloadStateEachSGE) stateReg <- mkReg(MERGE_SGE_PAYLOAD_INIT); @@ -864,121 +802,93 @@ module mkMergePayloadEachSGE#( isLast: False }; - function Action prepareNextPktSGE(PktMetaDataSGE sgeNextPktMetaData); - action - sgePktIsLastReg <= sgeNextPktMetaData.isLast; - // In case sgeFirstPktLastFragValidByteNumReg=32 and curPktLastFragValidByteNum=32 - // expand extra bit for overflow case. - hasExtraFragReg <= !sgeHasOnlyPktReg && (( - { 1'b0, sgeFirstPktLastFragValidByteNumReg } + - { 1'b0, sgeNextPktMetaData.curPktLastFragValidByteNum } - ) > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); - // $display( - // "time=%0t: prepareNextPktSGE", $time, - // ", sgeHasOnlyPktReg=", fshow(sgeHasOnlyPktReg), - // ", sgeFirstPktLastFragValidByteNumReg=%0d", sgeFirstPktLastFragValidByteNumReg, - // ", sgeNextPktMetaData.curPktLastFragValidByteNum=%0d", sgeNextPktMetaData.curPktLastFragValidByteNum, - // ", sgeNextPktMetaData.isLast=", fshow(sgeNextPktMetaData.isLast) - // ); - endaction - endfunction - function ActionValue#(DataStream) prepareNextSGE(); actionvalue let sgePktMetaData = sgePktMetaDataPipeIn.first; sgePktMetaDataPipeIn.deq; - let curPktLastFragValidByteNum = sgePktMetaData.curPktLastFragValidByteNum; + let firstPktLastFragValidByteNum = calcLastFragValidByteNum(sgePktMetaData.firstPktLen); + let lastPktLastFragValidByteNum = calcLastFragValidByteNum(sgePktMetaData.lastPktLen); + let { - curPktLastFragValidBitNum, - curPktLastFragInvalidByteNum, - curPktLastFragInvalidBitNum - } = calcFragBitNumAndByteNum(curPktLastFragValidByteNum); - // let { - // lastPktLastFragValidBitNum, - // lastPktLastFragInvalidByteNum, - // lastPktLastFragInvalidBitNum - // } = calcFragBitNumAndByteNum(sgeDetailMetaData.lastPktLastFragValidByteNum); - - let isOnlyPktSGE = sgePktMetaData.isFirst && sgePktMetaData.isLast; - sgePktIsLastReg <= sgePktMetaData.isLast; - sgeHasOnlyPktReg <= isOnlyPktSGE; - - // if (sgePktMetaData.isLast) begin - // hasExtraFragReg <= !isOnlyPktSGE && (( - // sgeFirstPktLastFragValidByteNumReg + - // curPktLastFragValidByteNum - // ) > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); - // end - - if (sgePktMetaData.isFirst) begin - sgeFirstPktLastFragValidByteNumReg <= curPktLastFragValidByteNum; - sgeFirstPktLastFragInvalidByteNumReg <= curPktLastFragInvalidByteNum; - sgeFirstPktLastFragInvalidBitNumReg <= curPktLastFragInvalidBitNum; - // sgeLastPktLastFragInvalidByteNumReg <= lastPktLastFragInvalidByteNum; - // sgeLastPktLastFragInvalidBitNumReg <= lastPktLastFragInvalidBitNum; - end + firstPktLastFragValidBitNum, + firstPktLastFragInvalidByteNum, + firstPktLastFragInvalidBitNum + } = calcFragBitNumAndByteNum(firstPktLastFragValidByteNum); + let { + lastPktLastFragValidBitNum, + lastPktLastFragInvalidByteNum, + lastPktLastFragInvalidBitNum + } = calcFragBitNumAndByteNum(lastPktLastFragValidByteNum); + + let sgeHasJustTwoPkts = isTwoR(sgePktMetaData.sgePktNum); + let sgeHasOnlyPkt = isLessOrEqOneR(sgePktMetaData.sgePktNum); + sgeHasOnlyPktReg <= sgeHasOnlyPkt; + + let hasExtraFrag = lastPktLastFragValidByteNum > firstPktLastFragInvalidByteNum; + hasExtraFragReg <= hasExtraFrag; + + // sgeFirstPktLastFragValidByteNumReg <= firstPktLastFragValidByteNum; + sgeFirstPktLastFragInvalidByteNumReg <= firstPktLastFragInvalidByteNum; + sgeFirstPktLastFragInvalidBitNumReg <= firstPktLastFragInvalidBitNum; - // let curPayloadFrag = sgePayloadPipeIn.first; - // sgePayloadPipeIn.deq; - - // let nextPrePayloadFrag = curPayloadFrag; - // if (isOnlyPktSGE) begin - // stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; - // end - // else begin - // if (curPayloadFrag.isLast) begin // Single fragment first packet - // nextPrePayloadFrag.byteEn = curPayloadFrag.byteEn >> curPktLastFragInvalidByteNum; - // nextPrePayloadFrag.data = curPayloadFrag.data >> curPktLastFragInvalidBitNum; - - // stateReg <= MERGE_SGE_PAYLOAD_MID_PKT; - // end - // else begin - // stateReg <= MERGE_SGE_PAYLOAD_FIRST_PKT; - // end - // end let curPayloadFrag = sgePayloadPipeIn.first; - let nextPrePayloadFrag = curPayloadFrag; - if (curPayloadFrag.isLast && !isOnlyPktSGE) begin // Single fragment first packet - nextPrePayloadFrag = emptyDataStream; - isPreFragValidReg <= False; - // nextPrePayloadFrag.isLast = False; - end - else begin - sgePayloadPipeIn.deq; - isPreFragValidReg <= True; - end + sgePayloadPipeIn.deq; - if (isOnlyPktSGE) begin + let remainingPktNum = 0; + let isFirstPkt = True; + let nextPrePayloadFrag = curPayloadFrag; + if (sgeHasOnlyPkt) begin stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; end else begin - stateReg <= MERGE_SGE_PAYLOAD_FIRST_PKT; + if (curPayloadFrag.isLast) begin // Single fragment first packet + nextPrePayloadFrag.isLast = False; + nextPrePayloadFrag.byteEn = curPayloadFrag.byteEn >> firstPktLastFragInvalidByteNum; + nextPrePayloadFrag.data = curPayloadFrag.data >> firstPktLastFragInvalidBitNum; + + isFirstPkt = False; + if (sgeHasJustTwoPkts) begin + stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; + end + else begin + stateReg <= MERGE_SGE_PAYLOAD_FIRST_OR_MID_PKT; + remainingPktNum = sgePktMetaData.sgePktNum - 2; + end + end + else begin + stateReg <= MERGE_SGE_PAYLOAD_FIRST_OR_MID_PKT; + remainingPktNum = sgePktMetaData.sgePktNum - 1; + end end + remainingPktNumReg <= remainingPktNum; + isFirstPktReg <= isFirstPkt; + isFirstFragReg <= True; // $display( // "time=%0t: prepareNextSGE", $time, - // ", sgePktMetaData.isFirst=", fshow(sgePktMetaData.isFirst), - // ", sgePktMetaData.isLast=", fshow(sgePktMetaData.isLast), - // ", curPktLastFragValidByteNum=%0d", curPktLastFragValidByteNum, - // ", curPktLastFragValidBitNum=%0d", curPktLastFragValidBitNum, - // ", curPktLastFragInvalidByteNum=%0d", curPktLastFragInvalidByteNum, - // ", curPktLastFragInvalidBitNum=%0d", curPktLastFragInvalidBitNum, - // ", sgeFirstPktLastFragValidByteNumReg=%0d", sgeFirstPktLastFragValidByteNumReg, + // ", sgeHasOnlyPkt=", fshow(sgeHasOnlyPkt), + // ", sgeHasJustTwoPkts=", fshow(sgeHasJustTwoPkts), + // ", sgePktMetaData.sgePktNum=%0d", sgePktMetaData.sgePktNum, + // ", sgePktMetaData.firstPktLen=%0d", sgePktMetaData.firstPktLen, + // ", sgePktMetaData.lastPktLen=%0d", sgePktMetaData.lastPktLen, + // ", firstPktLastFragValidByteNum=%0d", firstPktLastFragValidByteNum, + // ", firstPktLastFragValidBitNum=%0d", firstPktLastFragValidBitNum, + // ", firstPktLastFragInvalidByteNum=%0d", firstPktLastFragInvalidByteNum, + // ", firstPktLastFragInvalidBitNum=%0d", firstPktLastFragInvalidBitNum, + // // ", sgeFirstPktLastFragValidByteNumReg=%0d", sgeFirstPktLastFragValidByteNumReg, // ", sgeFirstPktLastFragInvalidByteNumReg=%0d", sgeFirstPktLastFragInvalidByteNumReg, // ", sgeFirstPktLastFragInvalidBitNumReg=%0d", sgeFirstPktLastFragInvalidBitNumReg, // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), - // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast) + // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), + // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn // ); - - isFirstFragReg <= True; return nextPrePayloadFrag; endactionvalue endfunction rule resetAndClear if (clearAll); pktPayloadOutQ.clear; - // sgeMergedMetaDataOutQ.clear; stateReg <= MERGE_SGE_PAYLOAD_INIT; endrule @@ -986,137 +896,52 @@ module mkMergePayloadEachSGE#( let nextPrePayloadFrag <- prepareNextSGE; prePayloadFragReg <= nextPrePayloadFrag; endrule -/* - rule mergeOnlyPktSGE if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_ONLY_PKT); - let nextPayloadFrag = emptyDataStream; - if (prePayloadFragReg.isLast) begin - if ( - sgePktIsLastReg && !( - sgeDetailMetaDataPipeIn.notEmpty && sgePayloadPipeIn.notEmpty - ) - ) begin - // Wait for next SGL, if no next SGE metadata or payload - stateReg <= MERGE_SGE_PAYLOAD_INIT; - end - else begin - nextPayloadFrag <- prepareNextSGE; - end - end - else begin - nextPayloadFrag = sgePayloadPipeIn.first; - sgePayloadPipeIn.deq; - end - prePayloadFragReg <= nextPayloadFrag; - - immAssert( - isOneR(sgeRemainingPktNumReg), - "sgeRemainingPktNumReg assertion @ mkMergePayloadEachSGE", - $format( - "sgeRemainingPktNumReg=%0d", sgeRemainingPktNumReg, - " should be one when stateReg=", fshow(stateReg) - ) - ); - - pktPayloadOutQ.enq(prePayloadFragReg); - endrule -*/ - rule mergeFirstPktSGE if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_FIRST_PKT); - let sgeNextPktMetaData = sgePktMetaDataPipeIn.first; + rule mergeFirstOrMidPktSGE if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_FIRST_OR_MID_PKT); let curPayloadFrag = sgePayloadPipeIn.first; sgePayloadPipeIn.deq; let nextPrePayloadFrag = curPayloadFrag; if (curPayloadFrag.isLast) begin - sgePktMetaDataPipeIn.deq; + nextPrePayloadFrag.isLast = False; - // Only right shift the last fragment of the first packet - nextPrePayloadFrag.byteEn = curPayloadFrag.byteEn >> sgeFirstPktLastFragInvalidByteNumReg; - nextPrePayloadFrag.data = curPayloadFrag.data >> sgeFirstPktLastFragInvalidBitNumReg; + if (isFirstPktReg) begin + isFirstPktReg <= False; + // Only right shift the last fragment of the first packet + nextPrePayloadFrag.byteEn = curPayloadFrag.byteEn >> sgeFirstPktLastFragInvalidByteNumReg; + nextPrePayloadFrag.data = curPayloadFrag.data >> sgeFirstPktLastFragInvalidBitNumReg; + end - // sgePktIsLastReg <= sgeNextPktMetaData.isLast; - if (sgeNextPktMetaData.isLast) begin + let isLastPkt = isLessOrEqOneR(remainingPktNumReg); + if (isLastPkt) begin stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; - prepareNextPktSGE(sgeNextPktMetaData); - // hasExtraFragReg <= checkExtraFrag(sgeNextPktMetaData.curPktLastFragValidByteNum); - end - else begin - stateReg <= MERGE_SGE_PAYLOAD_MID_PKT; end + remainingPktNumReg <= remainingPktNumReg - 1; end - isPreFragValidReg <= True; - nextPrePayloadFrag.isLast = False; prePayloadFragReg <= nextPrePayloadFrag; immAssert( - !sgePktIsLastReg, - "sgePktIsLastReg assertion @ mkMergePayloadEachSGE", + !isZeroR(remainingPktNumReg), + "remainingPktNumReg assertion @ mkMergePayloadEachSGE", $format( - "sgePktIsLastReg=", fshow(sgePktIsLastReg), - " should be false when stateReg=", fshow(stateReg) + "remainingPktNumReg=%0d", fshow(remainingPktNumReg), + " should > 0 when stateReg=", fshow(stateReg) ) ); - if (isPreFragValidReg) begin - isFirstFragReg <= False; - let outPayloadFrag = prePayloadFragReg; - pktPayloadOutQ.enq(outPayloadFrag); - // $display( - // "time=%0t: mergeFirstPktSGE", $time, - // ", sgeNextPktMetaData.isFirst=", fshow(sgeNextPktMetaData.isFirst), - // ", sgeNextPktMetaData.isLast=", fshow(sgeNextPktMetaData.isLast), - // ", sgeNextPktMetaData.curPktLastFragValidByteNum=%0d", sgeNextPktMetaData.curPktLastFragValidByteNum, - // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), - // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), - // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn, - // ", outPayloadFrag.isFirst=", fshow(outPayloadFrag.isFirst), - // ", outPayloadFrag.isLast=", fshow(outPayloadFrag.isLast), - // ", outPayloadFrag.byteEn=%h", outPayloadFrag.byteEn - // ); - end - endrule - - rule mergeMidPktSGE if (!clearAll && stateReg == MERGE_SGE_PAYLOAD_MID_PKT); - let sgeNextPktMetaData = sgePktMetaDataPipeIn.first; - - let curPayloadFrag = sgePayloadPipeIn.first; - sgePayloadPipeIn.deq; - - if (curPayloadFrag.isLast) begin - sgePktMetaDataPipeIn.deq; - // $display( - // "time=%0t: mergeMidPktSGE, sgePktMetaDataPipeIn.deq", $time, - // ", sgeNextPktMetaData.curPktLastFragValidByteNum=%0d", sgeNextPktMetaData.curPktLastFragValidByteNum - // ); - - // sgePktIsLastReg <= sgeNextPktMetaData.isLast; - if (sgeNextPktMetaData.isLast) begin - stateReg <= MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT; - prepareNextPktSGE(sgeNextPktMetaData); - // hasExtraFragReg <= checkExtraFrag(sgeNextPktMetaData.curPktLastFragValidByteNum); - end - end - let nextPrePayloadFrag = curPayloadFrag; - nextPrePayloadFrag.isLast = False; - prePayloadFragReg <= nextPrePayloadFrag; - let mergedFrag = mergeFragData( prePayloadFragReg, curPayloadFrag, sgeFirstPktLastFragInvalidByteNumReg, sgeFirstPktLastFragInvalidBitNumReg ); - isFirstFragReg <= False; - let outPayloadFrag = mergedFrag; + + let outPayloadFrag = isFirstPktReg ? prePayloadFragReg : mergedFrag; outPayloadFrag.isFirst = isFirstFragReg; - // outPayloadFrag.isLast = False; + isFirstFragReg <= False; pktPayloadOutQ.enq(outPayloadFrag); // $display( - // "time=%0t: mergeMidPktSGE", $time, - // ", sgeNextPktMetaData.isFirst=", fshow(sgeNextPktMetaData.isFirst), - // ", sgeNextPktMetaData.isLast=", fshow(sgeNextPktMetaData.isLast), - // ", sgeNextPktMetaData.curPktLastFragValidByteNum=%0d", sgeNextPktMetaData.curPktLastFragValidByteNum, - // // ", sgeFirstPktLastFragInvalidByteNumReg=%0d", sgeFirstPktLastFragInvalidByteNumReg, - // // ", sgeFirstPktLastFragInvalidBitNumReg=%0d", sgeFirstPktLastFragInvalidBitNumReg, - // // ", sgeNextPktMetaData.isLast=", fshow(sgeNextPktMetaData.isLast), + // "time=%0t: mergeFirstOrMidPktSGE", $time, + // ", sgeFirstPktLastFragInvalidByteNumReg=%0d", sgeFirstPktLastFragInvalidByteNumReg, + // ", sgeFirstPktLastFragInvalidBitNumReg=%0d", sgeFirstPktLastFragInvalidBitNumReg, // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast), // ", curPayloadFrag.byteEn=%h", curPayloadFrag.byteEn, @@ -1153,12 +978,14 @@ module mkMergePayloadEachSGE#( end prePayloadFragReg <= nextPayloadFrag; + let isLastPkt = isZeroR(remainingPktNumReg); immAssert( - sgePktIsLastReg, - "sgePktIsLastReg assertion @ mkMergePayloadEachSGE", + isLastPkt, + "isLastPkt assertion @ mkMergePayloadEachSGE", $format( - "sgePktIsLastReg=", fshow(sgePktIsLastReg), - " should be true when stateReg=", fshow(stateReg) + "isLastPkt=", fshow(isLastPkt), + " should be true when stateReg=", fshow(stateReg), + ", and remainingPktNumReg=%0d", remainingPktNumReg ) ); @@ -1192,9 +1019,9 @@ endmodule typedef enum { MERGE_SGL_PAYLOAD_INIT, MERGE_SGL_PAYLOAD_FIRST_OR_MID_SGE, + MERGE_SGL_PAYLOAD_LAST_OR_ONLY_SGE // MERGE_SGL_PAYLOAD_FIRST_SGE, // MERGE_SGL_PAYLOAD_MID_SGE, - MERGE_SGL_PAYLOAD_LAST_OR_ONLY_SGE // MERGE_SGL_PAYLOAD_LAST_SGE, // MERGE_SGL_PAYLOAD_ONLY_SGE } MergePayloadStateAllSGE deriving(Bits, Eq, FShow); @@ -1974,9 +1801,9 @@ module mkPayloadGenerator#( let totalLen = 0; if (!isZeroPayloadLen) begin - let totalPayloadMetaData = dmaReadCntrl.totalPayloadMetaDataPipeOut.first; - dmaReadCntrl.totalPayloadMetaDataPipeOut.deq; - totalLen = totalPayloadMetaData.totalLen; + let sglTotalPayloadLenMetaData = dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.first; + dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.deq; + totalLen = sglTotalPayloadLenMetaData.totalLen; end let { truncatedPktNum, residue } = truncateLenByPMTU(totalLen, wqe.pmtu); diff --git a/test/TestPayloadGen.bsv b/test/TestPayloadGen.bsv index 956bdfc..4394307 100644 --- a/test/TestPayloadGen.bsv +++ b/test/TestPayloadGen.bsv @@ -55,8 +55,6 @@ module mkTestAddrChunkSrv(Empty); PSN oneAsPSN = 1; let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); - // mkSink(dut.sgePktMetaDataPipeOut); - rule clearAll if (clearReg); clearReg <= False; endrule @@ -84,15 +82,18 @@ module mkTestAddrChunkSrv(Empty); let { tmpPktNum, pmtuResidue } = truncateLenByPMTU( totalLen, pmtu ); - let isOnlyPkt = isLessOrEqOne(totalPktNum); + let isOnlyPkt = isLessOrEqOne(totalPktNum); let isZeroResidue = isZero(pmtuResidue); - // let totalPktNum = tmpPktNum + (isZero(pmtuResidue) ? 0 : 1); - // residueReg <= pmtuResidue; + let firstPktLen = isOnlyPkt ? truncate(payloadLen) : tmpFirstPktLen; + let lastPktLen = isOnlyPkt ? truncate(payloadLen) : ( + isZeroResidue ? pmtuLen : zeroExtend(pmtuResidue) + ); + pktNumReg <= totalPktNum; pmtuReg <= pmtu; pmtuLenReg <= pmtuLen; - firstPktLenReg <= isOnlyPkt ? truncate(payloadLen) : tmpFirstPktLen; - lastPktLenReg <= isZeroResidue ? pmtuLen : zeroExtend(pmtuResidue); + firstPktLenReg <= firstPktLen; + lastPktLenReg <= lastPktLen; nextAddrReg <= alignedAddr; isFirstReg <= True; busyReg <= True; @@ -107,6 +108,14 @@ module mkTestAddrChunkSrv(Empty); dut.srvPort.request.put(addrChunkReq); reqQ.enq(addrChunkReq); + let sgePktMetaData = PktMetaDataSGE { + firstPktLen: firstPktLen, + lastPktLen : lastPktLen, + sgePktNum : totalPktNum, + pmtu : addrChunkReq.pmtu + }; + sgePktMetaDataRefQ.enq(sgePktMetaData); + countDown.decr; // $display( // "time=%0t: issueReq", $time, @@ -117,6 +126,8 @@ module mkTestAddrChunkSrv(Empty); // ", pmtuResidue=%0d", pmtuResidue, // ", tmpFirstPktLen=%0d", tmpFirstPktLen, // ", totalPktNum=%0d", totalPktNum, + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen, // ", pmtuLen=%0d", pmtuLen, // ", pmtu=", fshow(pmtu), // ", isOnlyPkt=", fshow(isOnlyPkt) @@ -128,12 +139,8 @@ module mkTestAddrChunkSrv(Empty); isFirstReg <= False; pktNumReg <= pktNumReg - 1; - // if (!isFirstReg) begin nextAddrReg <= addrAddPsnMultiplyPMTU(nextAddrReg, oneAsPSN, pmtuReg); - // end - // let isZeroResidue = isZero(residueReg); - // let isFirst = nextAddrReg == req.startAddr; let isLast = isLessOrEqOne(pktNumReg); if (isLast) begin reqQ.deq; @@ -154,16 +161,16 @@ module mkTestAddrChunkSrv(Empty); }; respQ.enq(addrChunkResp); - let sgePktMetaDataRef = PktMetaDataSGE { - curPktLastFragValidByteNum: curPktLastFragValidByteNum, - pktLen : chunkLen, - pmtu : pmtuReg, - isFirst : isFirstReg, - isLast : isLast, - isFirstSGE : req.isFirst, - isLastSGE : req.isLast - }; - sgePktMetaDataRefQ.enq(sgePktMetaDataRef); + // let sgePktMetaDataRef = PktMetaDataSGE { + // curPktLastFragValidByteNum: curPktLastFragValidByteNum, + // pktLen : chunkLen, + // pmtu : pmtuReg, + // isFirst : isFirstReg, + // isLast : isLast, + // isFirstSGE : req.isFirst, + // isLastSGE : req.isLast + // }; + // sgePktMetaDataRefQ.enq(sgePktMetaDataRef); endrule rule checkResp; @@ -205,6 +212,32 @@ module mkTestAddrChunkSrv(Empty); // $display("time=%0t: checkResp", $time); endrule + rule checkPktMetaDataSGE; + let sgePktMetaData = dut.sgePktMetaDataPipeOut.first; + dut.sgePktMetaDataPipeOut.deq; + let sgePktMetaDataRef = sgePktMetaDataRefQ.first; + sgePktMetaDataRefQ.deq; + + immAssert( + sgePktMetaData.firstPktLen == sgePktMetaDataRef.firstPktLen && + sgePktMetaData.lastPktLen == sgePktMetaDataRef.lastPktLen && + sgePktMetaData.sgePktNum == sgePktMetaDataRef.sgePktNum && + sgePktMetaData.pmtu == sgePktMetaDataRef.pmtu, + "sgePktMetaData assertion @ mkTestAddrChunkSrv", + $format( + "sgePktMetaData.firstPktLen=%0d should == sgePktMetaDataRef.firstPktLen=%0d", + sgePktMetaData.firstPktLen, sgePktMetaDataRef.firstPktLen, + ", sgePktMetaData.lastPktLen=%0d should == sgePktMetaDataRef.lastPktLen=%0d", + sgePktMetaData.lastPktLen, sgePktMetaDataRef.lastPktLen, + ", sgePktMetaData.sgePktNum=%0d", sgePktMetaData.sgePktNum, + " should == sgePktMetaDataRef.sgePktNum=%0d", sgePktMetaDataRef.sgePktNum, + ", sgePktMetaData.pmtu=", fshow(sgePktMetaData.pmtu), + " should == sgePktMetaData.pmtu=", fshow(sgePktMetaData.pmtu) + ) + ); + // $display("time=%0t: checkPktMetaDataSGE", $time); + endrule +/* rule checkPktMetaDataSGE; let sgePktMetaData = dut.sgePktMetaDataPipeOut.first; dut.sgePktMetaDataPipeOut.deq; @@ -239,6 +272,7 @@ module mkTestAddrChunkSrv(Empty); ); // $display("time=%0t: checkPktMetaDataSGE", $time); endrule +*/ endmodule (* doc = "testcase" *) @@ -290,7 +324,7 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); mkSink(dut.sgePktMetaDataPipeOut); - mkSink(dut.totalPayloadMetaDataPipeOut); + mkSink(dut.sglTotalPayloadLenMetaDataPipeOut); mkSink(dut.sgeMergedMetaDataPipeOut); rule clearAll if (stateReg == TEST_DMA_CNTRL_INIT); @@ -585,7 +619,7 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); // mkSink(dut.sgePktMetaDataPipeOut); - mkSink(dmaReadCntrl.totalPayloadMetaDataPipeOut); + mkSink(dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut); mkSink(dmaReadCntrl.sgeMergedMetaDataPipeOut); rule clearAll if (stateReg == TEST_MERGE_EACH_SGE_INIT); @@ -844,7 +878,7 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( // mkSink(dmaReadCntrl.sgePktMetaDataPipeOut); // mkSink(dmaReadCntrl.sgeMergedMetaDataPipeOut); - mkSink(dmaReadCntrl.totalPayloadMetaDataPipeOut); + mkSink(dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut); rule clearAll if (stateReg == TEST_MERGE_ALL_SGE_INIT); clearReg <= False; @@ -1213,8 +1247,8 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( let pmtu = pmtuQ.first; pmtuQ.deq; - let totalPayloadMetaData = dmaReadCntrl.totalPayloadMetaDataPipeOut.first; - dmaReadCntrl.totalPayloadMetaDataPipeOut.deq; + let totalPayloadMetaData = dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.first; + dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.deq; let totalLen = totalPayloadMetaData.totalLen; let { From d11f1932865f1371f310b04f470c458f31abfc64 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Mon, 22 Jan 2024 03:08:52 +0800 Subject: [PATCH 08/16] refactor hasExtraFrag and hasLessFrag --- src/PayloadGen.bsv | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index 1128b9e..1b76aa4 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -1151,8 +1151,8 @@ module mkMergePayloadAllSGE#( let sumInvalidByteNum = nextInvalidByteNumReg + lastFragInvalidByteNum; let sumInvalidBitNum = nextInvalidBitNumReg + lastFragInvalidBitNum; - // let hasLessFrag = !sgeIsOnlyReg && !(sumValidByteNum > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); - let hasLessFrag = !sgeIsOnlyReg && (sumInvalidByteNum >= fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); + let hasLessFrag = nextInvalidByteNumReg >= lastFragValidByteNum; + // let hasLessFrag = !sgeIsOnlyReg && (sumInvalidByteNum >= fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); hasLessFragReg <= hasLessFrag; // let curValidByteNum = nextValidByteNumReg; @@ -1411,10 +1411,11 @@ module mkAdjustPayloadSegment#( // lastPktLastFragInvalidBitNumReg <= lastPktLastFragInvalidBitNum; let sglHasOnlyPkt = isOneR(adjustedTotalPayloadMeta.adjustedPktNum); sglHasOnlyPktReg <= sglHasOnlyPkt; - let hasExtraFrag = !sglHasOnlyPkt && (( - { 1'b0, firstPktLastFragInvalidByteNum } + - { 1'b0, origLastFragValidByteNum } - ) > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH))); + let hasExtraFrag = firstPktLastFragValidByteNum < origLastFragValidByteNum; + // let hasExtraFrag = ( + // { 1'b0, firstPktLastFragInvalidByteNum } + + // { 1'b0, origLastFragValidByteNum } + // ) > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)); hasExtraFragReg <= hasExtraFrag; if (sglHasOnlyPkt) begin From ca28d9218a47e1873bba366a5234ad1eefad736b Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Thu, 25 Jan 2024 00:52:44 +0800 Subject: [PATCH 09/16] refactor PayloadGenerator --- src/PayloadGen.bsv | 362 ++++++++++++++++++++++----------- test/TestPayloadGen.bsv | 428 +++++++++++++++++++++++++++++----------- 2 files changed, 565 insertions(+), 225 deletions(-) diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index 1b76aa4..d7afbdb 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -1332,6 +1332,7 @@ typedef enum { // ADJUST_PAYLOAD_SEGMENT_LAST_PKT_EXTRA_FRAG } AdjustPayloadSegmentState deriving(Bits, Eq, FShow); +// TODO: output target address, isFirst, isLast of each packet module mkAdjustPayloadSegment#( Bool clearAll, PipeOut#(AdjustedTotalPayloadMetaData) adjustedTotalPayloadMetaDataPipeIn, @@ -1355,7 +1356,6 @@ module mkAdjustPayloadSegment#( Reg#(Bool) sglHasOnlyPktReg <- mkRegU; Reg#(Bool) sglHasOnlyFragOnlyPktReg <- mkRegU; - // Reg#(PktLen) pmtuPktLenReg <- mkRegU; // Reg#(PktFragNum) lastPktFragNumReg <- mkRegU; Reg#(PktFragNum) pmtuFragNumReg <- mkRegU; Reg#(PktFragNum) pktRemainingFragNumReg <- mkRegU; @@ -1380,7 +1380,6 @@ module mkAdjustPayloadSegment#( let adjustedTotalPayloadMeta = adjustedTotalPayloadMetaDataPipeIn.first; adjustedTotalPayloadMetaDataPipeIn.deq; - // pmtuPktLenReg <= calcPmtuLen(adjustedTotalPayloadMeta.pmtu); pmtuFragNumReg <= calcFragNumByPMTU(adjustedTotalPayloadMeta.pmtu); pktRemainingFragNumReg <= adjustedTotalPayloadMeta.firstPktFragNum; @@ -1561,7 +1560,7 @@ module mkAdjustPayloadSegment#( nextPayloadFrag = sglAllPayloadPipeIn.first; sglAllPayloadPipeIn.deq; - // nextPayloadFrag.isFirst = False; + nextPayloadFrag.isFirst = False; if (!sglHasOnlyPktReg && !hasExtraFragReg && nextPayloadFrag.isLast) begin // No extra fragment stateReg <= ADJUST_PAYLOAD_SEGMENT_INIT; @@ -1647,20 +1646,38 @@ module mkConnectPipeOut2BramQ#( endmodule typedef struct { - WorkQueueElem wqe; - // DmaReadMetaDataSGL sglDmaReadMetaData; - // PMTU pmtu; - // ADDR raddr; - // Bool segment; - // Bool addPadding; + WorkReqID wrID; // TODO: remote it + QPN sqpn; // TODO: remote it + ScatterGatherList sgl; + ADDR raddr; + PMTU pmtu; } PayloadGenReqSG deriving(Bits, FShow); typedef struct { - // Bool segment; - // Bool addPadding; - Bool isRespErr; + ADDR raddr; + PktLen pktLen; + PAD padCnt; + // PMTU pmtu; + Bool isZeroPayloadLen; + Bool isFirst; + Bool isLast; + Bool isRespErr; // TODO: handle DMA read error } PayloadGenRespSG deriving(Bits, FShow); +typedef struct { + ADDR remoteAddr; + PktLen firstPktLen; + PktLen lastPktLen; + PktLen pmtuLen; + ByteEnBitNum firstPktLastFragValidByteNum; + PAD firstPktPadCnt; + ByteEnBitNum lastPktLastFragValidByteNum; + PAD lastPktPadCnt; + PktNum totalPktNum; + Bool isOnlyPkt; + Bool isZeroPayloadLen; +} PaddingMetaData deriving(Bits, FShow); + interface PayloadGenerator; interface Server#(PayloadGenReqSG, PayloadGenRespSG) srvPort; interface DataStreamPipeOut payloadDataStreamPipeOut; @@ -1668,29 +1685,35 @@ interface PayloadGenerator; endinterface module mkPayloadGenerator#( - Bool clearAll, DmaReadCntrl dmaReadCntrl + Bool clearAll, Bool shouldAddPadding, DmaReadCntrl dmaReadCntrl )(PayloadGenerator); FIFOF#(PayloadGenReqSG) payloadGenReqQ <- mkFIFOF; FIFOF#(PayloadGenRespSG) payloadGenRespQ <- mkFIFOF; // Pipeline FIFO + FIFOF#(DataStream) sgePayloadOutQ <- mkFIFOF; FIFOF#(Tuple2#(PayloadGenReqSG, Bool)) adjustReqPktLenQ <- mkFIFOF; - FIFOF#(Tuple8#(PayloadGenReqSG, Length, PktNum, ResiduePMTU, PMTU, PktLen, PktNum, Bool)) adjustedFirstAndLastPktLenQ <- mkFIFOF; - FIFOF#(Tuple2#(PayloadGenReqSG, AdjustedTotalPayloadMetaData)) adjustedTotalPayloadMetaDataQ <- mkFIFOF; - // FIFOF#(Tuple3#(PayloadGenReqSG, ByteEn, PktFragNum)) pendingGenReqQ <- mkFIFOF; - // FIFOF#(Tuple3#(Bool, Bool, PktFragNum)) pendingGenRespQ <- mkFIFOF; - // FIFOF#(Tuple3#(DataStream, Bool, Bool)) payloadSegmentQ <- mkFIFOF; + FIFOF#(Tuple8#(PayloadGenReqSG, Length, PktNum, PktNum, PktLen, PktLen, PktLen, Bool)) adjustedFirstAndLastPktLenQ <- mkFIFOF; + FIFOF#(PaddingMetaData) addPadCntQ <- mkFIFOF; + FIFOF#(AdjustedTotalPayloadMetaData) adjustedTotalPayloadMetaDataQ <- mkFIFOF; + + let sgeMergedPayloadPipeOut <- mkMergePayloadEachSGE( + clearAll, dmaReadCntrl.sgePktMetaDataPipeOut, toPipeOut(sgePayloadOutQ) + ); + let sglMergedPayloadPipeOut <- mkMergePayloadAllSGE( + clearAll, dmaReadCntrl.sgeMergedMetaDataPipeOut, sgeMergedPayloadPipeOut + ); + let adjustedPayloadPipeOut <- mkAdjustPayloadSegment( + clearAll, toPipeOut(adjustedTotalPayloadMetaDataQ), sglMergedPayloadPipeOut + ); // TODO: check payloadOutQ buffer size is enough for DMA read delay? FIFOF#(DataStream) payloadBufQ <- mkSizedBRAMFIFOF(valueOf(DATA_STREAM_FRAG_BUF_SIZE)); let bramQ2PipeOut <- mkConnectBramQ2PipeOut(payloadBufQ); let payloadBufPipeOut = bramQ2PipeOut.pipeOut; - // let payloadBufPipeOut <- mkConnectBramQ2PipeOut(payloadBufQ); - Reg#(PktFragNum) pmtuFragCntReg <- mkRegU; - Reg#(Bool) shouldSetFirstReg <- mkReg(False); - Reg#(Bool) isFragCntZeroReg <- mkReg(False); - Reg#(Bool) isNormalStateReg <- mkReg(True); + Reg#(Bool) isFirstPktReg <- mkReg(True); // TODO: remove it + Reg#(PktNum) remainingPktNumReg <- mkReg(0); // TODO: remove it rule resetAndClear if (clearAll); payloadGenReqQ.clear; @@ -1699,16 +1722,11 @@ module mkPayloadGenerator#( adjustReqPktLenQ.clear; adjustedFirstAndLastPktLenQ.clear; adjustedTotalPayloadMetaDataQ.clear; - // pendingGenReqQ.clear; - // pendingGenRespQ.clear; - // payloadSegmentQ.clear; payloadBufQ.clear; - bramQ2PipeOut.clear; - shouldSetFirstReg <= False; - isFragCntZeroReg <= False; - isNormalStateReg <= True; + isFirstPktReg <= True; + remainingPktNumReg <= 0; // $display( // "time=%0t: reset and clear mkPayloadGenerator", $time, @@ -1716,89 +1734,54 @@ module mkPayloadGenerator#( // ); endrule - // rule debugNotFull if (!( - // payloadGenRespQ.notFull && - // pendingGenReqQ.notFull && - // // pendingGenRespQ.notFull && - // // payloadSegmentQ.notFull && - // payloadBufQ.notFull - // )); - // $display( - // "time=%0t: mkPayloadGenerator debugNotFull", $time, - // ", qpn=%h", cntrlStatus.comm.getSQPN, - // ", isSQ=", fshow(cntrlStatus.isSQ), - // ", payloadGenReqQ.notEmpty=", fshow(payloadGenReqQ.notEmpty), - // ", payloadGenRespQ.notFull=", fshow(payloadGenRespQ.notFull), - // ", pendingGenReqQ.notFull=", fshow(pendingGenReqQ.notFull), - // // ", pendingGenRespQ.notFull=", fshow(pendingGenRespQ.notFull), - // // ", payloadSegmentQ.notFull=", fshow(payloadSegmentQ.notFull), - // ", payloadBufQ.notFull=", fshow(payloadBufQ.notFull) - // ); - // endrule - - // rule debugNotEmpty if (!( - // payloadGenRespQ.notEmpty && - // pendingGenReqQ.notEmpty && - // // pendingGenRespQ.notEmpty && - // // payloadSegmentQ.notEmpty && - // payloadBufQ.notEmpty - // )); - // $display( - // "time=%0t: mkPayloadGenerator debugNotEmpty", $time, - // ", qpn=%h", cntrlStatus.comm.getSQPN, - // ", isSQ=", fshow(cntrlStatus.isSQ), - // ", payloadGenReqQ.notEmpty=", fshow(payloadGenReqQ.notEmpty), - // ", payloadGenRespQ.notEmpty=", fshow(payloadGenRespQ.notEmpty), - // ", pendingGenReqQ.notEmpty=", fshow(pendingGenReqQ.notEmpty), - // // ", pendingGenRespQ.notEmpty=", fshow(pendingGenRespQ.notEmpty), - // // ", payloadSegmentQ.notEmpty=", fshow(payloadSegmentQ.notEmpty), - // ", payloadBufQ.notEmpty=", fshow(payloadBufQ.notEmpty) - // ); - // endrule - rule recvReq if (!clearAll); let payloadGenReq = payloadGenReqQ.first; payloadGenReqQ.deq; - let wqe = payloadGenReq.wqe; let sglIdx = 0; - let sge = wqe.sgl[sglIdx]; - // If first SGE has zero length, then whole WQE has no payload + let sge = payloadGenReq.sgl[sglIdx]; + // If first SGE has zero length, then whole SGL has no payload let isZeroPayloadLen = isZeroR(sge.len); - if (isZeroPayloadLen) begin - immAssert( - sge.isLast, - "last SGE assertion @ mkDmaReadCntrl", - $format( - "sge.isLast=", fshow(sge.isLast), - " should be true when sglIdx=%d", sglIdx, - ", and sge.len=%d", sge.len - ) - ); - end + // if (isZeroPayloadLen) begin + // immAssert( + // sge.isLast, + // "last SGE assertion @ mkDmaReadCntrl", + // $format( + // "sge.isLast=", fshow(sge.isLast), + // " should be true when sglIdx=%d", sglIdx, + // ", and sge.len=%d", sge.len + // ) + // ); + // end if (!isZeroPayloadLen) begin let dmaReadCntrlReq = DmaReadCntrlReq { sglDmaReadMetaData: DmaReadMetaDataSGL { - sgl : wqe.sgl, - sqpn : wqe.sqpn, - wrID : wqe.id + sgl : payloadGenReq.sgl, + sqpn : payloadGenReq.sqpn, + wrID : payloadGenReq.wrID }, - pmtu : wqe.pmtu + pmtu : payloadGenReq.pmtu }; dmaReadCntrl.srvPort.request.put(dmaReadCntrlReq); end adjustReqPktLenQ.enq(tuple2(payloadGenReq, isZeroPayloadLen)); - $display( - "time=%0t: mkPayloadGenerator recvReq", $time, - ", payloadGenReq=", fshow(payloadGenReq) - ); + // $display( + // "time=%0t: mkPayloadGenerator recvReq", $time, + // ", payloadGenReq=", fshow(payloadGenReq) + // ); + endrule + + rule recvDmaReadCntrlResp if (!clearAll); + let dmaReadCntrlResp <- dmaReadCntrl.srvPort.response.get; + sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); + // TODO: handle DMA read error + let hasDmaRespErr = dmaReadCntrlResp.dmaReadResp.isRespErr; endrule rule adjustFirstAndLastPktLen if (!clearAll); let { payloadGenReq, isZeroPayloadLen } = adjustReqPktLenQ.first; adjustReqPktLenQ.deq; - let wqe = payloadGenReq.wqe; let totalLen = 0; if (!isZeroPayloadLen) begin @@ -1807,38 +1790,40 @@ module mkPayloadGenerator#( totalLen = sglTotalPayloadLenMetaData.totalLen; end - let { truncatedPktNum, residue } = truncateLenByPMTU(totalLen, wqe.pmtu); + let { truncatedPktNum, residue } = truncateLenByPMTU(totalLen, payloadGenReq.pmtu); + let origPktNum = truncatedPktNum + (isZeroR(residue) ? 0 : 1); + let { pmtuLen, firstPktLen, lastPktLen, totalPktNum, secondChunkStartAddr //, isSinglePkt } = calcPktNumAndPktLenByAddrAndPMTU( - wqe.raddr, totalLen, wqe.pmtu + payloadGenReq.raddr, totalLen, payloadGenReq.pmtu ); adjustedFirstAndLastPktLenQ.enq(tuple8( - payloadGenReq, totalLen, truncatedPktNum, residue, - wqe.pmtu, firstPktLen, totalPktNum, isZeroPayloadLen + payloadGenReq, totalLen, totalPktNum, origPktNum, + firstPktLen, lastPktLen, pmtuLen, isZeroPayloadLen )); - $display( - "time=%0t: mkPayloadGenerator adjustFirstAndLastPktLen", $time, - ", payloadGenReq=", fshow(payloadGenReq) - ); + // $display( + // "time=%0t: mkPayloadGenerator adjustFirstAndLastPktLen", $time, + // ", payloadGenReq=", fshow(payloadGenReq) + // ); endrule - rule calcDetailTotalPayloadMetaData if (!clearAll); + rule calcAdjustedTotalPayloadMetaData if (!clearAll); let { - payloadGenReq, totalLen, truncatedPktNum, residue, - pmtu, firstPktLen, totalPktNum, isZeroPayloadLen + payloadGenReq, totalLen, totalPktNum, origPktNum, + firstPktLen, lastPktLen, pmtuLen, isZeroPayloadLen } = adjustedFirstAndLastPktLenQ.first; adjustedFirstAndLastPktLenQ.deq; let origLastFragValidByteNum = calcLastFragValidByteNum(totalLen); let firstPktLastFragValidByteNum = calcLastFragValidByteNum(firstPktLen); - // let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLen); + let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLen); let firstPktFragNum = calcFragNumByPktLen(firstPktLen); // let lastPktFragNum = calcFragNumByPktLen(lastPktLen); - // let firstPktPadCnt = calcPadCnt(firstPktLen); - // let lastPktPadCnt = calcPadCnt(lastPktLen); - let origPktNum = truncatedPktNum + (isZeroR(residue) ? 0 : 1); + let firstPktPadCnt = calcPadCnt(firstPktLen); + let lastPktPadCnt = calcPadCnt(lastPktLen); + let isOnlyPkt = isLessOrEqOneR(totalPktNum); let adjustedTotalPayloadMetaData = AdjustedTotalPayloadMetaData { firstPktLen : firstPktLen, @@ -1852,18 +1837,165 @@ module mkPayloadGenerator#( origLastFragValidByteNum : origLastFragValidByteNum, adjustedPktNum : totalPktNum, origPktNum : origPktNum, - pmtu : pmtu + pmtu : payloadGenReq.pmtu // totalLen : totalLen }; if (!isZeroPayloadLen) begin - adjustedTotalPayloadMetaDataQ.enq(tuple2( - payloadGenReq, adjustedTotalPayloadMetaData - )); + adjustedTotalPayloadMetaDataQ.enq(adjustedTotalPayloadMetaData); end - $display( - "time=%0t: mkPayloadGenerator calcDetailTotalPayloadMetaData", $time, - ", payloadGenReq=", fshow(payloadGenReq) + + let paddingMetaData = PaddingMetaData { + remoteAddr : payloadGenReq.raddr, + firstPktLen : firstPktLen, + lastPktLen : lastPktLen, + pmtuLen : pmtuLen, + firstPktLastFragValidByteNum: firstPktLastFragValidByteNum, + firstPktPadCnt : firstPktPadCnt, + lastPktLastFragValidByteNum : lastPktLastFragValidByteNum, + lastPktPadCnt : lastPktPadCnt, + totalPktNum : totalPktNum, + isOnlyPkt : isOnlyPkt, + isZeroPayloadLen : isZeroPayloadLen + }; + + addPadCntQ.enq(paddingMetaData); + // $display( + // "time=%0t: mkPayloadGenerator calcAdjustedTotalPayloadMetaData", $time, + // ", adjustedTotalPayloadMetaData=", fshow(adjustedTotalPayloadMetaData) + // ); + endrule + + rule genRespAndAddPadding if (!clearAll); + let paddingMetaData = addPadCntQ.first; + let remoteAddr = paddingMetaData.remoteAddr; + let firstPktLen = paddingMetaData.firstPktLen; + let lastPktLen = paddingMetaData.lastPktLen; + let pmtuLen = paddingMetaData.pmtuLen; + let firstPktLastFragValidByteNum = paddingMetaData.firstPktLastFragValidByteNum; + let firstPktPadCnt = paddingMetaData.firstPktPadCnt; + let lastPktLastFragValidByteNum = paddingMetaData.lastPktLastFragValidByteNum; + let lastPktPadCnt = paddingMetaData.lastPktPadCnt; + let totalPktNum = paddingMetaData.totalPktNum; + let isOnlyPkt = paddingMetaData.isOnlyPkt; + let isZeroPayloadLen = paddingMetaData.isZeroPayloadLen; + + let firstPktLastFragValidByteNumWithPadding = firstPktLastFragValidByteNum + zeroExtend(firstPktPadCnt); + let lastPktLastFragValidByteNumWithPadding = lastPktLastFragValidByteNum + zeroExtend(lastPktPadCnt); + let firstPktLastFragByteEnWithPadding = genByteEn(firstPktLastFragValidByteNumWithPadding); + let lastPktLastFragByteEnWithPadding = genByteEn(lastPktLastFragValidByteNumWithPadding); + immAssert( + (fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)) - zeroExtend(firstPktPadCnt) >= firstPktLastFragValidByteNum) && + (fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)) - zeroExtend(lastPktPadCnt) >= lastPktLastFragValidByteNum), + "zero SGE assertion @ mkPayloadGenerator", + $format( + "firstPktLastFragValidByteNum=%d", firstPktLastFragValidByteNum, + " + firstPktPadCnt=%d", firstPktPadCnt, + " should not > DATA_BUS_BYTE_WIDTH=%d", valueOf(DATA_BUS_BYTE_WIDTH), + ", and lastPktLastFragValidByteNum=%d", lastPktLastFragValidByteNum, + " + lastPktPadCnt=%d", lastPktPadCnt, + " should not > DATA_BUS_BYTE_WIDTH=%d", valueOf(DATA_BUS_BYTE_WIDTH) + ) ); + + let remainingPktNum = remainingPktNumReg; + if (isFirstPktReg) begin + if (isOnlyPkt) begin + remainingPktNum = 0; + end + else begin + remainingPktNum = totalPktNum - 2; + end + end + else begin + remainingPktNum = remainingPktNumReg - 1; + end + + let isFirstPkt = isFirstPktReg; + let isLastPkt = isOnlyPkt || (!isFirstPktReg && isZeroR(remainingPktNumReg)); + let pktLen = isFirstPkt ? firstPktLen : (isLastPkt ? lastPktLen : pmtuLen); + let padCnt = isFirstPkt ? firstPktPadCnt : (isLastPkt ? lastPktPadCnt : 0); + + if (isZeroPayloadLen) begin + isFirstPktReg <= True; + addPadCntQ.deq; + + let payloadGenResp = PayloadGenRespSG { + raddr : remoteAddr, + pktLen : 0, + padCnt : 0, + isZeroPayloadLen: True, + isFirst : True, + isLast : True, + isRespErr : False + }; + payloadGenRespQ.enq(payloadGenResp); + end + else begin + let curPayloadFrag = adjustedPayloadPipeOut.first; + adjustedPayloadPipeOut.deq; + + let isFirstPktLastFrag = curPayloadFrag.isLast && isFirstPkt; + let isLastPktLastFrag = curPayloadFrag.isLast && isLastPkt; + + if (isLastPktLastFrag) begin + addPadCntQ.deq; + + if (shouldAddPadding) begin + curPayloadFrag.byteEn = lastPktLastFragByteEnWithPadding; + end + end + if (isFirstPktLastFrag) begin + if (shouldAddPadding) begin + curPayloadFrag.byteEn = firstPktLastFragByteEnWithPadding; + end + end + + if (curPayloadFrag.isLast) begin + isFirstPktReg <= isLastPktLastFrag; + remainingPktNumReg <= remainingPktNum; + end + // Every segmented payload has a payloadGenResp + if (curPayloadFrag.isFirst) begin + let payloadGenResp = PayloadGenRespSG { + // adjustedPktMetaData : AdjustedPktMetaData { + raddr : remoteAddr, + pktLen : pktLen, + padCnt : padCnt, + isZeroPayloadLen: False, + isFirst : isFirstPkt, + isLast : isLastPkt, + isRespErr : False + }; + + payloadGenRespQ.enq(payloadGenResp); + // $display( + // "time=%0t: genRespAndAddPadding", $time, + // ", payloadGenResp=", fshow(payloadGenResp) + // ); + end + + payloadBufQ.enq(curPayloadFrag); + // $display( + // "time=%0t: genRespAndAddPadding", $time, + // ", remainingPktNumReg=%0d", remainingPktNumReg, + // ", isFirstPktReg=", fshow(isFirstPktReg), + // ", isOnlyPkt=", fshow(isOnlyPkt), + // ", isFirstPktLastFrag=", fshow(isFirstPktLastFrag), + // ", isLastPktLastFrag=", fshow(isLastPktLastFrag), + // ", isFirstPkt=", fshow(isFirstPkt), + // ", isLastPkt=", fshow(isLastPkt), + // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), + // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast) + // ); + end + + // $display( + // "time=%0t: PayloadGenerator genRespAndAddPadding", $time, + // ", isZeroPayloadLen=", fshow(isZeroPayloadLen), + // ", isFirstPkt=", fshow(isFirstPkt), + // ", isLastPkt=", fshow(isLastPkt), + // ", remainingPktNum=%0d", remainingPktNum + // ); endrule interface srvPort = toGPServer(payloadGenReqQ, payloadGenRespQ); diff --git a/test/TestPayloadGen.bsv b/test/TestPayloadGen.bsv index 4394307..2121a43 100644 --- a/test/TestPayloadGen.bsv +++ b/test/TestPayloadGen.bsv @@ -237,42 +237,6 @@ module mkTestAddrChunkSrv(Empty); ); // $display("time=%0t: checkPktMetaDataSGE", $time); endrule -/* - rule checkPktMetaDataSGE; - let sgePktMetaData = dut.sgePktMetaDataPipeOut.first; - dut.sgePktMetaDataPipeOut.deq; - let sgePktMetaDataRef = sgePktMetaDataRefQ.first; - sgePktMetaDataRefQ.deq; - - immAssert( - sgePktMetaData.curPktLastFragValidByteNum == sgePktMetaDataRef.curPktLastFragValidByteNum && - sgePktMetaData.pktLen == sgePktMetaDataRef.pktLen && - sgePktMetaData.pmtu == sgePktMetaDataRef.pmtu && - sgePktMetaData.isFirst == sgePktMetaDataRef.isFirst && - sgePktMetaData.isLast == sgePktMetaDataRef.isLast && - sgePktMetaData.isFirstSGE == sgePktMetaDataRef.isFirstSGE && - sgePktMetaData.isLastSGE == sgePktMetaDataRef.isLastSGE, - "sgePktMetaData assertion @ mkTestAddrChunkSrv", - $format( - "sgePktMetaData.curPktLastFragValidByteNum=%h should == sgePktMetaDataRef.curPktLastFragValidByteNum=%h", - sgePktMetaData.curPktLastFragValidByteNum, sgePktMetaDataRef.curPktLastFragValidByteNum, - "sgePktMetaData.pktLen=%0d should == sgePktMetaDataRef.pktLen=%0d", - sgePktMetaData.pktLen, sgePktMetaDataRef.pktLen, - "sgePktMetaData.pmtu=", fshow(sgePktMetaData.pmtu), - " should == sgePktMetaData.pmtu=", fshow(sgePktMetaData.pmtu), - "sgePktMetaData.isFirst=", fshow(sgePktMetaData.isFirst), - " should == sgePktMetaData.isFirst=", fshow(sgePktMetaData.isFirst), - "sgePktMetaData.isLast=", fshow(sgePktMetaData.isLast), - " should == sgePktMetaData.isLast=", fshow(sgePktMetaData.isLast), - "sgePktMetaData.isFirstSGE=", fshow(sgePktMetaData.isFirstSGE), - " should == sgePktMetaData.isFirstSGE=", fshow(sgePktMetaData.isFirstSGE), - "sgePktMetaData.isLastSGE=", fshow(sgePktMetaData.isLastSGE), - " should == sgePktMetaData.isLastSGE=", fshow(sgePktMetaData.isLastSGE) - ) - ); - // $display("time=%0t: checkPktMetaDataSGE", $time); - endrule -*/ endmodule (* doc = "testcase" *) @@ -347,71 +311,7 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); let pmtu = pmtuPipeOut.first; pmtuPipeOut.deq; -/* - let len = payloadLen; - let oneAsPSN = 1; - let pmtuAlignedStartAddr = alignAddrByPMTU(startAddr, pmtu); - let secondChunkStartAddr = addrAddPsnMultiplyPMTU(pmtuAlignedStartAddr, oneAsPSN, pmtu); - let pmtuLen = calcPmtuLen(pmtu); - - Tuple4#(PktLen, PktNum, PktLen, PktLen) tmpTuple = case (pmtu) - IBV_MTU_256 : begin - Bit#(8) addrLowPart = truncate(startAddr); // [7 : 0] - Bit#(8) lenLowPart = truncate(len); - Bit#(8) pmtuMask = maxBound; - Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 8)) truncatedLen = truncateLSB(len); - tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); - end - IBV_MTU_512 : begin - Bit#(9) addrLowPart = truncate(startAddr); // [8 : 0] - Bit#(9) lenLowPart = truncate(len); - Bit#(9) pmtuMask = maxBound; - Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 9)) truncatedLen = truncateLSB(len); - tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); - end - IBV_MTU_1024: begin - Bit#(10) addrLowPart = truncate(startAddr); // [9 : 0] - Bit#(10) lenLowPart = truncate(len); - Bit#(10) pmtuMask = maxBound; - Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 10)) truncatedLen = truncateLSB(len); - tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); - end - IBV_MTU_2048: begin - Bit#(11) addrLowPart = truncate(startAddr); // [10 : 0] - Bit#(11) lenLowPart = truncate(len); - Bit#(11) pmtuMask = maxBound; - Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 11)) truncatedLen = truncateLSB(len); - tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); - end - IBV_MTU_4096: begin - Bit#(12) addrLowPart = truncate(startAddr); // [11 : 0] - Bit#(12) lenLowPart = truncate(len); - Bit#(12) pmtuMask = maxBound; - Bit#(TSub#(RDMA_MAX_LEN_WIDTH, 12)) truncatedLen = truncateLSB(len); - tuple4(zeroExtend(pmtuMask), zeroExtend(truncatedLen), zeroExtend(addrLowPart), zeroExtend(lenLowPart)); - end - endcase; - - let { pmtuMask, truncatedPktNum, addrLowPart, lenLowPart } = tmpTuple; - let maxFirstPktLen = pmtuLen - addrLowPart; - let tmpSum = addrLowPart + lenLowPart; - ResiduePMTU residue = truncateByPMTU(tmpSum, pmtu); - PktLen tmpLastPktLen = zeroExtend(residue); - - let pmtuInvMask = ~pmtuMask; - let residuePktNum = |(pmtuMask & tmpSum); - let extraPktNum = |(pmtuInvMask & tmpSum); - Bool hasResidue = unpack(residuePktNum); - Bool hasExtraPkt = unpack(extraPktNum); - let notFullPkt = isZeroR(truncatedPktNum); - - let sgePktNum = truncatedPktNum + zeroExtend(residuePktNum) + zeroExtend(extraPktNum); - // let firstPktLen = notFullPkt ? (hasExtraPkt ? maxFirstPktLen : lenLowPart) : maxFirstPktLen; - let firstPktLen = (notFullPkt && !hasExtraPkt) ? lenLowPart : maxFirstPktLen; - // let lastPktLen = notFullPkt ? (hasResidue ? tmpLastPktLen : 0) : (hasResidue ? tmpLastPktLen : pmtuLen); - let lastPktLen = notFullPkt ? (hasExtraPkt ? tmpLastPktLen : lenLowPart) : (hasResidue ? tmpLastPktLen : pmtuLen); - // let isSinglePkt = isLessOrEqOneR(totalPktNum); -*/ + let { pmtuLen, firstPktLen, lastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt } = calcPktNumAndPktLenByAddrAndPMTU(startAddr, payloadLen, pmtu); @@ -624,6 +524,8 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( rule clearAll if (stateReg == TEST_MERGE_EACH_SGE_INIT); clearReg <= False; + sglIdxReg <= 0; + remainingFragNumReg <= 0; stateReg <= TEST_MERGE_EACH_SGE_PREPARE; // $display("time=%0t: clearAll", $time); @@ -748,6 +650,19 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( end end + if (isZero(remainingFragNumReg)) begin + immAssert( + payloadFrag.isFirst || payloadFrag.isLast, + "payloadFrag isFirst isLast assertion @ mkTestMergeNormalOrSmallPayloadEachSGE", + $format( + "payloadFrag.isFirst=", fshow(payloadFrag.isFirst), + " or payloadFrag.isLast=", fshow(payloadFrag.isLast), + " should be true when remainingFragNumReg=%0d", + remainingFragNumReg + ) + ); + end + if (payloadFrag.isFirst) begin immAssert( isZero(remainingFragNumReg), @@ -882,6 +797,8 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( rule clearAll if (stateReg == TEST_MERGE_ALL_SGE_INIT); clearReg <= False; + sglIdxReg <= 0; + remainingFragNumReg <= 0; stateReg <= TEST_MERGE_ALL_SGE_PREPARE; // $display("time=%0t: clearAll", $time); @@ -994,10 +911,10 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( rule recvDmaReadCntrlResp; let dmaReadCntrlResp <- dmaReadCntrl.srvPort.response.get; sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); - $display( - "time=%0t: recvDmaReadCntrlResp", $time, - ", dmaReadCntrlResp=", fshow(dmaReadCntrlResp) - ); + // $display( + // "time=%0t: recvDmaReadCntrlResp", $time, + // ", dmaReadCntrlResp=", fshow(dmaReadCntrlResp) + // ); endrule rule checkMergedPayloadSGL if (stateReg == TEST_MERGE_ALL_SGE_RUN); @@ -1014,6 +931,19 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( end end + if (isZero(remainingFragNumReg)) begin + immAssert( + payloadFrag.isFirst || payloadFrag.isLast, + "payloadFrag isFirst isLast assertion @ mkTestMergeNormalOrSmallPayloadAllSGE", + $format( + "payloadFrag.isFirst=", fshow(payloadFrag.isFirst), + " or payloadFrag.isLast=", fshow(payloadFrag.isLast), + " should be true when remainingFragNumReg=%0d", + remainingFragNumReg + ) + ); + end + if (payloadFrag.isFirst) begin immAssert( isZero(remainingFragNumReg), @@ -1120,7 +1050,7 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( FIFOF#(AdjustedTotalPayloadMetaData) adjustedTotalPayloadMetaDataQ <- mkFIFOF; FIFOF#(Tuple3#(AdjustedTotalPayloadMetaData, PktFragNum, ByteEnBitNum)) adjustedTotalPayloadMetaDataRefQ <- mkFIFOF; - PipeOut#(ADDR) remoteAddrPipeOut <- mkGenericRandomPipeOut; + PipeOut#(ADDR) remoteAddrPipeOut <- mkGenericRandomPipeOut; PipeOut#(ADDR) localAddrPipeOut <- mkGenericRandomPipeOut; let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); @@ -1147,9 +1077,11 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( rule clearAll if (stateReg == TEST_ADJUST_PAYLOAD_INIT); clearReg <= False; + sglIdxReg <= 0; + remainingFragNumReg <= 0; stateReg <= TEST_ADJUST_PAYLOAD_PREPARE; - $display("time=%0t: clearAll", $time); + // $display("time=%0t: clearAll", $time); endrule rule genSGE if (stateReg == TEST_ADJUST_PAYLOAD_PREPARE); @@ -1355,9 +1287,22 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( remainingFragNum = remainingFragNumReg - 1; end remainingFragNumReg <= remainingFragNum; - $display( - "time=%0t: checkAdjustedPayloadSegment", $time, - ", remainingFragNum=%0d", remainingFragNum + // $display( + // "time=%0t: checkAdjustedPayloadSegment", $time, + // ", remainingFragNum=%0d", remainingFragNum + // ); + end + + if (isZero(remainingFragNumReg)) begin + immAssert( + payloadFrag.isFirst || payloadFrag.isLast, + "payloadFrag isFirst isLast assertion @ mkTestAdjustNormalOrSmallPayloadSegment", + $format( + "payloadFrag.isFirst=", fshow(payloadFrag.isFirst), + " or payloadFrag.isLast=", fshow(payloadFrag.isLast), + " should be true when remainingFragNumReg=%0d", + remainingFragNumReg + ) ); end @@ -1433,3 +1378,266 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( // ); endrule endmodule + +(* doc = "testcase" *) +module mkTestPayloadGenNormalCase(Empty); + let minPayloadLen = 2048; + let maxPayloadLen = 8192; + let result <- mkTestNormalOrSmallPayloadGen(minPayloadLen, maxPayloadLen); +endmodule + +(* doc = "testcase" *) +module mkTestPayloadGenSmallCase(Empty); + let minPayloadLen = 1; + let maxPayloadLen = 7; + let result <- mkTestNormalOrSmallPayloadGen(minPayloadLen, maxPayloadLen); +endmodule + +(* doc = "testcase" *) +module mkTestPayloadGenZeroCase(Empty); + let minPayloadLen = 0; + let maxPayloadLen = 0; + let result <- mkTestNormalOrSmallPayloadGen(minPayloadLen, maxPayloadLen); +endmodule + +module mkTestNormalOrSmallPayloadGen#( + // Both min and max are inclusive + Length minPayloadLen, Length maxPayloadLen +)(Empty); + let qpType = IBV_QPT_XRC_SEND; + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + + Reg#(Bool) clearReg <- mkReg(True); + Reg#(IdxSGL) sglIdxReg <- mkReg(0); + Reg#(Length) totalLenReg <- mkRegU; + Vector#(MAX_SGE, Reg#(ScatterGatherElem)) sglRegVec <- replicateM(mkRegU); + + PipeOut#(ADDR) remoteAddrPipeOut <- mkGenericRandomPipeOut; + PipeOut#(ADDR) localAddrPipeOut <- mkGenericRandomPipeOut; + let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); + let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); + + let simDmaReadSrv <- mkSimDmaReadSrv; + let dmaReadCntrl <- mkDmaReadCntrl(clearReg, simDmaReadSrv); + + let shouldAddPadding = False; + let dut <- mkPayloadGenerator(clearReg, shouldAddPadding, dmaReadCntrl); + + Reg#(PayloadGenRespSG) payloadGenRespReg <- mkRegU; + Reg#(Bool) isFirstFragReg <- mkRegU; + Reg#(PktNum) remainingPktNumReg <- mkRegU; // TODO: remove it + Reg#(PktFragNum) remainingFragNumReg <- mkRegU; + Reg#(PktFragNum) firstPktFragNumReg <- mkRegU; + Reg#(PktFragNum) lastPktFragNumReg <- mkRegU; + Reg#(PktFragNum) pmtuFragNumReg <- mkRegU; + Reg#(TestAdjustPayloadState) stateReg <- mkReg(TEST_ADJUST_PAYLOAD_INIT); + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + rule clearAll if (stateReg == TEST_ADJUST_PAYLOAD_INIT); + clearReg <= False; + sglIdxReg <= 0; + + stateReg <= TEST_ADJUST_PAYLOAD_PREPARE; + // $display("time=%0t: clearAll", $time); + endrule + + rule genSGE if (stateReg == TEST_ADJUST_PAYLOAD_PREPARE); + let localAddr = localAddrPipeOut.first; + localAddrPipeOut.deq; + + let payloadLen = payloadLenPipeOut.first; + payloadLenPipeOut.deq; + + let isFirst = isZero(sglIdxReg); + let isLast = isAllOnesR(sglIdxReg); + let sge = ScatterGatherElem { + laddr : localAddr, + len : payloadLen, + lkey : dontCareValue, + isFirst: isFirst, + isLast : isLast + }; + sglRegVec[sglIdxReg] <= sge; + + let totalLen = totalLenReg; + if (isFirst) begin + totalLen = payloadLen; + end + else begin + totalLen = totalLenReg + payloadLen; + end + totalLenReg <= totalLen; + + if (isLast) begin + sglIdxReg <= 0; + stateReg <= TEST_ADJUST_PAYLOAD_ISSUE; + end + else begin + sglIdxReg <= sglIdxReg + 1; + end + + let { + sgeFragNum, sgeLastFragByteEn, sgeLastFragValidByteNum + } = calcTotalFragNumByLength(payloadLen); + countDown.decr; + // $display( + // "time=%0t: genSGE", $time, + // ", sglIdxReg=%0d", sglIdxReg, + // ", localAddr=%h", localAddr, + // ", payloadLen=%0d", payloadLen, + // ", sgeFragNum=%0d", sgeFragNum, + // ", sgeLastFragByteEn=%h", sgeLastFragByteEn, + // ", sgeLastFragValidByteNum=%0d", sgeLastFragValidByteNum + // ); + endrule + + rule issuePayloadGenReq if (stateReg == TEST_ADJUST_PAYLOAD_ISSUE); + let sgl = readVReg(sglRegVec); + + let pmtu = pmtuPipeOut.first; + pmtuPipeOut.deq; + let remoteAddr = remoteAddrPipeOut.first; + remoteAddrPipeOut.deq; + + let payloadGenReq = PayloadGenReqSG { + wrID : dontCareValue, + sqpn : getDefaultQPN, + sgl : sgl, + raddr: remoteAddr, + pmtu : pmtu + }; + dut.srvPort.request.put(payloadGenReq); + + let { + pmtuLen, firstPktLen, lastPktLen, totalPktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU(remoteAddr, totalLenReg, pmtu); + let firstPktFragNum = calcFragNumByPktLen(firstPktLen); + let lastPktFragNum = calcFragNumByPktLen(lastPktLen); + let pmtuFragNum = calcFragNumByPktLen(pmtuLen); + + firstPktFragNumReg <= firstPktFragNum; + lastPktFragNumReg <= lastPktFragNum; + pmtuFragNumReg <= pmtuFragNum; + remainingPktNumReg <= totalPktNum - 1; + remainingFragNumReg <= firstPktFragNum - 1; + isFirstFragReg <= True; + // $display( + // "time=%0t: issueDmaReadCntrlReq", $time, + // ", pmtu=", fshow(pmtu), + // ", totalLenReg=%0d", totalLenReg, + // ", totalPktNum=%0d", totalPktNum, + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen, + // ", firstPktFragNum=%0d", firstPktFragNum, + // ", lastPktFragNum=%0d", lastPktFragNum, + // ", pmtuFragNum=%0d", pmtuFragNum, + // ", remoteAddr=%h", remoteAddr + // ); + + stateReg <= TEST_ADJUST_PAYLOAD_RUN; + for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin + let sge = sglRegVec[idx]; + let { + pmtuPktLen, sgeFirstPktLen, sgeLastPktLen, sgePktNum, sgeSecondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU(sge.laddr, sge.len, pmtu); + let sgeFirstPktFragNum = calcFragNumByPktLen(sgeFirstPktLen); + let sgeLastPktFragNum = calcFragNumByPktLen(sgeLastPktLen); + + // $display( + // "time=%0t: issuePayloadGenReq", $time, + // ", SGE idx=%0d", idx, + // ", sge.laddr=%h", sge.laddr, + // ", sgePktNum=%0d", sgePktNum, + // ", sgeFirstPktLen=%0d", sgeFirstPktLen, + // ", sgeLastPktLen=%0d", sgeLastPktLen, + // ", pmtuPktLen=%0d", pmtuPktLen, + // ", sgeFirstPktFragNum=%0d", sgeFirstPktFragNum, + // ", sgeLastPktFragNum=%0d", sgeLastPktFragNum + // ); + end + endrule + + rule checkResp if (stateReg == TEST_ADJUST_PAYLOAD_RUN); + let payloadGenResp = payloadGenRespReg; + let isFirstFrag = isFirstFragReg; + + if (isFirstFrag) begin + payloadGenResp <- dut.srvPort.response.get; + payloadGenRespReg <= payloadGenResp; + end + let isFirstPkt = payloadGenResp.isFirst; + let isLastPkt = payloadGenResp.isLast; + + if (payloadGenResp.isZeroPayloadLen) begin + stateReg <= TEST_ADJUST_PAYLOAD_INIT; + isFirstFrag = True; + end + else begin + let remainingPktNum = remainingPktNumReg; + let remainingFragNum = remainingFragNumReg; + + let isSecondLastPkt = isOne(remainingPktNumReg); + + let payloadFrag = dut.payloadDataStreamPipeOut.first; + dut.payloadDataStreamPipeOut.deq; + let isFirstPktLastFrag = payloadFrag.isLast && isFirstPkt; + let isLastPktLastFrag = payloadFrag.isLast && isLastPkt; + isFirstFrag = payloadFrag.isLast; + + if (payloadFrag.isLast) begin + if (isLastPkt) begin + stateReg <= TEST_ADJUST_PAYLOAD_INIT; + end + else begin + remainingPktNum = remainingPktNumReg - 1; + + if (isSecondLastPkt) begin + remainingFragNum = lastPktFragNumReg - 1; + end + else begin + remainingFragNum = pmtuFragNumReg - 1; + end + end + + immAssert( + isZero(remainingFragNumReg), + "remainingFragNumReg assertion @ mkTestNormalOrSmallPayloadGen", + $format( + "remainingFragNumReg=%h", remainingFragNumReg, + " should be zero when payloadFrag.isLast=", + fshow(payloadFrag.isLast) + ) + ); + end + else begin + remainingFragNum = remainingFragNumReg - 1; + end + remainingPktNumReg <= remainingPktNum; + remainingFragNumReg <= remainingFragNum; + // $display( + // "time=%0t: checkResp, each payload fragment", $time, + // ", isFirstFragReg=", fshow(isFirstFragReg), + // ", isFirstPktLastFrag=", fshow(isFirstPktLastFrag), + // ", isLastPktLastFrag=", fshow(isLastPktLastFrag), + // ", isFirstPkt=", fshow(isFirstPkt), + // ", isLastPkt=", fshow(isLastPkt), + // ", remainingPktNumReg=%0d", remainingPktNumReg, + // ", remainingFragNumReg=%0d", remainingFragNumReg, + // ", payloadFrag.isFirst=", fshow(payloadFrag.isFirst), + // ", payloadFrag.isLast=", fshow(payloadFrag.isLast) + // ); + end + isFirstFragReg <= isFirstFrag; + + // $display( + // "time=%0t: checkResp", $time, + // ", isZeroPayloadLen=", fshow(payloadGenResp.isZeroPayloadLen) + // ); + endrule +endmodule From c15b39241067e059ec749eac0300ea69dab4c850 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Thu, 25 Jan 2024 03:10:11 +0800 Subject: [PATCH 10/16] PayloadGenerator output ReqGenTotalMetaData --- src/PayloadGen.bsv | 130 +++++++++++++++++++++++++++++----------- test/TestPayloadGen.bsv | 68 +++++++++++++++++---- 2 files changed, 152 insertions(+), 46 deletions(-) diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index d7afbdb..afd9a9f 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -1657,29 +1657,51 @@ typedef struct { ADDR raddr; PktLen pktLen; PAD padCnt; - // PMTU pmtu; - Bool isZeroPayloadLen; Bool isFirst; Bool isLast; - Bool isRespErr; // TODO: handle DMA read error + Bool isRespErr; // TODO: handle DMA read error } PayloadGenRespSG deriving(Bits, FShow); typedef struct { - ADDR remoteAddr; + Length totalLen; + PktNum totalPktNum; + PMTU pmtu; + Bool isOnlyPkt; + Bool isZeroPayloadLen; +} ReqGenTotalMetaData deriving(Bits, FShow); + +typedef struct { + ADDR firstRemoteAddr; + ADDR secondRemoteAddr; + Length totalLen; + PktNum totalPktNum; + PktNum origPktNum; PktLen firstPktLen; PktLen lastPktLen; PktLen pmtuLen; + PMTU pmtu; + Bool isZeroPayloadLen; +} TmpAdjustMetaData deriving(Bits); + +typedef struct { + ADDR firstRemoteAddr; + ADDR secondRemoteAddr; + PktLen firstPktLen; + PktLen lastPktLen; + PktLen pmtuLen; ByteEnBitNum firstPktLastFragValidByteNum; - PAD firstPktPadCnt; + PAD firstPktPadCnt; ByteEnBitNum lastPktLastFragValidByteNum; - PAD lastPktPadCnt; - PktNum totalPktNum; - Bool isOnlyPkt; - Bool isZeroPayloadLen; -} PaddingMetaData deriving(Bits, FShow); + PAD lastPktPadCnt; + PktNum totalPktNum; + PMTU pmtu; + Bool isOnlyPkt; + Bool isZeroPayloadLen; +} TmpPaddingMetaData deriving(Bits); interface PayloadGenerator; interface Server#(PayloadGenReqSG, PayloadGenRespSG) srvPort; + interface PipeOut#(ReqGenTotalMetaData) reqGenTotalMetaDataPipeOut; interface DataStreamPipeOut payloadDataStreamPipeOut; method Bool payloadNotEmpty(); endinterface @@ -1687,14 +1709,15 @@ endinterface module mkPayloadGenerator#( Bool clearAll, Bool shouldAddPadding, DmaReadCntrl dmaReadCntrl )(PayloadGenerator); - FIFOF#(PayloadGenReqSG) payloadGenReqQ <- mkFIFOF; - FIFOF#(PayloadGenRespSG) payloadGenRespQ <- mkFIFOF; + FIFOF#(PayloadGenReqSG) payloadGenReqQ <- mkFIFOF; + FIFOF#(PayloadGenRespSG) payloadGenRespQ <- mkFIFOF; + FIFOF#(ReqGenTotalMetaData) reqGenTotalMetaDataQ <- mkFIFOF; // Pipeline FIFO FIFOF#(DataStream) sgePayloadOutQ <- mkFIFOF; FIFOF#(Tuple2#(PayloadGenReqSG, Bool)) adjustReqPktLenQ <- mkFIFOF; - FIFOF#(Tuple8#(PayloadGenReqSG, Length, PktNum, PktNum, PktLen, PktLen, PktLen, Bool)) adjustedFirstAndLastPktLenQ <- mkFIFOF; - FIFOF#(PaddingMetaData) addPadCntQ <- mkFIFOF; + FIFOF#(TmpAdjustMetaData) adjustedFirstAndLastPktLenQ <- mkFIFOF; + FIFOF#(TmpPaddingMetaData) addPadCntQ <- mkFIFOF; FIFOF#(AdjustedTotalPayloadMetaData) adjustedTotalPayloadMetaDataQ <- mkFIFOF; let sgeMergedPayloadPipeOut <- mkMergePayloadEachSGE( @@ -1712,8 +1735,9 @@ module mkPayloadGenerator#( let bramQ2PipeOut <- mkConnectBramQ2PipeOut(payloadBufQ); let payloadBufPipeOut = bramQ2PipeOut.pipeOut; - Reg#(Bool) isFirstPktReg <- mkReg(True); // TODO: remove it - Reg#(PktNum) remainingPktNumReg <- mkReg(0); // TODO: remove it + Reg#(ADDR) pktRemoteAddrReg <- mkRegU; + Reg#(Bool) isFirstPktReg <- mkReg(True); + Reg#(PktNum) remainingPktNumReg <- mkReg(0); rule resetAndClear if (clearAll); payloadGenReqQ.clear; @@ -1799,10 +1823,19 @@ module mkPayloadGenerator#( payloadGenReq.raddr, totalLen, payloadGenReq.pmtu ); - adjustedFirstAndLastPktLenQ.enq(tuple8( - payloadGenReq, totalLen, totalPktNum, origPktNum, - firstPktLen, lastPktLen, pmtuLen, isZeroPayloadLen - )); + let adjustMetaData = TmpAdjustMetaData { + firstRemoteAddr : payloadGenReq.raddr, + secondRemoteAddr: secondChunkStartAddr, + totalLen : totalLen, + totalPktNum : totalPktNum, + origPktNum : origPktNum, + firstPktLen : firstPktLen, + lastPktLen : lastPktLen, + pmtuLen : pmtuLen, + pmtu : payloadGenReq.pmtu, + isZeroPayloadLen: isZeroPayloadLen + }; + adjustedFirstAndLastPktLenQ.enq(adjustMetaData); // $display( // "time=%0t: mkPayloadGenerator adjustFirstAndLastPktLen", $time, // ", payloadGenReq=", fshow(payloadGenReq) @@ -1810,12 +1843,20 @@ module mkPayloadGenerator#( endrule rule calcAdjustedTotalPayloadMetaData if (!clearAll); - let { - payloadGenReq, totalLen, totalPktNum, origPktNum, - firstPktLen, lastPktLen, pmtuLen, isZeroPayloadLen - } = adjustedFirstAndLastPktLenQ.first; + let adjustMetaData = adjustedFirstAndLastPktLenQ.first; adjustedFirstAndLastPktLenQ.deq; + let firstRemoteAddr = adjustMetaData.firstRemoteAddr; + let secondRemoteAddr = adjustMetaData.secondRemoteAddr; + let totalLen = adjustMetaData.totalLen; + let totalPktNum = adjustMetaData.totalPktNum; + let origPktNum = adjustMetaData.origPktNum; + let firstPktLen = adjustMetaData.firstPktLen; + let lastPktLen = adjustMetaData.lastPktLen; + let pmtuLen = adjustMetaData.pmtuLen; + let pmtu = adjustMetaData.pmtu; + let isZeroPayloadLen = adjustMetaData.isZeroPayloadLen; + let origLastFragValidByteNum = calcLastFragValidByteNum(totalLen); let firstPktLastFragValidByteNum = calcLastFragValidByteNum(firstPktLen); let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLen); @@ -1837,15 +1878,16 @@ module mkPayloadGenerator#( origLastFragValidByteNum : origLastFragValidByteNum, adjustedPktNum : totalPktNum, origPktNum : origPktNum, - pmtu : payloadGenReq.pmtu + pmtu : pmtu // totalLen : totalLen }; if (!isZeroPayloadLen) begin adjustedTotalPayloadMetaDataQ.enq(adjustedTotalPayloadMetaData); end - let paddingMetaData = PaddingMetaData { - remoteAddr : payloadGenReq.raddr, + let paddingMetaData = TmpPaddingMetaData { + firstRemoteAddr : firstRemoteAddr, + secondRemoteAddr : secondRemoteAddr, firstPktLen : firstPktLen, lastPktLen : lastPktLen, pmtuLen : pmtuLen, @@ -1854,11 +1896,20 @@ module mkPayloadGenerator#( lastPktLastFragValidByteNum : lastPktLastFragValidByteNum, lastPktPadCnt : lastPktPadCnt, totalPktNum : totalPktNum, + pmtu : pmtu, isOnlyPkt : isOnlyPkt, isZeroPayloadLen : isZeroPayloadLen }; - addPadCntQ.enq(paddingMetaData); + + let reqGenTotalMetaData = ReqGenTotalMetaData { + totalLen : totalLen, + totalPktNum : totalPktNum, + pmtu : pmtu, + isOnlyPkt : isOnlyPkt, + isZeroPayloadLen: isZeroPayloadLen + }; + reqGenTotalMetaDataQ.enq(reqGenTotalMetaData); // $display( // "time=%0t: mkPayloadGenerator calcAdjustedTotalPayloadMetaData", $time, // ", adjustedTotalPayloadMetaData=", fshow(adjustedTotalPayloadMetaData) @@ -1867,7 +1918,9 @@ module mkPayloadGenerator#( rule genRespAndAddPadding if (!clearAll); let paddingMetaData = addPadCntQ.first; - let remoteAddr = paddingMetaData.remoteAddr; + + let firstRemoteAddr = paddingMetaData.firstRemoteAddr; + let secondRemoteAddr = paddingMetaData.secondRemoteAddr; let firstPktLen = paddingMetaData.firstPktLen; let lastPktLen = paddingMetaData.lastPktLen; let pmtuLen = paddingMetaData.pmtuLen; @@ -1876,6 +1929,7 @@ module mkPayloadGenerator#( let lastPktLastFragValidByteNum = paddingMetaData.lastPktLastFragValidByteNum; let lastPktPadCnt = paddingMetaData.lastPktPadCnt; let totalPktNum = paddingMetaData.totalPktNum; + let pmtu = paddingMetaData.pmtu; let isOnlyPkt = paddingMetaData.isOnlyPkt; let isZeroPayloadLen = paddingMetaData.isZeroPayloadLen; @@ -1896,9 +1950,13 @@ module mkPayloadGenerator#( " should not > DATA_BUS_BYTE_WIDTH=%d", valueOf(DATA_BUS_BYTE_WIDTH) ) ); - + let oneAsPSN = 1; + let remoteAddr = firstRemoteAddr; + let nextRemoteAddr = pktRemoteAddrReg; let remainingPktNum = remainingPktNumReg; if (isFirstPktReg) begin + nextRemoteAddr = secondRemoteAddr; + if (isOnlyPkt) begin remainingPktNum = 0; end @@ -1907,6 +1965,10 @@ module mkPayloadGenerator#( end end else begin + remoteAddr = pktRemoteAddrReg; + nextRemoteAddr = addrAddPsnMultiplyPMTU( + pktRemoteAddrReg, oneAsPSN, pmtu + ); remainingPktNum = remainingPktNumReg - 1; end @@ -1923,7 +1985,6 @@ module mkPayloadGenerator#( raddr : remoteAddr, pktLen : 0, padCnt : 0, - isZeroPayloadLen: True, isFirst : True, isLast : True, isRespErr : False @@ -1951,22 +2012,20 @@ module mkPayloadGenerator#( end if (curPayloadFrag.isLast) begin - isFirstPktReg <= isLastPktLastFrag; + isFirstPktReg <= isLastPktLastFrag; + pktRemoteAddrReg <= nextRemoteAddr; remainingPktNumReg <= remainingPktNum; end // Every segmented payload has a payloadGenResp if (curPayloadFrag.isFirst) begin let payloadGenResp = PayloadGenRespSG { - // adjustedPktMetaData : AdjustedPktMetaData { raddr : remoteAddr, pktLen : pktLen, padCnt : padCnt, - isZeroPayloadLen: False, isFirst : isFirstPkt, isLast : isLastPkt, isRespErr : False }; - payloadGenRespQ.enq(payloadGenResp); // $display( // "time=%0t: genRespAndAddPadding", $time, @@ -1999,6 +2058,7 @@ module mkPayloadGenerator#( endrule interface srvPort = toGPServer(payloadGenReqQ, payloadGenRespQ); + interface reqGenTotalMetaDataPipeOut = toPipeOut(reqGenTotalMetaDataQ); interface payloadDataStreamPipeOut = payloadBufPipeOut; method Bool payloadNotEmpty() = bramQ2PipeOut.notEmpty; endmodule diff --git a/test/TestPayloadGen.bsv b/test/TestPayloadGen.bsv index 2121a43..79317d1 100644 --- a/test/TestPayloadGen.bsv +++ b/test/TestPayloadGen.bsv @@ -1172,7 +1172,7 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); endrule - rule adjustTotalPayloadMetaData if (stateReg == TEST_ADJUST_PAYLOAD_RECV); + rule recvTotalPayloadLenMetaData if (stateReg == TEST_ADJUST_PAYLOAD_RECV); let remoteAddr = remoteAddrPipeOut.first; remoteAddrPipeOut.deq; @@ -1221,7 +1221,7 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( stateReg <= TEST_ADJUST_PAYLOAD_RUN; // $display( - // "time=%0t: adjustTotalPayloadMetaData", $time, + // "time=%0t: recvTotalPayloadLenMetaData", $time, // ", pmtu=", fshow(pmtu), // ", totalLen=%0d", totalLen, // ", firstPktLen=%0d", firstPktLen, @@ -1431,11 +1431,13 @@ module mkTestNormalOrSmallPayloadGen#( Reg#(PayloadGenRespSG) payloadGenRespReg <- mkRegU; Reg#(Bool) isFirstFragReg <- mkRegU; + Reg#(Bool) isZeroPayloadLenReg <- mkRegU; Reg#(PktNum) remainingPktNumReg <- mkRegU; // TODO: remove it Reg#(PktFragNum) remainingFragNumReg <- mkRegU; Reg#(PktFragNum) firstPktFragNumReg <- mkRegU; Reg#(PktFragNum) lastPktFragNumReg <- mkRegU; Reg#(PktFragNum) pmtuFragNumReg <- mkRegU; + Reg#(ADDR) expectedRemoteAddrReg <- mkRegU; Reg#(TestAdjustPayloadState) stateReg <- mkReg(TEST_ADJUST_PAYLOAD_INIT); let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); @@ -1521,12 +1523,13 @@ module mkTestNormalOrSmallPayloadGen#( let lastPktFragNum = calcFragNumByPktLen(lastPktLen); let pmtuFragNum = calcFragNumByPktLen(pmtuLen); - firstPktFragNumReg <= firstPktFragNum; - lastPktFragNumReg <= lastPktFragNum; - pmtuFragNumReg <= pmtuFragNum; - remainingPktNumReg <= totalPktNum - 1; - remainingFragNumReg <= firstPktFragNum - 1; - isFirstFragReg <= True; + expectedRemoteAddrReg <= remoteAddr; + firstPktFragNumReg <= firstPktFragNum; + lastPktFragNumReg <= lastPktFragNum; + pmtuFragNumReg <= pmtuFragNum; + remainingPktNumReg <= totalPktNum; + remainingFragNumReg <= firstPktFragNum - 1; + isFirstFragReg <= True; // $display( // "time=%0t: issueDmaReadCntrlReq", $time, // ", pmtu=", fshow(pmtu), @@ -1540,7 +1543,7 @@ module mkTestNormalOrSmallPayloadGen#( // ", remoteAddr=%h", remoteAddr // ); - stateReg <= TEST_ADJUST_PAYLOAD_RUN; + stateReg <= TEST_ADJUST_PAYLOAD_RECV; for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin let sge = sglRegVec[idx]; let { @@ -1563,6 +1566,34 @@ module mkTestNormalOrSmallPayloadGen#( end endrule + rule recvReqGenTotalMetaData if (stateReg == TEST_ADJUST_PAYLOAD_RECV); + let reqGenTotalMetaData = dut.reqGenTotalMetaDataPipeOut.first; + dut.reqGenTotalMetaDataPipeOut.deq; + + immAssert( + totalLenReg == reqGenTotalMetaData.totalLen, + "totalLen assertion @ mkTestNormalOrSmallPayloadGen", + $format( + "totalLenReg=%0d", totalLenReg, + " should == reqGenTotalMetaData.totalLen", + reqGenTotalMetaData.totalLen + ) + ); + immAssert( + remainingPktNumReg == reqGenTotalMetaData.totalPktNum, + "totalPktNum assertion @ mkTestNormalOrSmallPayloadGen", + $format( + "remainingPktNumReg=%0d", remainingPktNumReg, + " should == reqGenTotalMetaData.totalPktNum", + reqGenTotalMetaData.totalPktNum + ) + ); + isZeroPayloadLenReg <= reqGenTotalMetaData.isZeroPayloadLen; + remainingPktNumReg <= remainingPktNumReg - 1; + stateReg <= TEST_ADJUST_PAYLOAD_RUN; + // $display("time=%0t: recvReqGenTotalMetaData", $time); + endrule + rule checkResp if (stateReg == TEST_ADJUST_PAYLOAD_RUN); let payloadGenResp = payloadGenRespReg; let isFirstFrag = isFirstFragReg; @@ -1574,7 +1605,19 @@ module mkTestNormalOrSmallPayloadGen#( let isFirstPkt = payloadGenResp.isFirst; let isLastPkt = payloadGenResp.isLast; - if (payloadGenResp.isZeroPayloadLen) begin + let nextRemoteAddr = payloadGenResp.raddr + zeroExtend(payloadGenResp.pktLen); + immAssert( + expectedRemoteAddrReg == payloadGenResp.raddr, + "expectedRemoteAddrReg.raddr assertion @ mkTestNormalOrSmallPayloadGen", + $format( + "expectedRemoteAddrReg=%0d", expectedRemoteAddrReg, + " should == payloadGenResp.raddr=%0d", payloadGenResp.raddr, + " when remainingPktNumReg=%0d", remainingPktNumReg, + " and isFirstPkt=", fshow(isFirstPkt) + ) + ); + + if (isZeroPayloadLenReg) begin stateReg <= TEST_ADJUST_PAYLOAD_INIT; isFirstFrag = True; end @@ -1591,6 +1634,8 @@ module mkTestNormalOrSmallPayloadGen#( isFirstFrag = payloadFrag.isLast; if (payloadFrag.isLast) begin + expectedRemoteAddrReg <= nextRemoteAddr; + if (isLastPkt) begin stateReg <= TEST_ADJUST_PAYLOAD_INIT; end @@ -1637,7 +1682,8 @@ module mkTestNormalOrSmallPayloadGen#( // $display( // "time=%0t: checkResp", $time, - // ", isZeroPayloadLen=", fshow(payloadGenResp.isZeroPayloadLen) + // ", expectedRemoteAddrReg=%h", expectedRemoteAddrReg, + // ", isZeroPayloadLenReg=", fshow(isZeroPayloadLenReg) // ); endrule endmodule From 92b99723fac2337d5b71696955789d087ae4c2e5 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Tue, 30 Jan 2024 17:44:30 +0800 Subject: [PATCH 11/16] add mkSendQ --- src/DataTypes.bsv | 16 +- src/Headers.bsv | 7 +- src/PayloadGen.bsv | 273 ++++++++--- src/SendQ.bsv | 981 ++++++++++++++++++++++++++++++++++++++++ src/Utils.bsv | 18 +- test/TestPayloadGen.bsv | 248 +++++++++- test/TestSendQ.bsv | 551 ++++++++++++++++++++++ test/Utils4Test.bsv | 5 + 8 files changed, 1998 insertions(+), 101 deletions(-) create mode 100644 src/SendQ.bsv create mode 100644 test/TestSendQ.bsv diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index 3424e1e..4e9d7c7 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -299,7 +299,7 @@ typedef enum { DMA_SRC_SQ_ATOMIC, DMA_SRC_SQ_DISCARD // DMA_SRC_SQ_CANCEL -} DmaReqSrcType deriving(Bits, Eq, FShow); +} DmaReqSrcType deriving(Bits, Eq, FShow); // TODO: remove it typedef struct { DmaReadMetaData dmaReadMetaData; @@ -362,16 +362,16 @@ typedef enum { IBV_QPS_SQE, IBV_QPS_ERR, IBV_QPS_UNKNOWN, - IBV_QPS_CREATE // Not defined in rdma-core + IBV_QPS_CREATE // TODO: remote it. Not defined in rdma-core } StateQP deriving(Bits, Eq, FShow); typedef enum { - IBV_QPT_RC = 2, - IBV_QPT_UC = 3, - IBV_QPT_UD = 4, - // IBV_QPT_RAW_PACKET = 8, - IBV_QPT_XRC_SEND = 9, - IBV_QPT_XRC_RECV = 10 + IBV_QPT_RC = 2, + IBV_QPT_UC = 3, + IBV_QPT_UD = 4, + IBV_QPT_RAW_PACKET = 8, + IBV_QPT_XRC_SEND = 9, + IBV_QPT_XRC_RECV = 10 // IBV_QPT_DRIVER = 0xff } TypeQP deriving(Bits, Eq, FShow); diff --git a/src/Headers.bsv b/src/Headers.bsv index 4e8a23d..9e3ecf8 100644 --- a/src/Headers.bsv +++ b/src/Headers.bsv @@ -92,7 +92,8 @@ typedef enum { TRANS_TYPE_RD = 3'h2, // 3'b010 TRANS_TYPE_UD = 3'h3, // 3'b011 TRANS_TYPE_CNP = 3'h4, // 3'b100 - TRANS_TYPE_XRC = 3'h5 // 3'b101 + TRANS_TYPE_XRC = 3'h5, // 3'b101 + TRANS_TYPE_RAW = 3'h7 // 3'b111 Not defined in rdma-core } TransType deriving(Bits, Bounded, Eq, FShow); typedef SizeOf#(TransType) TRANS_TYPE_WIDTH; @@ -213,7 +214,7 @@ typedef TDiv#(ATOMIC_ACK_ETH_WIDTH, 8) ATOMIC_ACK_ETH_BYTE_WIDTH; // 4 bytes typedef struct { IMM data; -} ImmDt deriving(Bits, Bounded); +} ImmDt deriving(Bits, Bounded, FShow); typedef SizeOf#(ImmDt) IMM_DT_WIDTH; typedef TDiv#(IMM_DT_WIDTH, 8) IMM_DT_BYTE_WIDTH; @@ -221,7 +222,7 @@ typedef TDiv#(IMM_DT_WIDTH, 8) IMM_DT_BYTE_WIDTH; // 4 bytes typedef struct { RKEY rkey; -} IETH deriving(Bits, Bounded); +} IETH deriving(Bits, Bounded, FShow); typedef SizeOf#(IETH) IETH_WIDTH; typedef TDiv#(IETH_WIDTH, 8) IETH_BYTE_WIDTH; diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index afd9a9f..f067d4c 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -15,18 +15,19 @@ import Utils :: *; typedef Bit#(32) AddrIPv4; typedef Bit#(128) AddrIPv6; +typedef Bit#(48) MAC; typedef union tagged { AddrIPv4 IPv4; AddrIPv6 IPv6; -} IP deriving(Bits); +} IP deriving(Bits, Bounded); instance FShow#(IP); function Fmt fshow(IP ipAddr); case (ipAddr) matches tagged IPv4 .ipv4: begin return $format( - "ipv4=%d.%d.%d.%d", + "ipv4=%0d.%0d.%0d.%0d", ipv4[31 : 24], ipv4[23: 16], ipv4[15 : 8], ipv4[7 : 0] ); end @@ -42,32 +43,33 @@ instance FShow#(IP); endinstance typedef union tagged { - IMM ImmDt; - RKEY RKey2Inv; -} ImmDtOrInvRKey deriving(Bits, FShow); + IMM Imm; + RKEY RKey; +} ImmOrRKey deriving(Bits, FShow); typedef struct { WorkReqID id; // TODO: remove it WorkReqOpCode opcode; FlagsType#(WorkReqSendFlag) flags; TypeQP qpType; - PSN curPSN; + PSN psn; PMTU pmtu; IP dqpIP; + MAC macAddr; ScatterGatherList sgl; ADDR raddr; RKEY rkey; - // ADDR laddr; - // Length len; - // LKEY lkey; + // PKEY pkey; QPN sqpn; // TODO: remove it - Bool solicited; // Relevant only for the Send and RDMA Write with immediate data - // Maybe#(Long) comp; - // Maybe#(Long) swap; - Maybe#(ImmDtOrInvRKey) immDtOrInvRKey; + QPN dqpn; + // Bool solicited; // Relevant only for the Send and RDMA Write with immediate data + Maybe#(Long) comp; + Maybe#(Long) swap; + Maybe#(ImmOrRKey) immDtOrInvRKey; Maybe#(QPN) srqn; // for XRC - Maybe#(QPN) dqpn; // for UD Maybe#(QKEY) qkey; // for UD + Bool isFirst; + Bool isLast; } WorkQueueElem deriving(Bits); instance FShow#(WorkQueueElem); @@ -76,16 +78,18 @@ instance FShow#(WorkQueueElem); "WorkQueueElem { opcode=", fshow(wqe.opcode), ", flags=", fshow(wqe.flags), ", qpType=", fshow(wqe.qpType), - ", curPSN=%h", wqe.curPSN, + ", psn=%h", wqe.psn, ", pmtu=", fshow(wqe.pmtu), ", dqpIP=", fshow(wqe.dqpIP), + ", macAddr=%h", wqe.macAddr, ", sgl=", fshow(wqe.sgl), ", rkey=%h", wqe.rkey, ", raddr=%h", wqe.raddr, - // ", sqpn=%h", wqe.sqpn, - // ", sqpn=%h", wqe.sqpn, - ", solicited=", fshow(wqe.solicited), - // ", comp=", fshow(wqe.comp), ", swap=", fshow(wqe.swap), + ", sqpn=%h", wqe.sqpn, + ", dqpn=%h", wqe.dqpn, + // ", solicited=", fshow(wqe.solicited), + ", comp=", fshow(wqe.comp), + ", swap=", fshow(wqe.swap), ", immDtOrInvRKey=", fshow(wqe.immDtOrInvRKey), ", srqn=", fshow(wqe.srqn), ", dqpn=", fshow(wqe.dqpn), @@ -354,7 +358,148 @@ interface AddrChunkSrv; method Bool isIdle(); endinterface -// TODO: refactor AddrChunkSrv into fully pipeline +typedef struct { + PktNum remainingPktNum; + PktLen firstPktLen; + PktLen pmtuLen; + PktLen lastPktLen; + PMTU pmtu; + ADDR startAddr; + ADDR nextAddr; + Bool isOrigFirst; + Bool isOrigLast; +} ChunkMetaData deriving(Bits); + +module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); + FIFOF#(AddrChunkReq) reqQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + FIFOF#(AddrChunkResp) respQ <- mkFIFOF; + FIFOF#(PktMetaDataSGE) sgePktMetaDataOutQ <- mkFIFOF; + + // Pipeline FIFOF + FIFOF#(ChunkMetaData) chunkMetaDataQ <- mkFIFOF; + + Reg#(PktNum) remainingPktNumReg <- mkRegU; + Reg#(ADDR) nextChunkAddrReg <- mkRegU; + Reg#(Bool) isFirstChunkReg <- mkReg(True); + + rule resetAndClear if (clearAll); + reqQ.clear; + respQ.clear; + sgePktMetaDataOutQ.clear; + + chunkMetaDataQ.clear; + isFirstChunkReg <= True; + endrule + + rule recvReq if (!clearAll); + let addrChunkReq = reqQ.first; + reqQ.deq; + + immAssert( + !isZeroR(addrChunkReq.len), + "addrChunkReq.len assertion @ mkAddrChunkSrv", + $format( + "addrChunkReq.len=%0d cannot be zero", addrChunkReq.len + ) + ); + + let { + pmtuLen, firstPktLen, lastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU( + addrChunkReq.startAddr, addrChunkReq.len, addrChunkReq.pmtu + ); + + let chunkMetaData = ChunkMetaData { + remainingPktNum: sgePktNum - 1, + firstPktLen : firstPktLen, + pmtuLen : pmtuLen, + lastPktLen : lastPktLen, + pmtu : addrChunkReq.pmtu, + startAddr : addrChunkReq.startAddr, + nextAddr : secondChunkStartAddr, + isOrigFirst : addrChunkReq.isFirst, + isOrigLast : addrChunkReq.isLast + }; + chunkMetaDataQ.enq(chunkMetaData); + + let sgePktMetaData = PktMetaDataSGE { + firstPktLen: firstPktLen, + lastPktLen : lastPktLen, + sgePktNum : sgePktNum, + pmtu : addrChunkReq.pmtu + }; + sgePktMetaDataOutQ.enq(sgePktMetaData); + + // $display( + // "time=%0t: mkAddrChunkSrv recvReq", $time, + // ", addrChunkReq.len=%0d", addrChunkReq.len, + // ", sgePktNum=%0d", sgePktNum, + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen + // ); + endrule + + rule genResp if (!clearAll); + let chunkMetaData = chunkMetaDataQ.first; + + let firstPktLen = chunkMetaData.firstPktLen; + let pmtuLen = chunkMetaData.pmtuLen; + let lastPktLen = chunkMetaData.lastPktLen; + let pmtu = chunkMetaData.pmtu; + let startAddr = chunkMetaData.startAddr; + let nextAddr = chunkMetaData.nextAddr; + let isOrigFirst = chunkMetaData.isOrigFirst; + let isOrigLast = chunkMetaData.isOrigLast; + + let oneAsPSN = 1; + let nextChunkAddr = addrAddPsnMultiplyPMTU(nextChunkAddrReg, oneAsPSN, pmtu); + let remainingPktNum = remainingPktNumReg; + if (isFirstChunkReg) begin + nextChunkAddr = nextAddr; + remainingPktNum = chunkMetaData.remainingPktNum; + end + + let isLastChunk = isZeroR(remainingPktNum); + if (isLastChunk) begin + chunkMetaDataQ.deq; + end + else begin + remainingPktNumReg <= remainingPktNum - 1; + end + isFirstChunkReg <= isLastChunk; + nextChunkAddrReg <= nextChunkAddr; + + let chunkAddr = isFirstChunkReg ? startAddr : nextChunkAddrReg; + let chunkLen = isFirstChunkReg ? firstPktLen : (isLastChunk ? lastPktLen : pmtuLen); + let addrChunkResp = AddrChunkResp { + chunkAddr : chunkAddr, + chunkLen : chunkLen, + isFirst : isFirstChunkReg, + isLast : isLastChunk, + isOrigFirst: isOrigFirst, + isOrigLast : isOrigLast + }; + respQ.enq(addrChunkResp); + + // $display( + // "time=%0t: mkAddrChunkSrv genResp", $time, + // ", remainingPktNumReg=%0d", remainingPktNumReg, + // ", chunkAddr=%h", chunkAddr, + // ", nextChunkAddr=%h", nextChunkAddr, + // ", addrChunkResp=", fshow(addrChunkResp) + // ); + endrule + + interface srvPort = toGPServer(reqQ, respQ); + interface sgePktMetaDataPipeOut = toPipeOut(sgePktMetaDataOutQ); + method Bool isIdle() = !( + reqQ.notEmpty || + respQ.notEmpty || + chunkMetaDataQ.notEmpty || + sgePktMetaDataOutQ.notEmpty + ); +endmodule +/* module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); FIFOF#(AddrChunkReq) reqQ <- mkSizedFIFOF(valueOf(MAX_SGE)); FIFOF#(AddrChunkResp) respQ <- mkFIFOF; @@ -450,26 +595,7 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); isOrigLast : isOrigLastReg }; respQ.enq(addrChunkResp); -/* - let firstPktLastFragValidByteNum = calcLastFragValidByteNum(firstPktLenReg); - let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLenReg); - let curPktLastFragValidByteNum = isFirstReg ? - firstPktLastFragValidByteNum : ( - isLast ? lastPktLastFragValidByteNum : fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)) - ); - let sgePktMetaData = PktMetaDataSGE { - curPktLastFragValidByteNum: curPktLastFragValidByteNum, - pktLen : chunkLen, - pmtu : pmtuReg, - sgeHasJustTwoPkts : isTwoR(sgePktNumReg), - isFirst : isFirstReg, - isLast : isLast, - isFirstSGE : isOrigFirstReg, - isLastSGE : isOrigLastReg - }; - sgePktMetaDataOutQ.enq(sgePktMetaData); -*/ // $display( // "time=%0t: mkAddrChunkSrv genResp", $time, // ", isSQ=", fshow(isSQ), @@ -485,7 +611,7 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); method Bool isIdle() = !busyReg && !reqQ.notEmpty && !sgePktMetaDataOutQ.notEmpty && !respQ.notEmpty; endmodule - +*/ typedef struct { // DmaReqSrcType initiator; DmaReadMetaDataSGL sglDmaReadMetaData; @@ -499,6 +625,7 @@ typedef struct { } DmaReadCntrlResp deriving(Bits, FShow); typedef Server#(DmaReadCntrlReq, DmaReadCntrlResp) DmaCntrlReadSrv; +typedef Client#(DmaReadCntrlReq, DmaReadCntrlResp) DmaCntrlReadClt; interface DmaCntrl; method Bool isIdle(); @@ -601,21 +728,17 @@ module mkDmaReadCntrl#( end totalLenReg <= totalLen; sgeNumReg <= sgeNum; - // $display( - // "time=%0t: recvReq", $time, - // ", SGE sglIdx=%0d", sglIdx, - // ", sge.laddr=%h", sge.laddr, - // ", mergedLastPktLastFragValidByteNum=%0d", mergedLastPktLastFragValidByteNum, - // ", totalLen=%0d", totalLen, - // ", sgeNum=%0d", sgeNum - // // ", sgePktNum=%0d", sgePktNum, - // // ", firstPktLen=%0d", firstPktLen, - // // ", lastPktLen=%0d", lastPktLen, - // // ", pmtuLen=%0d", pmtuLen, - // // ", firstPktFragNum=%0d", firstPktFragNum, - // // ", lastPktFragNum=%0d", lastPktFragNum - // ); + if (isZeroR(sglIdxReg)) begin + immAssert( + sge.isFirst, + "first SGE assertion @ mkDmaReadCntrl", + $format( + "sge.isFirst=", fshow(sge.isFirst), + " should be true when sglIdxReg=%d", sglIdxReg + ) + ); + end // if (sglIdxReg == valueOf(TSub#(MAX_SGE, 1))) begin if (isAllOnesR(sglIdxReg)) begin immAssert( @@ -650,10 +773,20 @@ module mkDmaReadCntrl#( sglIdxReg <= sglIdxReg + 1; end // $display( - // "time=%0t: mkDmaReadCntrl recvReq", $time, - // ", sqpn=%h", curSQPN, - // ", dmaReadCntrlReq=", fshow(dmaReadCntrlReq), - // ", addrChunkReq=", fshow(addrChunkReq) + // "time=%0t: recvReq", $time, + // ", SGE sglIdx=%0d", sglIdx, + // ", sge.laddr=%h", sge.laddr, + // ", mergedLastPktLastFragValidByteNum=%0d", mergedLastPktLastFragValidByteNum, + // ", totalLen=%0d", totalLen, + // ", sgeNum=%0d", sgeNum, + // ", sge.isFirst=", fshow(sge.isFirst), + // ", sge.isLast=", fshow(sge.isLast) + // // ", sgePktNum=%0d", sgePktNum, + // // ", firstPktLen=%0d", firstPktLen, + // // ", lastPktLen=%0d", lastPktLen, + // // ", pmtuLen=%0d", pmtuLen, + // // ", firstPktFragNum=%0d", firstPktFragNum, + // // ", lastPktFragNum=%0d", lastPktFragNum // ); endrule @@ -1665,10 +1798,10 @@ typedef struct { typedef struct { Length totalLen; PktNum totalPktNum; - PMTU pmtu; + // PMTU pmtu; Bool isOnlyPkt; Bool isZeroPayloadLen; -} ReqGenTotalMetaData deriving(Bits, FShow); +} PayloadGenTotalMetaData deriving(Bits, FShow); typedef struct { ADDR firstRemoteAddr; @@ -1701,7 +1834,7 @@ typedef struct { interface PayloadGenerator; interface Server#(PayloadGenReqSG, PayloadGenRespSG) srvPort; - interface PipeOut#(ReqGenTotalMetaData) reqGenTotalMetaDataPipeOut; + interface PipeOut#(PayloadGenTotalMetaData) totalMetaDataPipeOut; interface DataStreamPipeOut payloadDataStreamPipeOut; method Bool payloadNotEmpty(); endinterface @@ -1709,9 +1842,9 @@ endinterface module mkPayloadGenerator#( Bool clearAll, Bool shouldAddPadding, DmaReadCntrl dmaReadCntrl )(PayloadGenerator); - FIFOF#(PayloadGenReqSG) payloadGenReqQ <- mkFIFOF; - FIFOF#(PayloadGenRespSG) payloadGenRespQ <- mkFIFOF; - FIFOF#(ReqGenTotalMetaData) reqGenTotalMetaDataQ <- mkFIFOF; + FIFOF#(PayloadGenReqSG) payloadGenReqQ <- mkFIFOF; + FIFOF#(PayloadGenRespSG) payloadGenRespQ <- mkFIFOF; + FIFOF#(PayloadGenTotalMetaData) totalMetaDataOutQ <- mkFIFOF; // Pipeline FIFO FIFOF#(DataStream) sgePayloadOutQ <- mkFIFOF; @@ -1753,8 +1886,7 @@ module mkPayloadGenerator#( remainingPktNumReg <= 0; // $display( - // "time=%0t: reset and clear mkPayloadGenerator", $time, - // ", pendingGenReqQ.notEmpty=", fshow(pendingGenReqQ.notEmpty) + // "time=%0t: reset and clear mkPayloadGenerator", $time // ); endrule @@ -1902,14 +2034,14 @@ module mkPayloadGenerator#( }; addPadCntQ.enq(paddingMetaData); - let reqGenTotalMetaData = ReqGenTotalMetaData { + let totalMetaData = PayloadGenTotalMetaData { totalLen : totalLen, totalPktNum : totalPktNum, - pmtu : pmtu, + // pmtu : pmtu, isOnlyPkt : isOnlyPkt, isZeroPayloadLen: isZeroPayloadLen }; - reqGenTotalMetaDataQ.enq(reqGenTotalMetaData); + totalMetaDataOutQ.enq(totalMetaData); // $display( // "time=%0t: mkPayloadGenerator calcAdjustedTotalPayloadMetaData", $time, // ", adjustedTotalPayloadMetaData=", fshow(adjustedTotalPayloadMetaData) @@ -2047,7 +2179,6 @@ module mkPayloadGenerator#( // ", curPayloadFrag.isLast=", fshow(curPayloadFrag.isLast) // ); end - // $display( // "time=%0t: PayloadGenerator genRespAndAddPadding", $time, // ", isZeroPayloadLen=", fshow(isZeroPayloadLen), @@ -2058,7 +2189,7 @@ module mkPayloadGenerator#( endrule interface srvPort = toGPServer(payloadGenReqQ, payloadGenRespQ); - interface reqGenTotalMetaDataPipeOut = toPipeOut(reqGenTotalMetaDataQ); + interface totalMetaDataPipeOut = toPipeOut(totalMetaDataOutQ); interface payloadDataStreamPipeOut = payloadBufPipeOut; method Bool payloadNotEmpty() = bramQ2PipeOut.notEmpty; endmodule diff --git a/src/SendQ.bsv b/src/SendQ.bsv new file mode 100644 index 0000000..8fee920 --- /dev/null +++ b/src/SendQ.bsv @@ -0,0 +1,981 @@ +import ClientServer :: *; +import FIFOF :: *; +import GetPut :: *; +import PAClib :: *; + +import DataTypes :: *; +import ExtractAndPrependPipeOut :: *; +import Headers :: *; +import PayloadGen :: *; +import PrimUtils :: *; +import Settings :: *; +import Utils :: *; + +typedef struct { + MAC macAddr; + IP ipAddr; + PktLen pktLen; +} PktInfo4UDP deriving(Bits, FShow); + +typedef struct { + HeaderData headerData; + HeaderByteNum headerLen; + Bool hasPayload; + Bool hasHeader; +} PktHeaderInfo deriving(Bits, FShow); + +// TODO: support hasHeader +function PktHeaderInfo genPktHeaderInfo( + HeaderData headerData, HeaderByteNum headerLen, Bool hasPayload +); + let hasHeader = True; + return PktHeaderInfo { + headerData: headerData, + headerLen : headerLen, + hasPayload: hasPayload, + hasHeader : hasHeader + }; +endfunction + +function Maybe#(RdmaOpCode) genFirstOrOnlyRdmaOpCode(WorkReqOpCode wrOpCode, Bool isOnlyReqPkt); + return case (wrOpCode) + IBV_WR_RDMA_WRITE : tagged Valid (isOnlyReqPkt ? RDMA_WRITE_ONLY : RDMA_WRITE_FIRST); + IBV_WR_RDMA_WRITE_WITH_IMM : tagged Valid (isOnlyReqPkt ? RDMA_WRITE_ONLY_WITH_IMMEDIATE : RDMA_WRITE_FIRST); + IBV_WR_SEND : tagged Valid (isOnlyReqPkt ? SEND_ONLY : SEND_FIRST); + IBV_WR_SEND_WITH_IMM : tagged Valid (isOnlyReqPkt ? SEND_ONLY_WITH_IMMEDIATE : SEND_FIRST); + IBV_WR_SEND_WITH_INV : tagged Valid (isOnlyReqPkt ? SEND_ONLY_WITH_INVALIDATE : SEND_FIRST); + IBV_WR_RDMA_READ_RESP : tagged Valid (isOnlyReqPkt ? RDMA_READ_RESPONSE_ONLY : RDMA_READ_RESPONSE_FIRST); + IBV_WR_RDMA_READ : tagged Valid RDMA_READ_REQUEST; + IBV_WR_ATOMIC_CMP_AND_SWP : tagged Valid COMPARE_SWAP; + IBV_WR_ATOMIC_FETCH_AND_ADD: tagged Valid FETCH_ADD; + default : tagged Invalid; + endcase; +endfunction + +function Maybe#(RdmaOpCode) genMiddleOrLastRdmaOpCode(WorkReqOpCode wrOpCode, Bool isLastReqPkt); + return case (wrOpCode) + IBV_WR_RDMA_WRITE : tagged Valid (isLastReqPkt ? RDMA_WRITE_LAST : RDMA_WRITE_MIDDLE); + IBV_WR_RDMA_WRITE_WITH_IMM: tagged Valid (isLastReqPkt ? RDMA_WRITE_LAST_WITH_IMMEDIATE : RDMA_WRITE_MIDDLE); + IBV_WR_SEND : tagged Valid (isLastReqPkt ? SEND_LAST : SEND_MIDDLE); + IBV_WR_SEND_WITH_IMM : tagged Valid (isLastReqPkt ? SEND_LAST_WITH_IMMEDIATE : SEND_MIDDLE); + IBV_WR_SEND_WITH_INV : tagged Valid (isLastReqPkt ? SEND_LAST_WITH_INVALIDATE : SEND_MIDDLE); + IBV_WR_RDMA_READ_RESP : tagged Valid (isLastReqPkt ? RDMA_READ_RESPONSE_LAST : RDMA_READ_RESPONSE_MIDDLE); + default : tagged Invalid; + endcase; +endfunction + +function Maybe#(XRCETH) genXRCETH(WorkQueueElem wqe); + return case (wqe.qpType) + IBV_QPT_XRC_SEND: tagged Valid XRCETH { + srqn: unwrapMaybe(wqe.srqn), + rsvd: unpack(0) + }; + default: tagged Invalid; + endcase; +endfunction + +function Maybe#(DETH) genDETH(WorkQueueElem wqe); + return case (wqe.qpType) + IBV_QPT_UD: tagged Valid DETH { + qkey: unwrapMaybe(wqe.qkey), + sqpn: wqe.sqpn, + rsvd: unpack(0) + }; + default: tagged Invalid; + endcase; +endfunction + +function Maybe#(RETH) genRETH( + WorkReqOpCode wrOpCode, ADDR raddr, RKEY rkey, Length dlen +); + return case (wrOpCode) + IBV_WR_RDMA_WRITE , + IBV_WR_RDMA_WRITE_WITH_IMM, + IBV_WR_RDMA_READ : tagged Valid RETH { + va : raddr, + rkey: rkey, + dlen: dlen + }; + default : tagged Invalid; + endcase; +endfunction + +// TODO: check fetch add needs both swap and comp? +function Maybe#(AtomicEth) genAtomicEth(WorkQueueElem wqe); + if (wqe.swap matches tagged Valid .swap &&& wqe.comp matches tagged Valid .comp) begin + return case (wqe.opcode) + IBV_WR_ATOMIC_CMP_AND_SWP , + IBV_WR_ATOMIC_FETCH_AND_ADD: tagged Valid AtomicEth { + va : wqe.raddr, + rkey: wqe.rkey, + swap: swap, + comp: comp + }; + default : tagged Invalid; + endcase; + end + else begin + return tagged Invalid; + end +endfunction + +function Maybe#(ImmDt) genImmDt(WorkQueueElem wqe); + case (wqe.opcode) + IBV_WR_RDMA_WRITE_WITH_IMM, + IBV_WR_SEND_WITH_IMM : + if ( + wqe.immDtOrInvRKey matches tagged Valid .immDtOrInvRKey &&& + immDtOrInvRKey matches tagged Imm .immDt + ) begin + return tagged Valid ImmDt { + data: immDt + }; + end + else begin + return tagged Invalid; + end + default : return tagged Invalid; + endcase + // return case (wqe.opcode) + // IBV_WR_RDMA_WRITE_WITH_IMM, + // IBV_WR_SEND_WITH_IMM : tagged Valid ImmDt { + // data: unwrapMaybe(wqe.immDtOrInvRKey) + // }; + // default : tagged Invalid; + // endcase; +endfunction + +function Maybe#(IETH) genIETH(WorkQueueElem wqe); + if ( + wqe.immDtOrInvRKey matches tagged Valid .immDtOrInvRKey &&& + immDtOrInvRKey matches tagged RKey .rkey2Inv &&& + wqe.opcode == IBV_WR_SEND_WITH_INV + ) begin + return tagged Valid IETH { + rkey: rkey2Inv + }; + end + else begin + return tagged Invalid; + end + // return case (wqe.opcode) + // IBV_WR_SEND_WITH_INV: tagged Valid IETH { + // rkey: unwrapMaybe(wqe.immDtOrInvRKey) + // }; + // default : tagged Invalid; + // endcase; +endfunction + +function Maybe#(PktHeaderInfo) genFirstOrOnlyPktHeader( + WorkQueueElem wqe, Bool isOnlyReqPkt, Bool solicited, PSN psn, PAD padCnt, + Bool ackReq, ADDR remoteAddr, Length dlen, Bool hasPayload +); + let maybeTrans = qpType2TransType(wqe.qpType); + let maybeOpCode = genFirstOrOnlyRdmaOpCode(wqe.opcode, isOnlyReqPkt); + + let isReadOrAtomicWR = isReadOrAtomicWorkReq(wqe.opcode); + if ( + maybeTrans matches tagged Valid .trans &&& + maybeOpCode matches tagged Valid .opcode + ) begin + let bth = BTH { + trans : trans, + opcode : opcode, + solicited: isOnlyReqPkt && solicited, + migReq : unpack(0), + padCnt : padCnt, + tver : unpack(0), + pkey : dontCareValue, // wqe.pkey, + fecn : unpack(0), + becn : unpack(0), + resv6 : unpack(0), + dqpn : wqe.dqpn, + ackReq : isOnlyReqPkt && ackReq, + resv7 : unpack(0), + psn : psn + }; + + let xrceth = genXRCETH(wqe); + let deth = genDETH(wqe); + let reth = genRETH(wqe.opcode, remoteAddr, wqe.rkey, dlen); + let atomicEth = genAtomicEth(wqe); + let immDt = genImmDt(wqe); + let ieth = genIETH(wqe); + + // If WR has zero length, then no payload, no matter what kind of opcode + // let hasPayload = workReqHasPayload(wr); + case (wqe.opcode) + IBV_WR_RDMA_WRITE: begin + return case (wqe.qpType) + IBV_QPT_RC, + IBV_QPT_UC: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(reth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(reth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_RDMA_WRITE_WITH_IMM: begin + return case (wqe.qpType) + IBV_QPT_RC, + IBV_QPT_UC: tagged Valid genPktHeaderInfo( + isOnlyReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(reth)), pack(unwrapMaybe(immDt))}) : + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(reth))}), + isOnlyReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + isOnlyReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(reth)), pack(unwrapMaybe(immDt)) }) : + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(reth)) }), + isOnlyReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_SEND: begin + return case (wqe.qpType) + IBV_QPT_RC, + IBV_QPT_UC: tagged Valid genPktHeaderInfo( + zeroExtendLSB(pack(bth)), + fromInteger(valueOf(BTH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_UD: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(deth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(DETH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_SEND_WITH_IMM: begin + return case (wqe.qpType) + IBV_QPT_RC, + IBV_QPT_UC: tagged Valid genPktHeaderInfo( + isOnlyReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(immDt)) }) : + zeroExtendLSB(pack(bth)), + isOnlyReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_UD: tagged Valid genPktHeaderInfo( + // UD always has only pkt + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(deth)), pack(unwrapMaybe(immDt)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(DETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + isOnlyReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(immDt)) }) : + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)) }), + isOnlyReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_SEND_WITH_INV: begin + return case (wqe.qpType) + IBV_QPT_RC: tagged Valid genPktHeaderInfo( + isOnlyReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(ieth)) }) : + zeroExtendLSB(pack(bth)), + isOnlyReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(IETH_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + isOnlyReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(ieth)) }) : + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)) }), + isOnlyReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IETH_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_RDMA_READ: begin + return case (wqe.qpType) + IBV_QPT_RC: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(reth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + False // Read requests have no payload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(reth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + False // Read requests have no payload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_ATOMIC_CMP_AND_SWP , + IBV_WR_ATOMIC_FETCH_AND_ADD: begin + return case (wqe.qpType) + IBV_QPT_RC: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(atomicEth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(ATOMIC_ETH_BYTE_WIDTH)), + False // Atomic requests have no payload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(atomicEth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(ATOMIC_ETH_BYTE_WIDTH)), + False // Atomic requests have no payload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_RDMA_READ_RESP: begin + return case (wqe.qpType) + IBV_QPT_RC , + IBV_QPT_XRC_SEND, + IBV_QPT_XRC_RECV: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(reth) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + hasPayload + ); + default : tagged Invalid; + endcase; + end + default: return tagged Invalid; + endcase + end + else begin + return tagged Invalid; + end +endfunction + +function Maybe#(PktHeaderInfo) genMiddleOrLastPktHeader( + WorkQueueElem wqe, Bool isLastReqPkt, Bool solicited, PSN psn, + PAD padCnt, Bool ackReq, ADDR remoteAddr, PktLen pktLen // , Bool hasPayload +); + let maybeTrans = qpType2TransType(wqe.qpType); + let maybeOpCode = genMiddleOrLastRdmaOpCode(wqe.opcode, isLastReqPkt); + + if ( + maybeTrans matches tagged Valid .trans &&& + maybeOpCode matches tagged Valid .opcode + ) begin + let bth = BTH { + trans : trans, + opcode : opcode, + solicited: isLastReqPkt && solicited, + migReq : unpack(0), + padCnt : padCnt, + tver : unpack(0), + pkey : dontCareValue, // wqe.pkey, + fecn : unpack(0), + becn : unpack(0), + resv6 : unpack(0), + dqpn : wqe.dqpn, + ackReq : isLastReqPkt && ackReq, + resv7 : unpack(0), + psn : psn + }; + + Length dlen = zeroExtend(pktLen); + let xrceth = genXRCETH(wqe); + let reth = genRETH(wqe.opcode, remoteAddr, wqe.rkey, dlen); + let immDt = genImmDt(wqe); + let ieth = genIETH(wqe); + + let hasPayload = True; + case (wqe.opcode) + IBV_WR_RDMA_WRITE:begin + return case (wqe.qpType) + IBV_QPT_RC: tagged Valid genPktHeaderInfo( + zeroExtendLSB(pack(bth)), + fromInteger(valueOf(BTH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_RDMA_WRITE_WITH_IMM: begin + return case (wqe.qpType) + IBV_QPT_RC: tagged Valid genPktHeaderInfo( + isLastReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(immDt))}) : + zeroExtendLSB(pack(bth)), + isLastReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + isLastReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(immDt)) }) : + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)) }), + isLastReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_SEND: begin + return case (wqe.qpType) + IBV_QPT_RC: tagged Valid genPktHeaderInfo( + zeroExtendLSB(pack(bth)), + fromInteger(valueOf(BTH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_SEND_WITH_IMM: begin + return case (wqe.qpType) + IBV_QPT_RC: tagged Valid genPktHeaderInfo( + isLastReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(immDt)) }) : + zeroExtendLSB(pack(bth)), + isLastReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + isLastReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(immDt)) }) : + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)) }), + isLastReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_SEND_WITH_INV: begin + return case (wqe.qpType) + IBV_QPT_RC: tagged Valid genPktHeaderInfo( + isLastReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(ieth)) }) : + zeroExtendLSB(pack(bth)), + isLastReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(IETH_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH)), + hasPayload + ); + IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( + isLastReqPkt ? + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(ieth)) }) : + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)) }), + isLastReqPkt ? + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IETH_BYTE_WIDTH)) : + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH)), + hasPayload + ); + default: tagged Invalid; + endcase; + end + IBV_WR_RDMA_READ_RESP: begin + return case (wqe.qpType) + IBV_QPT_RC , + IBV_QPT_XRC_SEND, + IBV_QPT_XRC_RECV: tagged Valid genPktHeaderInfo( + zeroExtendLSB({ pack(bth), pack(reth) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + hasPayload + ); + default : tagged Invalid; + endcase; + end + default: return tagged Invalid; + endcase + end + else begin + return tagged Invalid; + end +endfunction + +function Bool workReqNeedPayloadGen(WorkReqOpCode opcode); + return case (opcode) + IBV_WR_RDMA_WRITE , + IBV_WR_RDMA_WRITE_WITH_IMM, + IBV_WR_SEND , + IBV_WR_SEND_WITH_IMM , + IBV_WR_SEND_WITH_INV , + IBV_WR_RDMA_READ_RESP : True; + default : False; + endcase; +endfunction + +typedef TMul#(32, 1024) WQE_SLICE_MAX_SIZE; // 32KB + +typedef struct { + ADDR remoteAddr; + Length totalLen; + PSN curPSN; + PktLen pktLen; + PAD padCnt; + Bool hasPayload; + Bool ackReq; + Bool solicited; + Bool isFirstPkt; + Bool isLastPkt; + Bool isOnlyPkt; +} HeaderGenInfo deriving(Bits, FShow); + +interface SendQ; + interface Put#(WorkQueueElem) wqeInPut; + interface PipeOut#(PktInfo4UDP) udpInfoPipeOut; + interface DataStreamPipeOut rdmaDataStreamPipeOut; + method Bool isEmpty(); +endinterface + +module mkSendQ#( + Bool clearAll, + PayloadGenerator payloadGenerator +)(SendQ); + FIFOF#(WorkQueueElem) wqeInQ <- mkFIFOF; + FIFOF#(PktInfo4UDP) udpInfoOutQ <- mkFIFOF; + + // Pipeline FIFOF + FIFOF#(Tuple2#(WorkQueueElem, Bool)) totalMetaDataQ <- mkFIFOF; + FIFOF#(Tuple5#(WorkQueueElem, Length, PktNum, Bool, Bool)) psnUpdateQ <- mkFIFOF; + FIFOF#(Tuple2#(WorkQueueElem, HeaderGenInfo)) headerPrepareQ <- mkFIFOF; + FIFOF#(Tuple3#(WorkQueueElem, Maybe#(PktHeaderInfo), PktLen)) pendingHeaderQ <- mkFIFOF; + FIFOF#(PktInfo4UDP) udpPktInfoOutQ <- mkFIFOF; + FIFOF#(RdmaHeader) pktHeaderQ <- mkFIFOF; + // Reg#(PktNum) remainingPktNumReg <- mkRegU; + // Reg#(Bool) isFirstOrOnlyReqPktReg <- mkReg(True); + Reg#(PSN) curPsnReg <- mkRegU; + Reg#(Bool) wqeFirstPktReg <- mkRegU; + + let headerDataStreamAndMetaDataPipeOut <- mkHeader2DataStream( + clearAll, toPipeOut(pktHeaderQ) + ); + let rdmaPktDataStreamPipeOut <- mkPrependHeader2PipeOut( + clearAll, + headerDataStreamAndMetaDataPipeOut.headerDataStream, + headerDataStreamAndMetaDataPipeOut.headerMetaData, + payloadGenerator.payloadDataStreamPipeOut + ); + + (* no_implicit_conditions, fire_when_enabled *) + rule resetAndClear if (clearAll); + wqeInQ.clear; + udpInfoOutQ.clear; + + wqeFirstPktReg <= True; + // $display("time=%0t: reset and clear mkSendQ", $time); + endrule + + (* conflict_free = "recvWQE, \ + recvTotalMetaData, \ + updatePSN, \ + prepareHeader, \ + genPktHeader" *) + rule recvWQE if (!clearAll); + let wqe = wqeInQ.first; + wqeInQ.deq; + + let qpType = wqe.qpType; + immAssert( + qpType == IBV_QPT_RC || qpType == IBV_QPT_UC || + qpType == IBV_QPT_XRC_SEND || qpType == IBV_QPT_UD, + "qpType assertion @ mkSendQ", + $format( + "qpType=", fshow(qpType), " unsupported" + ) + ); + + let sgeIdx = 0; + let firstSGE = wqe.sgl[sgeIdx]; + let lenFirstSGE = firstSGE.len; + if (isAtomicWorkReq(wqe.opcode)) begin + immAssert( + lenFirstSGE == fromInteger(valueOf(ATOMIC_WORK_REQ_LEN)), + "atomic length assertion @ mkSendQ", + $format( + "lenFirstSGE=%0d", lenFirstSGE, + " should be %0d for atomic WQE", valueOf(ATOMIC_WORK_REQ_LEN) + ) + ); + end + if (isZeroR(lenFirstSGE)) begin + immAssert( + wqe.isFirst && wqe.isLast && firstSGE.isFirst && firstSGE.isLast, + "zero length assertion @ mkSendQ", + $format( + "wqe.isFirst=", fshow(wqe.isFirst), + ", wqe.isLast=", fshow(wqe.isLast), + ", firstSGE.isFirst=", fshow(firstSGE.isFirst), + ", firstSGE.isLast=", fshow(firstSGE.isLast), + " should be all true when lenFirstSGE=%0d", lenFirstSGE + ) + ); + end + + let hasImmDt = workReqHasImmDt(wqe.opcode); + let hasInv = workReqHasInv(wqe.opcode); + let hasComp = workReqHasComp(wqe.opcode); + let hasSwap = workReqHasSwap(wqe.opcode); + if (hasComp) begin + immAssert( + isValid(wqe.comp), + "hasComp assertion @ mkSendQ", + $format( + "wqe.comp=", fshow(wqe.comp), + " should be valid when wqe.opcode=", fshow(wqe.opcode) + ) + ); + end + if (hasSwap) begin + immAssert( + isValid(wqe.swap), + "hasSwap assertion @ mkSendQ", + $format( + "wqe.swap=", fshow(wqe.swap), + " should be valid when wqe.opcode=", fshow(wqe.opcode) + ) + ); + end + if (hasImmDt) begin + immAssert( + isValid(wqe.immDtOrInvRKey), + "hasImmDt assertion @ mkSendQ", + $format( + "wqe.immDtOrInvRKey=", fshow(wqe.immDtOrInvRKey), + " should be valid when wqe.opcode=", fshow(wqe.opcode) + ) + ); + let immDtOrInvRKey = unwrapMaybe(wqe.immDtOrInvRKey); + // TODO: check immDtOrInvRKey is Imm + end + if (hasInv) begin + immAssert( + isValid(wqe.immDtOrInvRKey), + "hasInv assertion @ mkSendQ", + $format( + "wqe.immDtOrInvRKey=", fshow(wqe.immDtOrInvRKey), + " should be valid when wqe.opcode=", fshow(wqe.opcode) + ) + ); + let immDtOrInvRKey = unwrapMaybe(wqe.immDtOrInvRKey); + // TODO: check immDtOrInvRKey is RKey + end + + let isSendWR = isSendWorkReq(wqe.opcode); + let needPayloadGen = workReqNeedPayloadGen(wqe.opcode); + if (needPayloadGen) begin + let payloadGenReq = PayloadGenReqSG { + wrID : wqe.id, + sqpn : wqe.sqpn, + sgl : wqe.sgl, + raddr: isSendWR ? 0 : wqe.raddr, + pmtu : wqe.pmtu + }; + payloadGenerator.srvPort.request.put(payloadGenReq); + end + totalMetaDataQ.enq(tuple2(wqe, needPayloadGen)); + // TODO: handle pending read/atomic request number limit + + // $display( + // "time=%0t: mkSendQ 1st stage recvWQE", $time, + // ", sqpn=%h", wqe.sqpn, + // ", id=%h", wqe.id, + // ", pmtu=", fshow(wqe.pmtu), + // ", needPayloadGen=", fshow(needPayloadGen) + // ); + endrule + + rule recvTotalMetaData if (!clearAll); + let { wqe, needPayloadGen } = totalMetaDataQ.first; + totalMetaDataQ.deq; + + let sglZeroIdx = 0; + let hasPayload = needPayloadGen; + let isOnlyPkt = !needPayloadGen; + let totalLen = wqe.sgl[sglZeroIdx].len; + let totalPktNum = 1; + if (needPayloadGen) begin + let payloadTotalMetaData = payloadGenerator.totalMetaDataPipeOut.first; + payloadGenerator.totalMetaDataPipeOut.deq; + + hasPayload = !payloadTotalMetaData.isZeroPayloadLen; + isOnlyPkt = payloadTotalMetaData.isOnlyPkt; + totalLen = payloadTotalMetaData.totalLen; + totalPktNum = payloadTotalMetaData.totalPktNum; + + if (!wqe.isFirst && !wqe.isLast) begin + immAssert( + !isOnlyPkt && + payloadTotalMetaData.totalLen == fromInteger(valueOf(WQE_SLICE_MAX_SIZE)), + "wqe slice length assertion @ mkSendQ", + $format( + "payloadTotalMetaData.totalLen=%0d", payloadTotalMetaData.totalLen, + " should == WQE_SLICE_MAX_SIZE=%0d", valueOf(WQE_SLICE_MAX_SIZE), + ", and isOnlyPkt=", fshow(isOnlyPkt), + " should be false when wqe.isFirst=", fshow(wqe.isFirst), + " and wqe.isLast=", fshow(wqe.isLast) + ) + ); + end + else if (!wqe.isFirst && wqe.isLast) begin + immAssert( + fromInteger(valueOf(WQE_SLICE_MAX_SIZE)) >= payloadTotalMetaData.totalLen, + "wqe slice length assertion @ mkSendQ", + $format( + "payloadTotalMetaData.totalLen=%0d", payloadTotalMetaData.totalLen, + " should be no more than WQE_SLICE_MAX_SIZE=%0d", valueOf(WQE_SLICE_MAX_SIZE), + " when wqe.isFirst=", fshow(wqe.isFirst), + " and wqe.isLast=", fshow(wqe.isLast) + ) + ); + end + end + psnUpdateQ.enq(tuple5( + wqe, totalLen, totalPktNum, hasPayload, isOnlyPkt + )); + $display( + "time=%0t: mkSendQ 2nd stage recvTotalMetaData", $time, + ", sqpn=%h", wqe.sqpn, + ", id=%h", wqe.id, + ", psn=%h", wqe.psn, + ", totalLen=%0d", totalLen, + ", totalPktNum=%0d", totalPktNum, + ", hasPayload=", fshow(hasPayload), + ", isOnlyPkt=", fshow(isOnlyPkt), + ", needPayloadGen=", fshow(needPayloadGen) + ); + endrule + + rule updatePSN if (!clearAll); + let { + wqe, totalLen, totalPktNum, hasPayload, isOnlyPkt + } = psnUpdateQ.first; + + let curPSN = curPsnReg; + if (wqeFirstPktReg) begin + curPSN = wqe.psn; + end + curPsnReg <= curPSN + 1; + + let remoteAddr = wqe.raddr; + let pktPayloadLen = 0; + let padCnt = 0; + let wqeLastPkt = isOnlyPkt; + if (hasPayload) begin + let payloadGenResp <- payloadGenerator.srvPort.response.get; + remoteAddr = payloadGenResp.raddr; + pktPayloadLen = payloadGenResp.pktLen; + padCnt = payloadGenResp.padCnt; + wqeLastPkt = payloadGenResp.isLast; + end + wqeFirstPktReg <= wqeLastPkt; + + if (wqeLastPkt) begin + psnUpdateQ.deq; + end + + let isFirstPkt = wqeFirstPktReg && wqe.isFirst; + let isLastPkt = wqeLastPkt && wqe.isLast; + + if (!isLastPkt) begin + immAssert( + !isOnlyPkt, + "isOnlyPkt assertion @ mkSendQ", + $format( + "isOnlyPkt=", fshow(isOnlyPkt), + " should be false when wqe.isFirstPkt=", fshow(isFirstPkt), + " and wqe.isLastPkt=", fshow(isLastPkt) + ) + ); + end + if (isFirstPkt && isLastPkt) begin + immAssert( + isOnlyPkt, + "isOnlyPkt assertion @ mkSendQ", + $format( + "isOnlyPkt=", fshow(isOnlyPkt), + " should be true when wqe.isFirstPkt=", fshow(isFirstPkt), + " and wqe.isLastPkt=", fshow(isLastPkt) + ) + ); + end + + let ackReq = containWorkReqFlag(wqe.flags, IBV_SEND_SIGNALED); + let solicited = containWorkReqFlag(wqe.flags, IBV_SEND_SOLICITED); + let headerGenInfo = HeaderGenInfo { + remoteAddr: remoteAddr, + totalLen : totalLen, + curPSN : curPSN, + pktLen : pktPayloadLen, + padCnt : padCnt, + hasPayload: hasPayload, + ackReq : ackReq, + solicited : solicited, + isFirstPkt: isFirstPkt, + isLastPkt : isLastPkt, + isOnlyPkt : isOnlyPkt + }; + headerPrepareQ.enq(tuple2(wqe, headerGenInfo)); + // $display( + // "time=%0t: mkSendQ 3th stage updatePSN", $time, + // ", sqpn=%h", wqe.sqpn, + // ", id=%h", wqe.id, + // ", curPSN=%h", curPSN, + // ", pktPayloadLen=%0d", pktPayloadLen, + // ", padCnt=%0d", padCnt, + // ", isFirstPkt=", fshow(isFirstPkt), + // ", isLastPkt=", fshow(isLastPkt), + // ", isOnlyPkt=", fshow(isOnlyPkt) + // ); + endrule + + rule prepareHeader if (!clearAll); + let { wqe, headerGenInfo } = headerPrepareQ.first; + headerPrepareQ.deq; + + let remoteAddr = headerGenInfo.remoteAddr; + let totalLen = headerGenInfo.totalLen; + let curPSN = headerGenInfo.curPSN; + let pktPayloadLen = headerGenInfo.pktLen; + let padCnt = headerGenInfo.padCnt; + let hasPayload = headerGenInfo.hasPayload; + let ackReq = headerGenInfo.ackReq; + let solicited = headerGenInfo.solicited; + let isFirstPkt = headerGenInfo.isFirstPkt; + let isLastPkt = headerGenInfo.isLastPkt; + let isOnlyPkt = headerGenInfo.isOnlyPkt; + + let maybePktHeaderInfo = dontCareValue; + if (isFirstPkt) begin + let maybeFirstOrOnlyPktHeaderInfo = genFirstOrOnlyPktHeader( + wqe, isOnlyPkt, solicited, curPSN, padCnt, ackReq, remoteAddr, totalLen, hasPayload + ); + immAssert( + isValid(maybeFirstOrOnlyPktHeaderInfo), + "maybeFirstOrOnlyPktHeaderInfo assertion @ mkSendQ", + $format( + "maybeFirstOrOnlyPktHeaderInfo=", fshow(maybeFirstOrOnlyPktHeaderInfo), + " is not valid, and current WQE=", fshow(wqe) + ) + ); + + maybePktHeaderInfo = maybeFirstOrOnlyPktHeaderInfo; + end + else begin + let maybeMiddleOrLastPktHeaderInfo = genMiddleOrLastPktHeader( + wqe, isLastPkt, solicited, curPSN, padCnt, ackReq, remoteAddr, pktPayloadLen + ); + immAssert( + isValid(maybeMiddleOrLastPktHeaderInfo), + "maybeMiddleOrLastPktHeaderInfo assertion @ mkSendQ", + $format( + "maybeMiddleOrLastPktHeaderInfo=", fshow(maybeMiddleOrLastPktHeaderInfo), + " is not valid, and current WQE=", fshow(wqe) + ) + ); + + maybePktHeaderInfo = maybeMiddleOrLastPktHeaderInfo; + end + + let pktLenWithPadCnt = pktPayloadLen + zeroExtend(padCnt); + immAssert( + isZero(pktLenWithPadCnt[1:0]), + "pktLenWithPadCnt assertion @ mkSendQ", + $format( + "pktLenWithPadCnt=%0d", pktLenWithPadCnt, + " should be multiple of four, when padCnt=%0d", padCnt, + " pktPayloadLen=%0d", pktPayloadLen + ) + ); + + pendingHeaderQ.enq(tuple3(wqe, maybePktHeaderInfo, pktLenWithPadCnt)); + // $display( + // "time=%0t: mkSendQ 4th stage prepareHeader", $time, + // ", sqpn=%h", wqe.sqpn, + // ", id=%h", wqe.id, + // ", curPSN=%h", curPSN, + // ", isFirstPkt=", fshow(isFirstPkt), + // ", isLastPkt=", fshow(isLastPkt), + // ", isOnlyPkt=", fshow(isOnlyPkt), + // ", isValid(maybePktHeaderInfo)=", fshow(isValid(maybePktHeaderInfo)) + // ); + endrule + + rule genPktHeader if (!clearAll); + let { wqe, maybePktHeaderInfo, pktLenWithPadCnt } = pendingHeaderQ.first; + pendingHeaderQ.deq; + + if (maybePktHeaderInfo matches tagged Valid .pktHeaderInfo) begin + let headerData = pktHeaderInfo.headerData; + let headerLen = pktHeaderInfo.headerLen; + let hasPayload = pktHeaderInfo.hasPayload; + let pktHeader = genRdmaHeader(headerData, headerLen, hasPayload); + + let udpPktInfo = PktInfo4UDP { + macAddr: wqe.macAddr, + ipAddr : wqe.dqpIP, + pktLen : pktLenWithPadCnt + zeroExtend(headerLen) + }; + + pktHeaderQ.enq(pktHeader); + udpPktInfoOutQ.enq(udpPktInfo); + // $display( + // "time=%0t: mkSendQ 5th stage genPktHeader", $time, + // ", sqpn=%h", wqe.sqpn, + // ", id=%h", wqe.id, + // ", pktLenWithPadCnt=%0d", pktLenWithPadCnt, + // ", headerLen=%0d", headerLen, + // ", udpPktInfo.macAddr=%h", udpPktInfo.macAddr, + // ", udpPktInfo.ipAddr=", fshow(udpPktInfo.ipAddr), + // ", udpPktInfo.pktLen=%0d", udpPktInfo.pktLen + // ); + end + endrule + + interface wqeInPut = toPut(wqeInQ); + interface rdmaDataStreamPipeOut = rdmaPktDataStreamPipeOut; + // interface rdmaDataStreamPipeOut = payloadGenerator.payloadDataStreamPipeOut; + interface udpInfoPipeOut = toPipeOut(udpPktInfoOutQ); + method Bool isEmpty() = !( + wqeInQ.notEmpty || + udpInfoOutQ.notEmpty || + totalMetaDataQ.notEmpty || + psnUpdateQ.notEmpty || + headerPrepareQ.notEmpty || + pendingHeaderQ.notEmpty || + udpPktInfoOutQ.notEmpty || + pktHeaderQ.notEmpty + ); +endmodule diff --git a/src/Utils.bsv b/src/Utils.bsv index 6dece30..c6062f7 100644 --- a/src/Utils.bsv +++ b/src/Utils.bsv @@ -718,12 +718,13 @@ endfunction function Maybe#(TransType) qpType2TransType(TypeQP qpt); return case (qpt) - IBV_QPT_RC : tagged Valid TRANS_TYPE_RC; - IBV_QPT_UC : tagged Valid TRANS_TYPE_UC; - IBV_QPT_UD : tagged Valid TRANS_TYPE_UD; - IBV_QPT_XRC_RECV, - IBV_QPT_XRC_SEND: tagged Valid TRANS_TYPE_XRC; - default : tagged Invalid; + IBV_QPT_RC : tagged Valid TRANS_TYPE_RC; + IBV_QPT_UC : tagged Valid TRANS_TYPE_UC; + IBV_QPT_UD : tagged Valid TRANS_TYPE_UD; + IBV_QPT_XRC_RECV , + IBV_QPT_XRC_SEND : tagged Valid TRANS_TYPE_XRC; + IBV_QPT_RAW_PACKET: tagged Valid TRANS_TYPE_RAW; + default : tagged Invalid; endcase; endfunction @@ -737,6 +738,7 @@ function Bool transTypeMatchQpType(TransType tt, TypeQP qpt, Bool isRespPkt); (!isRespPkt && qpt == IBV_QPT_XRC_RECV) || (isRespPkt && qpt == IBV_QPT_XRC_SEND) ); + TRANS_TYPE_RAW: (qpt == IBV_QPT_RAW_PACKET); default: False; endcase; endfunction @@ -1392,6 +1394,10 @@ function Bool isReadWorkReq(WorkReqOpCode opcode); return opcode == IBV_WR_RDMA_READ; endfunction +function Bool isReadRespWorkReq(WorkReqOpCode opcode); + return opcode == IBV_WR_RDMA_READ_RESP; +endfunction + function Bool isAtomicWorkReq(WorkReqOpCode opcode); return workReqHasSwap(opcode); endfunction diff --git a/test/TestPayloadGen.bsv b/test/TestPayloadGen.bsv index 79317d1..3dabe4d 100644 --- a/test/TestPayloadGen.bsv +++ b/test/TestPayloadGen.bsv @@ -239,6 +239,228 @@ module mkTestAddrChunkSrv(Empty); endrule endmodule +(* doc = "testcase" *) +module mkTestDmaReadCntrlScatterGatherListCase(Empty); + let minPayloadLen = 2048; + let maxPayloadLen = 8192; + let qpType = IBV_QPT_XRC_SEND; + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + + Reg#(Bool) clearReg <- mkReg(True); + Reg#(IdxSGL) sglIdxReg <- mkReg(0); + Reg#(TotalFragNum) sglTotalFragNumReg <- mkRegU; + Reg#(TotalFragNum) remainingFragNumReg <- mkReg(0); + Reg#(Length) totalLenReg <- mkRegU; + Reg#(ScatterGatherList) sglReg <- mkRegU; + + FIFOF#(Tuple2#(ScatterGatherList, PMTU)) sglQ <- mkFIFOF; + FIFOF#(TotalFragNum) sglTotalFragNumQ <- mkFIFOF; + + Vector#(1, PipeOut#(NumSGE)) sgeNumPipeOutVec <- + mkRandomValueInRangePipeOut(fromInteger(1), fromInteger(valueOf(MAX_SGE))); + let sgeNumPipeOut = sgeNumPipeOutVec[0]; + + PipeOut#(Bool) randomCancelPipeOut <- mkGenericRandomPipeOut; + PipeOut#(ADDR) startAddrPipeOut <- mkGenericRandomPipeOut; + let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); + let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); + + let simDmaReadSrv <- mkSimDmaReadSrv; + let dut <- mkDmaReadCntrl(clearReg, simDmaReadSrv); + + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + mkSink(dut.sgePktMetaDataPipeOut); + mkSink(dut.sglTotalPayloadLenMetaDataPipeOut); + mkSink(dut.sgeMergedMetaDataPipeOut); + + rule clearAll if (clearReg); + clearReg <= False; + sglIdxReg <= 0; + remainingFragNumReg <= 0; + + sglQ.clear; + sglTotalFragNumQ.clear; + // $display("time=%0t: clearAll", $time); + endrule + + rule genSGE if (!clearReg); + let sgeNum = sgeNumPipeOut.first; + let pmtu = pmtuPipeOut.first; + + let startAddr = startAddrPipeOut.first; + startAddrPipeOut.deq; + + let payloadLen = payloadLenPipeOut.first; + payloadLenPipeOut.deq; + + let sgl = sglReg; + let isFirst = isZero(sglIdxReg); + let isLast = isAllOnesR(sglIdxReg) || (sgeNum - 1 == zeroExtend(sglIdxReg)); + let sge = ScatterGatherElem { + laddr : startAddr, + len : payloadLen, + lkey : dontCareValue, + isFirst: isFirst, + isLast : isLast + }; + sgl[sglIdxReg] = sge; + sglReg <= sgl; + + let { + pmtuLen, firstPktLen, lastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU(startAddr, payloadLen, pmtu); + let { + firstPktFragNum, firstPktByteEn, firstPktFragValidByteNum + } = calcTotalFragNumByLength(zeroExtend(firstPktLen)); + let midPktLen = payloadLen - zeroExtend(firstPktLen); + if (isLessOrEqOne(sgePktNum)) begin + lastPktLen = 0; + end + else begin + midPktLen = payloadLen - zeroExtend(firstPktLen) - zeroExtend(lastPktLen); + end + let { + lastPktFragNum, lastPktByteEn, lastPktFragValidByteNum + } = calcTotalFragNumByLength(zeroExtend(lastPktLen)); + let { + midPktFragNum, midPktByteEn, midPktFragValidByteNum + } = calcTotalFragNumByLength(midPktLen); + let sgeTotalFragNum = firstPktFragNum + midPktFragNum + lastPktFragNum; + + let totalLen = totalLenReg; + let sglTotalfragNum = sglTotalFragNumReg; + if (isFirst) begin + totalLen = payloadLen; + sglTotalfragNum = sgeTotalFragNum; + end + else begin + totalLen = totalLenReg + payloadLen; + sglTotalfragNum = sglTotalFragNumReg + sgeTotalFragNum; + end + totalLenReg <= totalLen; + sglTotalFragNumReg <= sglTotalfragNum; + + if (isLast) begin + sgeNumPipeOut.deq; + pmtuPipeOut.deq; + sglQ.enq(tuple2(sgl, pmtu)); + sglTotalFragNumQ.enq(sglTotalfragNum); + + sglIdxReg <= 0; + end + else begin + sglIdxReg <= sglIdxReg + 1; + end + + // $display( + // "time=%0t: genSGE", $time, + // ", sgeNum=%0d", sgeNum, + // ", sglIdxReg=%0d", sglIdxReg, + // ", startAddr=%h", startAddr, + // ", payloadLen=%0d", payloadLen, + // ", sgeTotalFragNum=%0d", sgeTotalFragNum, + // ", isFirst=", fshow(isFirst), + // ", isLast=", fshow(isLast) + // ); + endrule + + rule issueReq if (!clearReg); + let { sgl, pmtu } = sglQ.first; + sglQ.deq; + + let dmaReadCntrlReq = DmaReadCntrlReq { + pmtu : pmtu, + sglDmaReadMetaData: DmaReadMetaDataSGL { + sgl : sgl, + sqpn : getDefaultQPN, + wrID : dontCareValue + // initiator: DMA_SRC_RQ_RD, + // startAddr: startAddr, + // len : payloadLen, + } + }; + + dut.srvPort.request.put(dmaReadCntrlReq); + countDown.decr; + // $display( + // "time=%0t: issueReq", $time, + // ", startAddr=%h", startAddr, + // ", secondChunkStartAddr=%h", secondChunkStartAddr, + // ", payloadLen=%0d", payloadLen, + // ", pmtu=", fshow(pmtu), + // ", sgePktNum=%0d", sgePktNum, + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen, + // ", pmtuLen=%0d", pmtuLen, + // ", sgeTotalFragNum=%0d", sgeTotalFragNum, + // ", firstPktFragNum=%0d", firstPktFragNum, + // ", lastPktFragNum=%0d", lastPktFragNum, + // ", midPktLen=%0d", midPktLen + // // ", pmtuMask=%h", pmtuMask, + // // ", truncatedPktNum=%0d", truncatedPktNum, + // // ", addrLowPart=%0d", addrLowPart, + // // ", lenLowPart=%0d", lenLowPart, + // // ", tmpLastPktLen=%0d", tmpLastPktLen, + // // ", hasResidue=", fshow(hasResidue), + // // ", hasExtraPkt=", fshow(hasExtraPkt), + // // ", notFullPkt=", fshow(notFullPkt) + // ); + endrule + + rule checkResp if (!clearReg); + let dmaReadCntrlResp <- dut.srvPort.response.get; + let sglFirstFrag = dmaReadCntrlResp.isOrigFirst; + let sglLastFrag = dmaReadCntrlResp.isOrigLast; + let isFirstFrag = dmaReadCntrlResp.dmaReadResp.dataStream.isFirst; + let isLastFrag = dmaReadCntrlResp.dmaReadResp.dataStream.isLast; + + let sglTotalFragNum = sglTotalFragNumQ.first; + let remainingFragNum = remainingFragNumReg; + if (!isLessOrEqOne(sglTotalFragNum)) begin + if (sglFirstFrag) begin + remainingFragNum = sglTotalFragNum - 2; + end + else begin + remainingFragNum = remainingFragNumReg - 1; + end + end + remainingFragNumReg <= remainingFragNum; + + if (sglLastFrag) begin + sglTotalFragNumQ.deq; + + immAssert( + isZero(remainingFragNumReg) && isLastFrag, + "remaining fragNum assertion @ mkTestDmaReadCntrlScatterGatherListCase", + $format( + "remainingFragNumReg=%0d", remainingFragNumReg, + " should be zero when sglFirstFrag=", fshow(sglFirstFrag), + ", sglLastFrag=", fshow(sglLastFrag), + ", isFirstFrag=", fshow(isFirstFrag), + " and isLastFrag=", fshow(isLastFrag) + ) + ); + end + // $display( + // "time=%0t: checkResp", $time, + // ", sglTotalFragNum=%0d", sglTotalFragNum, + // ", remainingFragNum=%0d", remainingFragNum, + // ", remainingFragNumReg=%0d", remainingFragNumReg, + // ", isFirstFrag=", fshow(isFirstFrag), + // ", isLastFrag=", fshow(isLastFrag), + // ", sglFirstFrag=", fshow(sglFirstFrag), + // ", sglLastFrag=", fshow(sglLastFrag) + // ); + endrule +endmodule + (* doc = "testcase" *) module mkTestDmaReadCntrlNormalCase(Empty); let normalOrCancelCase = True; @@ -393,7 +615,7 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); isFinalRespLastFragReg <= dmaReadCntrlResp.dmaReadResp.dataStream.isLast; let isFirstFrag = dmaReadCntrlResp.isOrigFirst && dmaReadCntrlResp.dmaReadResp.dataStream.isFirst; - let isLastFrag = dmaReadCntrlResp.isOrigLast && dmaReadCntrlResp.dmaReadResp.dataStream.isLast; + let isLastFrag = dmaReadCntrlResp.isOrigLast && dmaReadCntrlResp.dmaReadResp.dataStream.isLast; let totalFragNum = totalFragNumQ.first; let remainingFragNum = remainingFragNumReg; @@ -539,7 +761,7 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( payloadLenPipeOut.deq; let isFirst = isZero(sglIdxReg); - let isLast = isAllOnesR(sglIdxReg); + let isLast = isAllOnesR(sglIdxReg); let sge = ScatterGatherElem { laddr : startAddr, len : payloadLen, @@ -1566,32 +1788,32 @@ module mkTestNormalOrSmallPayloadGen#( end endrule - rule recvReqGenTotalMetaData if (stateReg == TEST_ADJUST_PAYLOAD_RECV); - let reqGenTotalMetaData = dut.reqGenTotalMetaDataPipeOut.first; - dut.reqGenTotalMetaDataPipeOut.deq; + rule recvTotalMetaData if (stateReg == TEST_ADJUST_PAYLOAD_RECV); + let totalMetaData = dut.totalMetaDataPipeOut.first; + dut.totalMetaDataPipeOut.deq; immAssert( - totalLenReg == reqGenTotalMetaData.totalLen, + totalLenReg == totalMetaData.totalLen, "totalLen assertion @ mkTestNormalOrSmallPayloadGen", $format( "totalLenReg=%0d", totalLenReg, - " should == reqGenTotalMetaData.totalLen", - reqGenTotalMetaData.totalLen + " should == totalMetaData.totalLen", + totalMetaData.totalLen ) ); immAssert( - remainingPktNumReg == reqGenTotalMetaData.totalPktNum, + remainingPktNumReg == totalMetaData.totalPktNum, "totalPktNum assertion @ mkTestNormalOrSmallPayloadGen", $format( "remainingPktNumReg=%0d", remainingPktNumReg, - " should == reqGenTotalMetaData.totalPktNum", - reqGenTotalMetaData.totalPktNum + " should == totalMetaData.totalPktNum", + totalMetaData.totalPktNum ) ); - isZeroPayloadLenReg <= reqGenTotalMetaData.isZeroPayloadLen; + isZeroPayloadLenReg <= totalMetaData.isZeroPayloadLen; remainingPktNumReg <= remainingPktNumReg - 1; stateReg <= TEST_ADJUST_PAYLOAD_RUN; - // $display("time=%0t: recvReqGenTotalMetaData", $time); + // $display("time=%0t: recvTotalMetaData", $time); endrule rule checkResp if (stateReg == TEST_ADJUST_PAYLOAD_RUN); diff --git a/test/TestSendQ.bsv b/test/TestSendQ.bsv new file mode 100644 index 0000000..54df6c3 --- /dev/null +++ b/test/TestSendQ.bsv @@ -0,0 +1,551 @@ +import BuildVector :: *; +import ClientServer :: *; +import Connectable :: *; +import FIFOF :: *; +import GetPut :: *; +import PAClib :: *; +import Vector :: *; + +import ExtractAndPrependPipeOut :: *; +import DataTypes :: *; +import Headers :: *; +import InputPktHandle :: *; +import MetaData :: *; +import PayloadGen :: *; +import PrimUtils :: *; +import SendQ :: *; +import Settings :: *; +import SimDma :: *; +import Utils :: *; +import Utils4Test :: *; + +module mkSimGenWorkQueueElemByOpCode#( + PipeOut#(WorkReqOpCode) workReqOpCodePipeIn, + // PipeOut#(PSN) psnPipeIn, + Length minLength, + Length maxLength, + TypeQP qpType, + WorkReqSendFlag flags +)(Vector#(vSz, PipeOut#(WorkQueueElem))); + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + + Vector#(1, PipeOut#(NumSGE)) sgeNumPipeOutVec <- + mkRandomValueInRangePipeOut(fromInteger(1), fromInteger(valueOf(MAX_SGE))); + + PipeOut#(WorkReqID) workReqIdPipeOut <- mkGenericRandomPipeOut; + PipeOut#(PSN) psnPipeOut <- mkGenericRandomPipeOut; + PipeOut#(IP) ipAddrPipeOut <- mkGenericRandomPipeOut; + PipeOut#(MAC) macAddrPipeOut <- mkGenericRandomPipeOut; + PipeOut#(Long) compPipeOut <- mkGenericRandomPipeOut; + PipeOut#(Long) swapPipeOut <- mkGenericRandomPipeOut; + PipeOut#(IMM) immDtPipeOut <- mkGenericRandomPipeOut; + PipeOut#(RKEY) rkey2InvPipeOut <- mkGenericRandomPipeOut; + PipeOut#(ADDR) remoteAddrPipeOut <- mkGenericRandomPipeOut; + PipeOut#(ADDR) localAddrPipeOut <- mkGenericRandomPipeOut; + + let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); + let payloadLenPipeOut <- mkRandomLenPipeOut(minLength, maxLength); + + FIFOF#(WorkQueueElem) wqeOutQ <- mkFIFOF; + Vector#(vSz, PipeOut#(WorkQueueElem)) resultPipeOutVec <- + mkForkVector(toPipeOut(wqeOutQ)); + + Reg#(IdxSGL) sglIdxReg <- mkReg(0); + Vector#(MAX_SGE, Reg#(ScatterGatherElem)) sglRegVec <- replicateM(mkRegU); + Reg#(Bool) busyReg <- mkReg(True); + + rule genSGE if (busyReg); + let sgeNum = sgeNumPipeOutVec[0].first; + + let localAddr = localAddrPipeOut.first; + localAddrPipeOut.deq; + + let payloadLen = payloadLenPipeOut.first; + payloadLenPipeOut.deq; + + let isFirst = isZero(sglIdxReg); + let isLast = isAllOnesR(sglIdxReg); // || (sgeNum - 1 == zeroExtend(sglIdxReg)); + let sge = ScatterGatherElem { + laddr : localAddr, + len : payloadLen, + lkey : dontCareValue, + isFirst: isFirst, + isLast : isLast + }; + sglRegVec[sglIdxReg] <= sge; + + if (isLast) begin + sgeNumPipeOutVec[0].deq; + sglIdxReg <= 0; + busyReg <= False; + end + else begin + sglIdxReg <= sglIdxReg + 1; + end + + // $display( + // "time=%0t: genSGE", $time, + // ", sgeNum=%0d", sgeNum, + // ", sglIdxReg=%0d", sglIdxReg, + // ", localAddr=%h", localAddr, + // ", payloadLen=%0d", payloadLen, + // ", isFirst=", fshow(isFirst), + // ", isLast=", fshow(isLast) + // ); + endrule + + rule genWQE if (!busyReg); + let wrID = workReqIdPipeOut.first; + workReqIdPipeOut.deq; + + let wrOpCode = workReqOpCodePipeIn.first; + workReqOpCodePipeIn.deq; + + let psn = psnPipeOut.first; + psnPipeOut.deq; + + let ipAddr = ipAddrPipeOut.first; + ipAddrPipeOut.deq; + let macAddr = macAddrPipeOut.first; + macAddrPipeOut.deq; + + let pmtu = pmtuPipeOut.first; + pmtuPipeOut.deq; + + let remoteAddr = remoteAddrPipeOut.first; + remoteAddrPipeOut.deq; + + let comp = compPipeOut.first; + compPipeOut.deq; + + let swap = swapPipeOut.first; + swapPipeOut.deq; + + let immDt = immDtPipeOut.first; + immDtPipeOut.deq; + + let rkey2Inv = rkey2InvPipeOut.first; + rkey2InvPipeOut.deq; + + let isReadWR = isReadWorkReq(wrOpCode); + let isAtomicWR = isAtomicWorkReq(wrOpCode); + let hasImmDt = workReqHasImmDt(wrOpCode); + let hasInv = workReqHasInv(wrOpCode); + let hasComp = workReqHasComp(wrOpCode); + let hasSwap = workReqHasSwap(wrOpCode); + + QPN sqpn = getDefaultQPN; + QPN srqn = sqpn; // For XRC + QPN dqpn = getDefaultQPN; + QKEY qkey = dontCareValue; + + let sglZeroIdx = 0; + let sgl = readVReg(sglRegVec); + if (isAtomicWR) begin + sgl[sglZeroIdx].isLast = True; + sgl[sglZeroIdx].len = fromInteger(valueOf(ATOMIC_WORK_REQ_LEN)); + end + if (isReadWR) begin + sgl[sglZeroIdx].isLast = True; + end + + let wqe = WorkQueueElem { + id : wrID, + opcode : wrOpCode, + flags : enum2Flag(flags), + qpType : qpType, + psn : psn, + pmtu : pmtu, + dqpIP : ipAddr, + macAddr: macAddr, + sgl : sgl, + raddr : remoteAddr, + rkey : dontCareValue, + // pkey : dontCareValue, + sqpn : sqpn, + dqpn : dqpn, + comp : hasComp ? (tagged Valid comp) : (tagged Invalid), + swap : hasSwap ? (tagged Valid swap) : (tagged Invalid), + immDtOrInvRKey: hasImmDt ? tagged Valid tagged Imm immDt : (hasInv ? tagged Valid tagged RKey rkey2Inv : tagged Invalid), + srqn : tagged Valid srqn, // for XRC + qkey : tagged Valid qkey, // for UD + isFirst: True, + isLast : True + }; + wqeOutQ.enq(wqe); + busyReg <= True; + // $display("time=%0t: generate random WQE=", $time, fshow(wqe)); + endrule + + return resultPipeOutVec; +endmodule + +module mkRandomWorkQueueElemWithOutPayload#( + Length minLength, Length maxLength +)(Vector#(vSz, PipeOut#(WorkQueueElem))); + Vector#(3, WorkReqOpCode) workReqOpCodeVec = vec( + IBV_WR_RDMA_READ, + IBV_WR_ATOMIC_CMP_AND_SWP, + IBV_WR_ATOMIC_FETCH_AND_ADD + ); + let qpType = IBV_QPT_XRC_SEND; + let wrFlags = IBV_SEND_SIGNALED; + let workReqOpCodePipeOut <- mkRandomItemFromVec(workReqOpCodeVec); + let resultPipeOutVec <- mkSimGenWorkQueueElemByOpCode( + workReqOpCodePipeOut, minLength, maxLength, qpType, wrFlags + ); + return resultPipeOutVec; +endmodule + +module mkRandomWorkQueueElemWithPayload#( + Length minLength, Length maxLength +)(Vector#(vSz, PipeOut#(WorkQueueElem))); + Vector#(6, WorkReqOpCode) workReqOpCodeVec = vec( + IBV_WR_SEND, + IBV_WR_SEND_WITH_IMM, + IBV_WR_SEND_WITH_INV, + IBV_WR_RDMA_WRITE, + IBV_WR_RDMA_WRITE_WITH_IMM, + IBV_WR_RDMA_READ_RESP + ); + let qpType = IBV_QPT_XRC_SEND; + let wrFlags = IBV_SEND_SIGNALED; + let workReqOpCodePipeOut <- mkRandomItemFromVec(workReqOpCodeVec); + let resultPipeOutVec <- mkSimGenWorkQueueElemByOpCode( + workReqOpCodePipeOut, minLength, maxLength, qpType, wrFlags + ); + return resultPipeOutVec; +endmodule +/* +module mkRandomWorkQueueElemRawPkt#( + Length minLength, Length maxLength +)(Vector#(vSz, PipeOut#(WorkQueueElem))); + Vector#(6, WorkReqOpCode) workReqOpCodeVec = vec( + IBV_WR_SEND, + IBV_WR_SEND_WITH_IMM, + IBV_WR_SEND_WITH_INV, + IBV_WR_RDMA_WRITE, + IBV_WR_RDMA_WRITE_WITH_IMM, + IBV_WR_RDMA_READ_RESP + ); + let qpType = IBV_QPT_RAW_PACKET; + let wrFlags = IBV_SEND_SIGNALED; + let workReqOpCodePipeOut <- mkRandomItemFromVec(workReqOpCodeVec); + let resultPipeOutVec <- mkSimGenWorkQueueElemByOpCode( + workReqOpCodeVec, minLength, maxLength, qpType, wrFlags + ); + return resultPipeOutVec; +endmodule +*/ +(* doc = "testcase" *) +module mkTestSendQueueNormalCase(Empty); + let minDmaLength = 1; + let maxDmaLength = 10241; + let genPayload = True; + let result <- mkTestSendQueueNormalAndZeroLenCase( + genPayload, minDmaLength, maxDmaLength + ); +endmodule + +(* doc = "testcase" *) +module mkTestSendQueueNoPayloadCase(Empty); + let minDmaLength = 1; + let maxDmaLength = 10241; + let genPayload = False; + let result <- mkTestSendQueueNormalAndZeroLenCase( + genPayload, minDmaLength, maxDmaLength + ); +endmodule + +module mkTestSendQueueNormalAndZeroLenCase#( + Bool genPayload, Length minDmaLength, Length maxDmaLength +)(Empty); + let pmtuVec = vec( + IBV_MTU_256, + IBV_MTU_512, + IBV_MTU_1024, + IBV_MTU_2048, + IBV_MTU_4096 + ); + + Reg#(Bool) clearReg <- mkReg(True); + + // WQE generation + Vector#(2, PipeOut#(WorkQueueElem)) wqePipeOutVec <- genPayload ? + mkRandomWorkQueueElemWithPayload(minDmaLength, maxDmaLength) : + mkRandomWorkQueueElemWithOutPayload(minDmaLength, maxDmaLength); + + // Request payload DataStream generation + // let simDmaReadSrv <- mkSimDmaReadSrvAndDataStreamPipeOut; + // let dataStreamWithPaddingPipeOut <- mkDataStreamAddPadding( + // simDmaReadSrv.dataStream + // ); + // let dataStreamWithPaddingPipeOut4Ref <- mkBufferN(getMaxFragBufSize, dataStreamWithPaddingPipeOut); + + let simDmaReadSrv <- mkSimDmaReadSrv; + let dmaReadCntrl <- mkDmaReadCntrl(clearReg, simDmaReadSrv); + let shouldAddPadding = True; + let payloadGenerator <- mkPayloadGenerator(clearReg, shouldAddPadding, dmaReadCntrl); + + let dut <- mkSendQ(clearReg, payloadGenerator); + mkConnection(dut.wqeInPut, toGet(wqePipeOutVec[0])); + let wqePipeOut4Ref <- mkBufferN(getMaxFragBufSize, wqePipeOutVec[1]); + + // Extract header DataStream, HeaderMetaData and payload DataStream + let headerAndMetaDataAndPayloadPipeOut <- mkExtractHeaderFromRdmaPktPipeOut( + dut.rdmaDataStreamPipeOut + ); + // Convert header DataStream to RdmaHeader + let rdmaHeaderPipeOut <- mkDataStream2Header( + headerAndMetaDataAndPayloadPipeOut.headerAndMetaData.headerDataStream, + headerAndMetaDataAndPayloadPipeOut.headerAndMetaData.headerMetaData + ); + // // Remove empty payload DataStream + // let filteredPayloadDataStreamPipeOut <- mkPipeFilter( + // filterEmptyDataStream, + // headerAndMetaDataAndPayloadPipeOut.payload + // ); + let udpInfoPipeOut4Ref <- mkBufferN(getMaxFragBufSize, dut.udpInfoPipeOut); + FIFOF#(Tuple2#(HeaderByteNum, MAC)) headerLenAndMacAddrQ <- mkFIFOF; + + Reg#(PktLen) payloadLenReg <- mkRegU; + Reg#(PSN) curPsnReg <- mkRegU; + Reg#(Bool) psnIsValidReg <- mkReg(False); + + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + // mkSink(wqePipeOut4Ref); + // mkSink(dut.udpInfoPipeOut); + // mkSink(dut.rdmaDataStreamPipeOut); + // mkSink(filteredPayloadDataStreamPipeOut); + + rule clearAll if (clearReg); + clearReg <= False; + psnIsValidReg <= False; + $display("time=%0t: clearAll", $time); + endrule +/* + rule compareWorkReq; + let pendingWR = pendingWorkReqPipeOut4Comp.first; + pendingWorkReqPipeOut4Comp.deq; + + let refWorkReq = workReqPipeOut4Ref.first; + workReqPipeOut4Ref.deq; + + immAssert( + pendingWR.wr.id == refWorkReq.id && + pendingWR.wr.opcode == refWorkReq.opcode, + "pendingWR.wr assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "pendingWR.wr=", fshow(pendingWR.wr), + " should == refWorkReq=", fshow(refWorkReq) + ) + ); + // $display("time=%0t: WR=", $time, fshow(pendingWR.wr)); + endrule +*/ + rule compareRdmaReqHeader if (!clearReg); + let rdmaHeader = rdmaHeaderPipeOut.first; + rdmaHeaderPipeOut.deq; + + let { transType, rdmaOpCode } = + extractTranTypeAndRdmaOpCode(rdmaHeader.headerData); + let bth = extractBTH(rdmaHeader.headerData); + $display("time=%0t: BTH=", $time, fshow(bth)); + + if (psnIsValidReg) begin + curPsnReg <= curPsnReg + 1; + + immAssert( + bth.psn == curPsnReg, + "bth.psn correctness assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format("bth.psn=%h shoud == curPsnReg=%h", bth.psn, curPsnReg) + ); + end + else begin + curPsnReg <= bth.psn + 1; + end + + let refWQE = wqePipeOut4Ref.first; + let wrStartPSN = refWQE.psn; + headerLenAndMacAddrQ.enq(tuple2( + rdmaHeader.headerMetaData.headerLen, refWQE.macAddr + )); + + if (isOnlyRdmaOpCode(rdmaOpCode)) begin + wqePipeOut4Ref.deq; + // dut.udpInfoPipeOut.deq; + psnIsValidReg <= False; + + // let isReadWR = isReadWorkReq(refWQE.opcode); + // if (isReadWR) begin + immAssert( + bth.psn == wrStartPSN, + "bth.psn read request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "bth.psn=%h should == wrStartPSN=%h when refWQE.opcode=", + bth.psn, wrStartPSN, fshow(refWQE.opcode) + ) + ); + // end + // else begin + // immAssert( + // bth.psn == wrStartPSN && bth.psn == wrEndPSN, + // "bth.psn only request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", + // $format( + // "bth.psn=%h should == wrStartPSN=%h and bth.psn=%h should == wrEndPSN=%h", + // bth.psn, wrStartPSN, bth.psn, wrEndPSN, + // ", when refWQE.opcode=", + // fshow(refWQE.opcode) + // ) + // ); + // end + end + else if (isLastRdmaOpCode(rdmaOpCode)) begin + wqePipeOut4Ref.deq; + psnIsValidReg <= False; + + // immAssert( + // bth.psn == wrEndPSN, + // "bth.psn last request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", + // $format("bth.psn=%h shoud == wrEndPSN=%h", bth.psn, wrEndPSN) + // ); + end + else if (isFirstRdmaOpCode(rdmaOpCode)) begin + // dut.udpInfoPipeOut.deq; + psnIsValidReg <= True; + + immAssert( + bth.psn == wrStartPSN, + "bth.psn first request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format("bth.psn=%h shoud == wrStartPSN=%h", bth.psn, wrStartPSN) + ); + end + else begin + immAssert( + isMiddleRdmaOpCode(rdmaOpCode), + "rdmaOpCode middle request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "rdmaOpCode=", fshow(rdmaOpCode), " should be middle RDMA request opcode" + ) + ); + // immAssert( + // psnInRangeExclusive(bth.psn, wrStartPSN, wrEndPSN), + // "bth.psn between wrStartPSN and wrEndPSN assertion @ mkTestSendQueueNormalAndZeroLenCase", + // $format( + // "bth.psn=%h should > wrStartPSN=%h and bth.psn=%h should < wrEndPSN=%h", + // bth.psn, wrStartPSN, bth.psn, wrEndPSN, + // ", when refWQE.opcode=", fshow(refWQE.opcode), + // " and rdmaOpCode=", fshow(rdmaOpCode) + // ) + // ); + end + + let isRespPkt = True; + immAssert( + transTypeMatchQpType(transType, refWQE.qpType, isRespPkt), + "transTypeMatchQpType assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "transType=", fshow(transType), + " should match qpType=", fshow(refWQE.qpType), + " and isRespPkt=", fshow(isRespPkt) + ) + ); + immAssert( + rdmaReqOpCodeMatchWorkReqOpCode(rdmaOpCode, refWQE.opcode), + "rdmaReqOpCodeMatchWorkReqOpCode assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "RDMA request opcode=", fshow(rdmaOpCode), + " should match workReqOpCode=", fshow(refWQE.opcode) + ) + ); + + // It must compare header not payload, + // since WR might have zero length + countDown.decr; + endrule + + rule checkPktLen if (!clearReg); + let pktFrag = headerAndMetaDataAndPayloadPipeOut.payload.first; + headerAndMetaDataAndPayloadPipeOut.payload.deq; + + let maybePktFragLen = calcFragByteNumFromByteEn(pktFrag.byteEn); + immAssert( + isValid(maybePktFragLen), + "maybePktFragLen assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "isValid(maybePktFragLen)=", fshow(isValid(maybePktFragLen)), + " should be valid" + ) + ); + + let pktFragLen = unwrapMaybe(maybePktFragLen); + let totalPayloadLen = payloadLenReg; + if (pktFrag.isFirst) begin + totalPayloadLen = zeroExtend(pktFragLen); + end + else begin + totalPayloadLen = payloadLenReg + zeroExtend(pktFragLen); + end + payloadLenReg <= totalPayloadLen; + + if (pktFrag.isLast) begin + let udpPktInfo = udpInfoPipeOut4Ref.first; + udpInfoPipeOut4Ref.deq; + + let { headerLen, expectedMacAddr } = headerLenAndMacAddrQ.first; + headerLenAndMacAddrQ.deq; + + immAssert( + udpPktInfo.macAddr == expectedMacAddr, + "macAddr assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "udpPktInfo.macAddr=%h shoud == expectedMacAddr=%h", + udpPktInfo.macAddr, expectedMacAddr + ) + ); + + immAssert( + udpPktInfo.pktLen == totalPayloadLen + zeroExtend(headerLen), + "udpPktInfo.pktLen assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "udpPktInfo.pktLen=%0d", udpPktInfo.pktLen, + " should == totalPayloadLen=%0d", totalPayloadLen, + " + headerLen=%0d", headerLen + ) + ); + $display( + "time=%0t: checkPktLen", $time, + ", udpPktInfo.pktLen=%0d", udpPktInfo.pktLen, + ", totalPayloadLen=%0d", totalPayloadLen, + ", headerLen=%0d", headerLen, + ", udpPktInfo.macAddr=%h", udpPktInfo.macAddr, + ", udpPktInfo.ipAddr=", fshow(udpPktInfo.ipAddr) + ); + end + endrule +/* + rule compareRdmaReqPayload; + let payloadDataStream = filteredPayloadDataStreamPipeOut.first; + filteredPayloadDataStreamPipeOut.deq; + + let refDataStream = dataStreamWithPaddingPipeOut4Ref.first; + dataStreamWithPaddingPipeOut4Ref.deq; + + immAssert( + payloadDataStream == refDataStream, + "payloadDataStream assertion @ mkTestSendQueueNormalAndZeroLenCase", + $format( + "payloadDataStream=", fshow(payloadDataStream), + " should == refDataStream=", fshow(refDataStream) + ) + ); + endrule +*/ +endmodule diff --git a/test/Utils4Test.bsv b/test/Utils4Test.bsv index 4f83e0d..9f47e8b 100644 --- a/test/Utils4Test.bsv +++ b/test/Utils4Test.bsv @@ -505,6 +505,11 @@ function Bool rdmaReqOpCodeMatchWorkReqOpCode(RdmaOpCode rdmaOpCode, WorkReqOpCo SEND_LAST_WITH_INVALIDATE , SEND_ONLY_WITH_INVALIDATE : (wrOpCode == IBV_WR_SEND_WITH_INV); + RDMA_READ_RESPONSE_FIRST , + RDMA_READ_RESPONSE_MIDDLE , + RDMA_READ_RESPONSE_LAST , + RDMA_READ_RESPONSE_ONLY : (wrOpCode == IBV_WR_RDMA_READ_RESP); + default : False; endcase; endfunction From ffab849f1c692bd0d2c8e6d78a65a3361cea0949 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Wed, 31 Jan 2024 00:14:17 +0800 Subject: [PATCH 12/16] add send response to mkSendQ --- run_one.sh | 2 +- src/SendQ.bsv | 50 +++++++++++++++++++++++++++++++--------------- test/TestSendQ.bsv | 25 +++-------------------- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/run_one.sh b/run_one.sh index 6eb1759..eeda42f 100755 --- a/run_one.sh +++ b/run_one.sh @@ -13,7 +13,7 @@ TEST_LOG=run.log TEST_DIR=test cd $TEST_DIR truncate -s 0 $TEST_LOG -FILES=`ls TestPayloadGen.bsv` +FILES=`ls TestSendQ.bsv` for FILE in $FILES; do # echo $FILE TESTCASES=`grep -Phzo 'doc.*?\nmodule\s+\S+(?=\()' $FILE | xargs -0 -I {} echo "{}" | grep module | cut -d ' ' -f 2` diff --git a/src/SendQ.bsv b/src/SendQ.bsv index 8fee920..0f93ddd 100644 --- a/src/SendQ.bsv +++ b/src/SendQ.bsv @@ -553,8 +553,12 @@ typedef struct { Bool isOnlyPkt; } HeaderGenInfo deriving(Bits, FShow); +typedef struct { + Bool hasDmaRespErr; +} SendResp deriving(Bits, FShow); + interface SendQ; - interface Put#(WorkQueueElem) wqeInPut; + interface Server#(WorkQueueElem, SendResp) srvPort; interface PipeOut#(PktInfo4UDP) udpInfoPipeOut; interface DataStreamPipeOut rdmaDataStreamPipeOut; method Bool isEmpty(); @@ -564,18 +568,18 @@ module mkSendQ#( Bool clearAll, PayloadGenerator payloadGenerator )(SendQ); - FIFOF#(WorkQueueElem) wqeInQ <- mkFIFOF; + FIFOF#(WorkQueueElem) reqQ <- mkFIFOF; + FIFOF#(SendResp) respQ <- mkFIFOF; FIFOF#(PktInfo4UDP) udpInfoOutQ <- mkFIFOF; + FIFOF#(PktInfo4UDP) udpPktInfoOutQ <- mkFIFOF; // Pipeline FIFOF FIFOF#(Tuple2#(WorkQueueElem, Bool)) totalMetaDataQ <- mkFIFOF; FIFOF#(Tuple5#(WorkQueueElem, Length, PktNum, Bool, Bool)) psnUpdateQ <- mkFIFOF; FIFOF#(Tuple2#(WorkQueueElem, HeaderGenInfo)) headerPrepareQ <- mkFIFOF; - FIFOF#(Tuple3#(WorkQueueElem, Maybe#(PktHeaderInfo), PktLen)) pendingHeaderQ <- mkFIFOF; - FIFOF#(PktInfo4UDP) udpPktInfoOutQ <- mkFIFOF; + FIFOF#(Tuple4#(WorkQueueElem, Maybe#(PktHeaderInfo), PktLen, Bool)) pendingHeaderQ <- mkFIFOF; FIFOF#(RdmaHeader) pktHeaderQ <- mkFIFOF; - // Reg#(PktNum) remainingPktNumReg <- mkRegU; - // Reg#(Bool) isFirstOrOnlyReqPktReg <- mkReg(True); + Reg#(PSN) curPsnReg <- mkRegU; Reg#(Bool) wqeFirstPktReg <- mkRegU; @@ -591,9 +595,16 @@ module mkSendQ#( (* no_implicit_conditions, fire_when_enabled *) rule resetAndClear if (clearAll); - wqeInQ.clear; + reqQ.clear; + respQ.clear; udpInfoOutQ.clear; + totalMetaDataQ.clear; + psnUpdateQ.clear; + headerPrepareQ.clear; + pendingHeaderQ.clear; + pktHeaderQ.clear; + wqeFirstPktReg <= True; // $display("time=%0t: reset and clear mkSendQ", $time); endrule @@ -604,8 +615,8 @@ module mkSendQ#( prepareHeader, \ genPktHeader" *) rule recvWQE if (!clearAll); - let wqe = wqeInQ.first; - wqeInQ.deq; + let wqe = reqQ.first; + reqQ.deq; let qpType = wqe.qpType; immAssert( @@ -920,7 +931,8 @@ module mkSendQ#( ) ); - pendingHeaderQ.enq(tuple3(wqe, maybePktHeaderInfo, pktLenWithPadCnt)); + let shouldGenResp = isOnlyPkt || isLastPkt; + pendingHeaderQ.enq(tuple4(wqe, maybePktHeaderInfo, pktLenWithPadCnt, shouldGenResp)); // $display( // "time=%0t: mkSendQ 4th stage prepareHeader", $time, // ", sqpn=%h", wqe.sqpn, @@ -934,7 +946,7 @@ module mkSendQ#( endrule rule genPktHeader if (!clearAll); - let { wqe, maybePktHeaderInfo, pktLenWithPadCnt } = pendingHeaderQ.first; + let { wqe, maybePktHeaderInfo, pktLenWithPadCnt, shouldGenResp } = pendingHeaderQ.first; pendingHeaderQ.deq; if (maybePktHeaderInfo matches tagged Valid .pktHeaderInfo) begin @@ -951,6 +963,12 @@ module mkSendQ#( pktHeaderQ.enq(pktHeader); udpPktInfoOutQ.enq(udpPktInfo); + if (shouldGenResp) begin + let sendResp = SendResp { + hasDmaRespErr: False + }; + respQ.enq(sendResp); + end // $display( // "time=%0t: mkSendQ 5th stage genPktHeader", $time, // ", sqpn=%h", wqe.sqpn, @@ -964,18 +982,18 @@ module mkSendQ#( end endrule - interface wqeInPut = toPut(wqeInQ); + interface srvPort = toGPServer(reqQ, respQ); + // interface wqeInPut = toPut(reqQ); interface rdmaDataStreamPipeOut = rdmaPktDataStreamPipeOut; - // interface rdmaDataStreamPipeOut = payloadGenerator.payloadDataStreamPipeOut; interface udpInfoPipeOut = toPipeOut(udpPktInfoOutQ); method Bool isEmpty() = !( - wqeInQ.notEmpty || - udpInfoOutQ.notEmpty || + reqQ.notEmpty || + respQ.notEmpty || + udpPktInfoOutQ.notEmpty || totalMetaDataQ.notEmpty || psnUpdateQ.notEmpty || headerPrepareQ.notEmpty || pendingHeaderQ.notEmpty || - udpPktInfoOutQ.notEmpty || pktHeaderQ.notEmpty ); endmodule diff --git a/test/TestSendQ.bsv b/test/TestSendQ.bsv index 54df6c3..d450878 100644 --- a/test/TestSendQ.bsv +++ b/test/TestSendQ.bsv @@ -21,7 +21,6 @@ import Utils4Test :: *; module mkSimGenWorkQueueElemByOpCode#( PipeOut#(WorkReqOpCode) workReqOpCodePipeIn, - // PipeOut#(PSN) psnPipeIn, Length minLength, Length maxLength, TypeQP qpType, @@ -294,7 +293,7 @@ module mkTestSendQueueNormalAndZeroLenCase#( let payloadGenerator <- mkPayloadGenerator(clearReg, shouldAddPadding, dmaReadCntrl); let dut <- mkSendQ(clearReg, payloadGenerator); - mkConnection(dut.wqeInPut, toGet(wqePipeOutVec[0])); + mkConnection(dut.srvPort.request, toGet(wqePipeOutVec[0])); let wqePipeOut4Ref <- mkBufferN(getMaxFragBufSize, wqePipeOutVec[1]); // Extract header DataStream, HeaderMetaData and payload DataStream @@ -379,8 +378,8 @@ module mkTestSendQueueNormalAndZeroLenCase#( )); if (isOnlyRdmaOpCode(rdmaOpCode)) begin + let sendResp <- dut.srvPort.response.get; wqePipeOut4Ref.deq; - // dut.udpInfoPipeOut.deq; psnIsValidReg <= False; // let isReadWR = isReadWorkReq(refWQE.opcode); @@ -408,6 +407,7 @@ module mkTestSendQueueNormalAndZeroLenCase#( // end end else if (isLastRdmaOpCode(rdmaOpCode)) begin + let sendResp <- dut.srvPort.response.get; wqePipeOut4Ref.deq; psnIsValidReg <= False; @@ -418,7 +418,6 @@ module mkTestSendQueueNormalAndZeroLenCase#( // ); end else if (isFirstRdmaOpCode(rdmaOpCode)) begin - // dut.udpInfoPipeOut.deq; psnIsValidReg <= True; immAssert( @@ -530,22 +529,4 @@ module mkTestSendQueueNormalAndZeroLenCase#( ); end endrule -/* - rule compareRdmaReqPayload; - let payloadDataStream = filteredPayloadDataStreamPipeOut.first; - filteredPayloadDataStreamPipeOut.deq; - - let refDataStream = dataStreamWithPaddingPipeOut4Ref.first; - dataStreamWithPaddingPipeOut4Ref.deq; - - immAssert( - payloadDataStream == refDataStream, - "payloadDataStream assertion @ mkTestSendQueueNormalAndZeroLenCase", - $format( - "payloadDataStream=", fshow(payloadDataStream), - " should == refDataStream=", fshow(refDataStream) - ) - ); - endrule -*/ endmodule From 3ed89b3653fea14fd237d08f048102bd22423913 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Wed, 31 Jan 2024 21:30:03 +0800 Subject: [PATCH 13/16] support raw packet --- Makefile.test | 22 +- run.sh | 18 ++ run_one.sh | 2 +- src/DataTypes.bsv | 11 +- src/ExtractAndPrependPipeOut.bsv | 81 ++++--- src/Headers.bsv | 90 ++++++- src/InputPktHandle.bsv | 4 +- src/PayloadGen.bsv | 12 +- src/PrimUtils.bsv | 4 + src/SendQ.bsv | 184 ++++++++------ src/TransportLayer.bsv | 257 -------------------- src/Utils.bsv | 48 +++- test/TestExtractAndPrependPipeOut.bsv | 7 +- test/TestPayloadGen.bsv | 3 - test/TestReqGenSQ.bsv | 3 + test/TestSendQ.bsv | 335 ++++++++++++++++++-------- test/Utils4Test.bsv | 7 +- 17 files changed, 593 insertions(+), 495 deletions(-) diff --git a/Makefile.test b/Makefile.test index b635a97..386b69a 100644 --- a/Makefile.test +++ b/Makefile.test @@ -20,7 +20,9 @@ TESTBENCHS = \ TestSpecialFIFOF.bsv \ TestTransportLayer.bsv \ TestUtils.bsv \ - TestWorkCompGen.bsv + TestWorkCompGen.bsv \ + TestPayloadGen.bsv \ + TestSendQ.bsv SimDma.bsv = mkTestFixedPktLenDataStreamPipeOut \ mkTestDmaReadAndWriteSrv @@ -103,6 +105,24 @@ TestWorkCompGen.bsv = mkTestWorkCompGenNormalCaseRQ \ mkTestWorkCompGenNormalCaseSQ \ mkTestWorkCompGenErrFlushCaseSQ +TestPayloadGen.bsv = mkTestAddrChunkSrv \ + mkTestDmaReadCntrlScatterGatherListCase \ + mkTestDmaReadCntrlNormalCase \ + mkTestDmaReadCntrlCancelCase \ + mkTestMergeNormalPayloadEachSGE \ + mkTestMergeSmallPayloadEachSGE \ + mkTestMergeNormalPayloadAllSGE \ + mkTestMergeSmallPayloadAllSGE \ + mkTestAdjustNormalPayloadSegmentCase \ + mkTestAdjustSmallPayloadSegmentCase \ + mkTestPayloadGenNormalCase \ + mkTestPayloadGenSmallCase \ + mkTestPayloadGenZeroCase + +TestSendQ.bsv = mkTestSendQueueRawPktCase \ + mkTestSendQueueNormalCase \ + mkTestSendQueueNoPayloadCase + all: $(TESTBENCHS) %.bsv: diff --git a/run.sh b/run.sh index e8032f8..99126ae 100755 --- a/run.sh +++ b/run.sh @@ -116,6 +116,24 @@ mkdir -p $LOG_DIR # make -j8 TESTFILE=TestWorkCompGen.bsv TOPMODULE=mkTestWorkCompGenNormalCaseSQ 2>&1 | tee -a $ALL_LOG # make -j8 TESTFILE=TestWorkCompGen.bsv TOPMODULE=mkTestWorkCompGenErrFlushCaseSQ 2>&1 | tee -a $ALL_LOG +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestAddrChunkSrv +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestDmaReadCntrlScatterGatherListCase +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestDmaReadCntrlNormalCase +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestDmaReadCntrlCancelCase +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestMergeNormalPayloadEachSGE +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestMergeSmallPayloadEachSGE +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestMergeNormalPayloadAllSGE +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestMergeSmallPayloadAllSGE +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestAdjustNormalPayloadSegmentCase +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestAdjustSmallPayloadSegmentCase +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestPayloadGenNormalCase +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestPayloadGenSmallCase +# make -j8 TESTFILE=TestPayloadGen.bsv TOPMODULE=mkTestPayloadGenZeroCase + +# make -j8 TESTFILE=TestSendQ.bsv TOPMODULE=mkTestSendQueueRawPktCase +# make -j8 TESTFILE=TestSendQ.bsv TOPMODULE=mkTestSendQueueNormalCase +# make -j8 TESTFILE=TestSendQ.bsv TOPMODULE=mkTestSendQueueNoPayloadCase + make -j8 -f Makefile.test all TESTDIR=$TEST_DIR LOGDIR=$LOG_DIR cat $LOG_DIR/*.log | tee $ALL_LOG diff --git a/run_one.sh b/run_one.sh index eeda42f..6eb1759 100755 --- a/run_one.sh +++ b/run_one.sh @@ -13,7 +13,7 @@ TEST_LOG=run.log TEST_DIR=test cd $TEST_DIR truncate -s 0 $TEST_LOG -FILES=`ls TestSendQ.bsv` +FILES=`ls TestPayloadGen.bsv` for FILE in $FILES; do # echo $FILE TESTCASES=`grep -Phzo 'doc.*?\nmodule\s+\S+(?=\()' $FILE | xargs -0 -I {} echo "{}" | grep module | cut -d ' ' -f 2` diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index 4e9d7c7..73b0082 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -37,8 +37,10 @@ typedef AETH_VALUE_WIDTH TIMER_WIDTH; typedef TLog#(MAX_SGE) SGE_IDX_WIDTH; typedef TAdd#(1, SGE_IDX_WIDTH) SGE_NUM_WIDTH; -// 12 + 4 + 28 = 44 bytes -typedef TAdd#(TAdd#(BTH_BYTE_WIDTH, XRCETH_BYTE_WIDTH), ATOMIC_ETH_BYTE_WIDTH) HEADER_MAX_BYTE_LENGTH; +// 12 + 4 + 16 + 16 = 48 bytes +typedef TAdd#(TAdd#(BTH_BYTE_WIDTH, XRCETH_BYTE_WIDTH), TAdd#(RETH_BYTE_WIDTH, LETH_BYTE_WIDTH)) HEADER_MAX_BYTE_LENGTH; +// // 12 + 4 + 28 = 44 bytes +// typedef TAdd#(TAdd#(BTH_BYTE_WIDTH, XRCETH_BYTE_WIDTH), ATOMIC_ETH_BYTE_WIDTH) HEADER_MAX_BYTE_LENGTH; typedef TDiv#(DATA_BUS_WIDTH, 8) DATA_BUS_BYTE_WIDTH; // 32 typedef TLog#(DATA_BUS_BYTE_WIDTH) DATA_BUS_BYTE_NUM_WIDTH; // 5 @@ -183,6 +185,7 @@ typedef struct { HeaderFragNum headerFragNum; ByteEnBitNum lastFragValidByteNum; Bool hasPayload; + Bool isEmptyHeader; } HeaderMetaData deriving(Bits, Bounded, Eq); instance FShow#(HeaderMetaData); @@ -196,8 +199,8 @@ endinstance // HeaderData and HeaderByteEn are left aligned typedef struct { - HeaderData headerData; - HeaderByteEn headerByteEn; + HeaderData headerData; + HeaderByteEn headerByteEn; HeaderMetaData headerMetaData; } RdmaHeader deriving(Bits, Bounded, FShow); diff --git a/src/ExtractAndPrependPipeOut.bsv b/src/ExtractAndPrependPipeOut.bsv index b6a730e..100a9a6 100644 --- a/src/ExtractAndPrependPipeOut.bsv +++ b/src/ExtractAndPrependPipeOut.bsv @@ -10,15 +10,13 @@ import PrimUtils :: *; import Settings :: *; import Utils :: *; -function Bool isZeroByteEn(Bit#(nSz) byteEn); // provisos(Add#(1, anysize, nSz)); - return isZero({ msb(byteEn), lsb(byteEn) }); -endfunction - interface HeaderDataStreamAndMetaDataPipeOut; interface DataStreamPipeOut headerDataStream; interface PipeOut#(HeaderMetaData) headerMetaData; endinterface +// If header is empty, then only output headerMetaData, +// and no headerDataStream module mkHeader2DataStream#( Bool clearAll, PipeOut#(RdmaHeader) headerPipeIn @@ -62,7 +60,7 @@ module mkHeader2DataStream#( Bool isFirst = !headerValidReg; Bool isLast = False; - if (isZero(remainingHeaderFragNum)) begin + if (curHeader.headerMetaData.isEmptyHeader || isZero(remainingHeaderFragNum)) begin headerPipeIn.deq; headerValidReg <= False; isLast = True; @@ -77,7 +75,8 @@ module mkHeader2DataStream#( headerLen : remainingHeaderLen, headerFragNum : remainingHeaderFragNum, lastFragValidByteNum: curHeader.headerMetaData.lastFragValidByteNum, - hasPayload : curHeader.headerMetaData.hasPayload + hasPayload : curHeader.headerMetaData.hasPayload, + isEmptyHeader : curHeader.headerMetaData.isEmptyHeader } }; end @@ -99,10 +98,13 @@ module mkHeader2DataStream#( // ", bth.psn=%h, bth.opcode=", bth.psn, fshow(bth.opcode) // ); // end - headerDataStreamOutQ.enq(dataStream); + if (!curHeader.headerMetaData.isEmptyHeader) begin + headerDataStreamOutQ.enq(dataStream); + end let bth = extractBTH(curHeader.headerData); // $display( - // "time=%0t: outputHeader, start output packet", $time, + // "time=%0t: mkHeader2DataStream outputHeader", $time, + // ", start output packet, isEmptyHeader=", fshow(curHeader.headerMetaData.isEmptyHeader), // ", bth.dqpn=%h", bth.dqpn, // ", bth.opcode=", fshow(bth.opcode), // ", bth.psn=%h", bth.psn @@ -231,7 +233,6 @@ typedef enum { // headerLen cannot be zero, but dataPipeIn can have empty DataStream. // If header has no payload, then it will not dequeue dataPipeIn. module mkPrependHeader2PipeOut#( - // CntrlStatus cntrlStatus, Bool clearAll, DataStreamPipeOut headerPipeIn, PipeOut#(HeaderMetaData) headerMetaDataPipeIn, @@ -273,30 +274,48 @@ module mkPrependHeader2PipeOut#( rule popHeaderMetaData if (!clearAll && stageReg == HEADER_META_DATA_POP); let headerMetaData = headerMetaDataPipeIn.first; headerMetaDataPipeIn.deq; - // $display("time=%0t: headerMetaData=", $time, fshow(headerMetaData)); - immAssert( - !isZero(headerMetaData.headerLen), - "headerMetaData.headerLen non-zero assertion @ mkPrependHeader2PipeOut", - $format( - "headerMetaData.headerLen=%h should not be zero", - headerMetaData.headerLen - ) - ); + // $display( + // "time=%0t: mkPrependHeader2PipeOut popHeaderMetaData", $time, + // ", headerMetaData=", fshow(headerMetaData) + // ); let headerFragNum = headerMetaData.headerFragNum; let headerLastFragValidByteNum = headerMetaData.lastFragValidByteNum; let { headerLastFragValidBitNum, headerLastFragInvalidByteNum, headerLastFragInvalidBitNum } = calcFragBitNumAndByteNum(headerLastFragValidByteNum); + if (headerMetaData.isEmptyHeader) begin + immAssert( + isZero(headerLastFragValidBitNum) && isZero(headerLastFragValidByteNum), + "empty header assertion @ mkPrependHeader2PipeOut", + $format( + "headerLastFragValidBitNum=%0d", headerLastFragValidBitNum, + " and headerLastFragValidByteNum=%0d", headerLastFragValidByteNum, + " should be zero when isEmptyHeader=", + fshow(headerMetaData.isEmptyHeader) + ) + ); + end + else begin + immAssert( + !isZero(headerMetaData.headerLen), + "headerMetaData.headerLen non-zero assertion @ mkPrependHeader2PipeOut", + $format( + "headerLen=%0d", headerMetaData.headerLen, + " should not be zero when isEmptyHeader=", + fshow(headerMetaData.isEmptyHeader) + ) + ); + end headerLastFragValidByteNumReg <= headerLastFragValidByteNum; headerLastFragValidBitNumReg <= headerLastFragValidBitNum; headerLastFragInvalidByteNumReg <= headerLastFragInvalidByteNum; headerLastFragInvalidBitNumReg <= headerLastFragInvalidBitNum; - headerFragCntReg <= headerFragNum - 1; + headerFragCntReg <= headerMetaData.isEmptyHeader ? 0 : (headerFragNum - 1); headerHasPayloadReg <= headerMetaData.hasPayload; + stageReg <= headerMetaData.isEmptyHeader ? DATA_OUTPUT : HEADER_OUTPUT; isFirstReg <= True; - stageReg <= HEADER_OUTPUT; endrule rule outputHeader if (!clearAll && stageReg == HEADER_OUTPUT); @@ -332,8 +351,8 @@ module mkPrependHeader2PipeOut#( }; preDataStreamReg <= headerLastFragDataStream; // $display( - // "time=%0t:", $time, - // " headerHasPayloadReg=%b", headerHasPayloadReg, + // "time=%0t: mkPrependHeader2PipeOut outputHeader", $time, + // ", headerHasPayloadReg=%b", headerHasPayloadReg, // ", headerLastFragValidByteNum=%0d", headerLastFragValidByteNumReg, // ", headerLastFragValidBitNum=%0d", headerLastFragValidBitNumReg, // ", headerLastFragInvalidByteNum=%0d", headerLastFragInvalidByteNumReg, @@ -376,14 +395,6 @@ module mkPrependHeader2PipeOut#( isFirst: isFirstReg, isLast: curDataStreamFrag.isLast && noExtraLastFrag }; - // $display( - // "time=%0t", $time, - // " prepend: headerLastFragInvalidByteNumReg=%0d, noExtraLastFrag=%b", - // headerLastFragInvalidByteNumReg, noExtraLastFrag, - // ", preDataStreamReg=", fshow(preDataStreamReg), - // ", curDataStreamFrag=", fshow(curDataStreamFrag), - // ", outDataStream=", fshow(outDataStream) - // ); dataStreamOutQ.enq(outDataStream); if (curDataStreamFrag.isLast) begin @@ -394,6 +405,14 @@ module mkPrependHeader2PipeOut#( stageReg <= EXTRA_LAST_FRAG_OUTPUT; end end + // $display( + // "time=%0t: mkPrependHeader2PipeOut outputData", $time, + // ", headerLastFragInvalidByteNumReg=%0d, noExtraLastFrag=", + // headerLastFragInvalidByteNumReg, fshow(noExtraLastFrag), + // ", preDataStreamReg=", fshow(preDataStreamReg), + // ", curDataStreamFrag=", fshow(curDataStreamFrag), + // ", outDataStream=", fshow(outDataStream) + // ); endrule rule extraLastFrag if (!clearAll && stageReg == EXTRA_LAST_FRAG_OUTPUT); @@ -592,7 +611,7 @@ module mkExtractHeaderFromDataStreamPipeOut#( // "time=%0t:", $time, // " extract headerLastFragValidByteNumReg=%0d", headerLastFragValidByteNumReg, // ", headerLastFragInvalidByteNumReg=%0d", headerLastFragInvalidByteNumReg, - // ", noExtraLastFrag=%b", noExtraLastFrag, + // ", noExtraLastFrag=", fshow(noExtraLastFrag), // ", preDataStreamReg=", fshow(preDataStreamReg), // ", curDataStreamFrag=", fshow(curDataStreamFrag), // ", outDataStream=", fshow(outDataStream), diff --git a/src/Headers.bsv b/src/Headers.bsv index 9e3ecf8..93946c1 100644 --- a/src/Headers.bsv +++ b/src/Headers.bsv @@ -92,8 +92,7 @@ typedef enum { TRANS_TYPE_RD = 3'h2, // 3'b010 TRANS_TYPE_UD = 3'h3, // 3'b011 TRANS_TYPE_CNP = 3'h4, // 3'b100 - TRANS_TYPE_XRC = 3'h5, // 3'b101 - TRANS_TYPE_RAW = 3'h7 // 3'b111 Not defined in rdma-core + TRANS_TYPE_XRC = 3'h5 // 3'b101 } TransType deriving(Bits, Bounded, Eq, FShow); typedef SizeOf#(TransType) TRANS_TYPE_WIDTH; @@ -187,18 +186,28 @@ typedef struct { ADDR va; RKEY rkey; Length dlen; -} RETH deriving(Bits, Bounded); +} RETH deriving(Bits, Bounded, FShow); typedef SizeOf#(RETH) RETH_WIDTH; typedef TDiv#(RETH_WIDTH, 8) RETH_BYTE_WIDTH; +// 16 bytes +typedef struct { + ADDR va; + LKEY lkey; + Length dlen; +} LETH deriving(Bits, Bounded, FShow); + +typedef SizeOf#(LETH) LETH_WIDTH; +typedef TDiv#(LETH_WIDTH, 8) LETH_BYTE_WIDTH; + // 28 bytes typedef struct { ADDR va; RKEY rkey; Long swap; Long comp; -} AtomicEth deriving(Bits, Bounded); +} AtomicEth deriving(Bits, Bounded, FShow); typedef SizeOf#(AtomicEth) ATOMIC_ETH_WIDTH; typedef TDiv#(ATOMIC_ETH_WIDTH, 8) ATOMIC_ETH_BYTE_WIDTH; @@ -206,7 +215,7 @@ typedef TDiv#(ATOMIC_ETH_WIDTH, 8) ATOMIC_ETH_BYTE_WIDTH; // 8 byes typedef struct { Long orig; -} AtomicAckEth deriving(Bits, Bounded); +} AtomicAckEth deriving(Bits, Bounded, FShow); typedef SizeOf#(AtomicAckEth) ATOMIC_ACK_ETH_WIDTH; typedef TDiv#(ATOMIC_ACK_ETH_WIDTH, 8) ATOMIC_ACK_ETH_BYTE_WIDTH; @@ -232,7 +241,7 @@ typedef struct { QKEY qkey; ReservedZero#(8) rsvd; QPN sqpn; -} DETH deriving(Bits, Bounded); +} DETH deriving(Bits, Bounded, FShow); typedef SizeOf#(DETH) DETH_WIDTH; typedef TDiv#(DETH_WIDTH, 8) DETH_BYTE_WIDTH; @@ -241,7 +250,7 @@ typedef TDiv#(DETH_WIDTH, 8) DETH_BYTE_WIDTH; typedef struct { ReservedZero#(8) rsvd; QPN srqn; -} XRCETH deriving(Bits, Bounded); +} XRCETH deriving(Bits, Bounded, FShow); typedef SizeOf#(XRCETH) XRCETH_WIDTH; typedef TDiv#(XRCETH_WIDTH, 8) XRCETH_BYTE_WIDTH; @@ -250,13 +259,14 @@ typedef TDiv#(XRCETH_WIDTH, 8) XRCETH_BYTE_WIDTH; typedef struct { ReservedZero#(LONG_WIDTH) rsvd1; ReservedZero#(LONG_WIDTH) rsvd2; -} PayloadCNP deriving(Bits, Bounded); +} PayloadCNP deriving(Bits, Bounded, FShow); typedef SizeOf#(PayloadCNP) CNP_PAYLOAD_WIDTH; typedef TDiv#(CNP_PAYLOAD_WIDTH, 8) CNP_PAYLOAD_BYTE_WIDTH; // RC headers: // BTH + IETH = 20 bytes +// BTH + RETH + LETH = 44 bytes // BTH + RETH + ImmDT = 32 bytes // BTH + AtomicEth = 40 bytes // BTH + AETH + AtomicAckEth = 24 bytes @@ -264,6 +274,7 @@ typedef TDiv#(CNP_PAYLOAD_WIDTH, 8) CNP_PAYLOAD_BYTE_WIDTH; // XRC headers: // BTH + XRCETH + IETH = 24 bytes // BTH + XRCETH + RETH = 32 bytes +// BTH + XRCETH + RETH + LETH = 48 bytes // BTH + XRCETH + RETH + ImmDT = 36 bytes // BTH + XRCETH + AtomicEth = 44 bytes // BTH + AETH + AtomicAckEth = 24 bytes @@ -328,10 +339,71 @@ function Integer calcHeaderLenByTransTypeAndRdmaOpCode( // CNP notification fromInteger(valueOf(ROCE_CNP)): valueOf(BTH_BYTE_WIDTH) + valueOf(CNP_PAYLOAD_BYTE_WIDTH); - default: 0; // error("invalid transType and rdmaOpCode in calcHeaderLenByTransTypeAndRdmaOpCode()"); + default: 0; endcase; endfunction +/* +// TODO: enable this after refactor +function Integer calcHeaderLenByTransTypeAndRdmaOpCode( + TransType transType, RdmaOpCode rdmaOpCode +); + return case ({ pack(transType), pack(rdmaOpCode) } ) + // RC requests + fromInteger(valueOf(RC_SEND_FIRST)) : valueOf(BTH_BYTE_WIDTH); + fromInteger(valueOf(RC_SEND_MIDDLE)) : valueOf(BTH_BYTE_WIDTH); + fromInteger(valueOf(RC_SEND_LAST)) : valueOf(BTH_BYTE_WIDTH); + fromInteger(valueOf(RC_SEND_LAST_WITH_IMMEDIATE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + fromInteger(valueOf(RC_SEND_ONLY)) : valueOf(BTH_BYTE_WIDTH); + fromInteger(valueOf(RC_SEND_ONLY_WITH_IMMEDIATE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_WRITE_FIRST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_WRITE_MIDDLE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_WRITE_LAST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_WRITE_LAST_WITH_IMMEDIATE)): valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_WRITE_ONLY)) : valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE)): valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_READ_REQUEST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(LETH_BYTE_WIDTH); + fromInteger(valueOf(RC_COMPARE_SWAP)) : valueOf(BTH_BYTE_WIDTH) + valueOf(ATOMIC_ETH_BYTE_WIDTH); + fromInteger(valueOf(RC_FETCH_ADD)) : valueOf(BTH_BYTE_WIDTH) + valueOf(ATOMIC_ETH_BYTE_WIDTH); + fromInteger(valueOf(RC_SEND_LAST_WITH_INVALIDATE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(IETH_BYTE_WIDTH); + fromInteger(valueOf(RC_SEND_ONLY_WITH_INVALIDATE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(IETH_BYTE_WIDTH); + // RC and XRC responses + fromInteger(valueOf(RC_RDMA_READ_RESPONSE_FIRST)), fromInteger(valueOf(XRC_RDMA_READ_RESPONSE_FIRST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_READ_RESPONSE_MIDDLE)), fromInteger(valueOf(XRC_RDMA_READ_RESPONSE_MIDDLE)): valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_READ_RESPONSE_LAST)), fromInteger(valueOf(XRC_RDMA_READ_RESPONSE_LAST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(RC_RDMA_READ_RESPONSE_ONLY)), fromInteger(valueOf(XRC_RDMA_READ_RESPONSE_ONLY)) : valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(RC_ACKNOWLEDGE)), fromInteger(valueOf(XRC_ACKNOWLEDGE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(AETH_BYTE_WIDTH); + fromInteger(valueOf(RC_ATOMIC_ACKNOWLEDGE)), fromInteger(valueOf(XRC_ATOMIC_ACKNOWLEDGE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(AETH_BYTE_WIDTH) + valueOf(ATOMIC_ACK_ETH_BYTE_WIDTH); + + // XRC requests + fromInteger(valueOf(XRC_SEND_FIRST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_SEND_MIDDLE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_SEND_LAST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_SEND_LAST_WITH_IMMEDIATE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + fromInteger(valueOf(XRC_SEND_ONLY)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_SEND_ONLY_WITH_IMMEDIATE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + fromInteger(valueOf(XRC_RDMA_WRITE_FIRST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_RDMA_WRITE_MIDDLE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_RDMA_WRITE_LAST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_RDMA_WRITE_LAST_WITH_IMMEDIATE)): valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + fromInteger(valueOf(XRC_RDMA_WRITE_ONLY)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_RDMA_WRITE_ONLY_WITH_IMMEDIATE)): valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + fromInteger(valueOf(XRC_RDMA_READ_REQUEST)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(LETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_COMPARE_SWAP)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(ATOMIC_ETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_FETCH_ADD)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(ATOMIC_ETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_SEND_LAST_WITH_INVALIDATE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IETH_BYTE_WIDTH); + fromInteger(valueOf(XRC_SEND_ONLY_WITH_INVALIDATE)) : valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(IETH_BYTE_WIDTH); + + // UD requests + fromInteger(valueOf(UD_SEND_ONLY)) : valueOf(BTH_BYTE_WIDTH) + valueOf(DETH_BYTE_WIDTH); + fromInteger(valueOf(UD_SEND_ONLY_WITH_IMMEDIATE)): valueOf(BTH_BYTE_WIDTH) + valueOf(DETH_BYTE_WIDTH) + valueOf(IMM_DT_BYTE_WIDTH); + + // CNP notification + fromInteger(valueOf(ROCE_CNP)): valueOf(BTH_BYTE_WIDTH) + valueOf(CNP_PAYLOAD_BYTE_WIDTH); + default: 0; + endcase; +endfunction +*/ function Bool rdmaOpCodeHasPayload(RdmaOpCode opcode); return case (opcode) SEND_FIRST , diff --git a/src/InputPktHandle.bsv b/src/InputPktHandle.bsv index 1593ad4..64423a5 100644 --- a/src/InputPktHandle.bsv +++ b/src/InputPktHandle.bsv @@ -370,7 +370,7 @@ module mkInputRdmaPktBufAndHeaderValidation#( let payloadFrag = payloadPipeIn.first; payloadPipeIn.deq; let payloadHasSingleFrag = payloadFrag.isFirst && payloadFrag.isLast; - let fragHasNoData = isZero(payloadFrag.byteEn); + let fragHasNoData = isZeroByteEn(payloadFrag.byteEn); if (payloadFrag.isFirst) begin let rdmaHeader = rdmaHeaderPipeOut.first; @@ -662,7 +662,7 @@ module mkInputRdmaPktBufAndHeaderValidation#( ) ); let fragLen = unwrapMaybe(payloadFragLen); - let isByteEnNonZero = !isZeroR(fragLen); + let isByteEnNonZero = !isZeroByteEn(payloadFrag.byteEn); let isByteEnAllOne = isAllOnesR(payloadFrag.byteEn); ByteEnBitNum fragLenWithOutPad = fragLen - zeroExtend(bthPadCnt); diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index f067d4c..4377a6b 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -1792,7 +1792,6 @@ typedef struct { PAD padCnt; Bool isFirst; Bool isLast; - Bool isRespErr; // TODO: handle DMA read error } PayloadGenRespSG deriving(Bits, FShow); typedef struct { @@ -1931,8 +1930,6 @@ module mkPayloadGenerator#( rule recvDmaReadCntrlResp if (!clearAll); let dmaReadCntrlResp <- dmaReadCntrl.srvPort.response.get; sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); - // TODO: handle DMA read error - let hasDmaRespErr = dmaReadCntrlResp.dmaReadResp.isRespErr; endrule rule adjustFirstAndLastPktLen if (!clearAll); @@ -2118,8 +2115,7 @@ module mkPayloadGenerator#( pktLen : 0, padCnt : 0, isFirst : True, - isLast : True, - isRespErr : False + isLast : True }; payloadGenRespQ.enq(payloadGenResp); end @@ -2148,15 +2144,15 @@ module mkPayloadGenerator#( pktRemoteAddrReg <= nextRemoteAddr; remainingPktNumReg <= remainingPktNum; end + // Generate response by the end of the payload // Every segmented payload has a payloadGenResp - if (curPayloadFrag.isFirst) begin + if (curPayloadFrag.isLast) begin let payloadGenResp = PayloadGenRespSG { raddr : remoteAddr, pktLen : pktLen, padCnt : padCnt, isFirst : isFirstPkt, - isLast : isLastPkt, - isRespErr : False + isLast : isLastPkt }; payloadGenRespQ.enq(payloadGenResp); // $display( diff --git a/src/PrimUtils.bsv b/src/PrimUtils.bsv index 63aaf3b..7672f57 100644 --- a/src/PrimUtils.bsv +++ b/src/PrimUtils.bsv @@ -23,6 +23,10 @@ function Bool isZeroR(Bit#(nSz) bits) provisos( end endfunction +function Bool isZeroByteEn(Bit#(nSz) byteEn); // provisos(Add#(1, anysize, nSz)); + return isZero({ msb(byteEn), lsb(byteEn) }); +endfunction + function Bool isLessOrEqOne(Bit#(nSz) bits); // provisos(Add#(1, anysize, nSz)); Bool ret = isZero(bits >> 1); // Bool ret = isZero(bits >> 1) && unpack(bits[0]); diff --git a/src/SendQ.bsv b/src/SendQ.bsv index 0f93ddd..fb432c7 100644 --- a/src/SendQ.bsv +++ b/src/SendQ.bsv @@ -8,6 +8,7 @@ import ExtractAndPrependPipeOut :: *; import Headers :: *; import PayloadGen :: *; import PrimUtils :: *; +import Reserved :: *; import Settings :: *; import Utils :: *; @@ -24,16 +25,23 @@ typedef struct { Bool hasHeader; } PktHeaderInfo deriving(Bits, FShow); -// TODO: support hasHeader function PktHeaderInfo genPktHeaderInfo( HeaderData headerData, HeaderByteNum headerLen, Bool hasPayload ); - let hasHeader = True; return PktHeaderInfo { headerData: headerData, headerLen : headerLen, hasPayload: hasPayload, - hasHeader : hasHeader + hasHeader : True + }; +endfunction + +function PktHeaderInfo genEmptyPktHeaderInfo(Bool hasPayload); + return PktHeaderInfo { + headerData: dontCareValue, + headerLen : 0, + hasPayload: hasPayload, + hasHeader : False }; endfunction @@ -100,6 +108,18 @@ function Maybe#(RETH) genRETH( endcase; endfunction +function Maybe#(LETH) genLETH(WorkQueueElem wqe, Length dlen); + let firstIdxSGE = 0; + return case (wqe.opcode) + IBV_WR_RDMA_READ: tagged Valid LETH { + va : wqe.sgl[firstIdxSGE].laddr, + lkey: wqe.sgl[firstIdxSGE].lkey, + dlen: dlen + }; + default : tagged Invalid; + endcase; +endfunction + // TODO: check fetch add needs both swap and comp? function Maybe#(AtomicEth) genAtomicEth(WorkQueueElem wqe); if (wqe.swap matches tagged Valid .swap &&& wqe.comp matches tagged Valid .comp) begin @@ -136,13 +156,6 @@ function Maybe#(ImmDt) genImmDt(WorkQueueElem wqe); end default : return tagged Invalid; endcase - // return case (wqe.opcode) - // IBV_WR_RDMA_WRITE_WITH_IMM, - // IBV_WR_SEND_WITH_IMM : tagged Valid ImmDt { - // data: unwrapMaybe(wqe.immDtOrInvRKey) - // }; - // default : tagged Invalid; - // endcase; endfunction function Maybe#(IETH) genIETH(WorkQueueElem wqe); @@ -158,12 +171,6 @@ function Maybe#(IETH) genIETH(WorkQueueElem wqe); else begin return tagged Invalid; end - // return case (wqe.opcode) - // IBV_WR_SEND_WITH_INV: tagged Valid IETH { - // rkey: unwrapMaybe(wqe.immDtOrInvRKey) - // }; - // default : tagged Invalid; - // endcase; endfunction function Maybe#(PktHeaderInfo) genFirstOrOnlyPktHeader( @@ -198,6 +205,7 @@ function Maybe#(PktHeaderInfo) genFirstOrOnlyPktHeader( let xrceth = genXRCETH(wqe); let deth = genDETH(wqe); let reth = genRETH(wqe.opcode, remoteAddr, wqe.rkey, dlen); + let leth = genLETH(wqe, dlen); let atomicEth = genAtomicEth(wqe); let immDt = genImmDt(wqe); let ieth = genIETH(wqe); @@ -322,13 +330,13 @@ function Maybe#(PktHeaderInfo) genFirstOrOnlyPktHeader( IBV_WR_RDMA_READ: begin return case (wqe.qpType) IBV_QPT_RC: tagged Valid genPktHeaderInfo( - zeroExtendLSB({ pack(bth), pack(unwrapMaybe(reth)) }), - fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(reth)), pack(unwrapMaybe(leth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), False // Read requests have no payload ); IBV_QPT_XRC_SEND: tagged Valid genPktHeaderInfo( - zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(reth)) }), - fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), + zeroExtendLSB({ pack(bth), pack(unwrapMaybe(xrceth)), pack(unwrapMaybe(reth)), pack(unwrapMaybe(leth)) }), + fromInteger(valueOf(BTH_BYTE_WIDTH) + valueOf(XRCETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH) + valueOf(RETH_BYTE_WIDTH)), False // Read requests have no payload ); default: tagged Invalid; @@ -551,10 +559,11 @@ typedef struct { Bool isFirstPkt; Bool isLastPkt; Bool isOnlyPkt; + Bool qpRawPkt; } HeaderGenInfo deriving(Bits, FShow); typedef struct { - Bool hasDmaRespErr; + ReservedZero#(0) rsvd; } SendResp deriving(Bits, FShow); interface SendQ; @@ -574,10 +583,10 @@ module mkSendQ#( FIFOF#(PktInfo4UDP) udpPktInfoOutQ <- mkFIFOF; // Pipeline FIFOF - FIFOF#(Tuple2#(WorkQueueElem, Bool)) totalMetaDataQ <- mkFIFOF; - FIFOF#(Tuple5#(WorkQueueElem, Length, PktNum, Bool, Bool)) psnUpdateQ <- mkFIFOF; + FIFOF#(Tuple3#(WorkQueueElem, Bool, Bool)) totalMetaDataQ <- mkFIFOF; + FIFOF#(Tuple6#(WorkQueueElem, Length, PktNum, Bool, Bool, Bool)) psnUpdateQ <- mkFIFOF; FIFOF#(Tuple2#(WorkQueueElem, HeaderGenInfo)) headerPrepareQ <- mkFIFOF; - FIFOF#(Tuple4#(WorkQueueElem, Maybe#(PktHeaderInfo), PktLen, Bool)) pendingHeaderQ <- mkFIFOF; + FIFOF#(Tuple6#(MAC, IP, Maybe#(PktHeaderInfo), PktLen, Bool, Bool)) pendingHeaderQ <- mkFIFOF; FIFOF#(RdmaHeader) pktHeaderQ <- mkFIFOF; Reg#(PSN) curPsnReg <- mkRegU; @@ -620,8 +629,12 @@ module mkSendQ#( let qpType = wqe.qpType; immAssert( - qpType == IBV_QPT_RC || qpType == IBV_QPT_UC || - qpType == IBV_QPT_XRC_SEND || qpType == IBV_QPT_UD, + qpType == IBV_QPT_RC || + qpType == IBV_QPT_XRC_RECV || + qpType == IBV_QPT_XRC_SEND || + qpType == IBV_QPT_UC || + qpType == IBV_QPT_UD || + qpType == IBV_QPT_RAW_PACKET, "qpType assertion @ mkSendQ", $format( "qpType=", fshow(qpType), " unsupported" @@ -704,32 +717,39 @@ module mkSendQ#( // TODO: check immDtOrInvRKey is RKey end + let qpRawPkt = isRawPktTypeQP(wqe.qpType); let isSendWR = isSendWorkReq(wqe.opcode); - let needPayloadGen = workReqNeedPayloadGen(wqe.opcode); + let needPayloadGen = qpRawPkt || workReqNeedPayloadGen(wqe.opcode); + + let remoteAddr = wqe.raddr; + if (qpRawPkt || isSendWR) begin + remoteAddr = 0; + end if (needPayloadGen) begin let payloadGenReq = PayloadGenReqSG { wrID : wqe.id, sqpn : wqe.sqpn, sgl : wqe.sgl, - raddr: isSendWR ? 0 : wqe.raddr, + raddr: remoteAddr, pmtu : wqe.pmtu }; payloadGenerator.srvPort.request.put(payloadGenReq); end - totalMetaDataQ.enq(tuple2(wqe, needPayloadGen)); + totalMetaDataQ.enq(tuple3(wqe, qpRawPkt, needPayloadGen)); // TODO: handle pending read/atomic request number limit // $display( // "time=%0t: mkSendQ 1st stage recvWQE", $time, // ", sqpn=%h", wqe.sqpn, // ", id=%h", wqe.id, + // ", macAddr=%h", wqe.macAddr, // ", pmtu=", fshow(wqe.pmtu), // ", needPayloadGen=", fshow(needPayloadGen) // ); endrule rule recvTotalMetaData if (!clearAll); - let { wqe, needPayloadGen } = totalMetaDataQ.first; + let { wqe, qpRawPkt, needPayloadGen } = totalMetaDataQ.first; totalMetaDataQ.deq; let sglZeroIdx = 0; @@ -773,25 +793,26 @@ module mkSendQ#( ); end end - psnUpdateQ.enq(tuple5( - wqe, totalLen, totalPktNum, hasPayload, isOnlyPkt + psnUpdateQ.enq(tuple6( + wqe, totalLen, totalPktNum, hasPayload, isOnlyPkt, qpRawPkt )); - $display( - "time=%0t: mkSendQ 2nd stage recvTotalMetaData", $time, - ", sqpn=%h", wqe.sqpn, - ", id=%h", wqe.id, - ", psn=%h", wqe.psn, - ", totalLen=%0d", totalLen, - ", totalPktNum=%0d", totalPktNum, - ", hasPayload=", fshow(hasPayload), - ", isOnlyPkt=", fshow(isOnlyPkt), - ", needPayloadGen=", fshow(needPayloadGen) - ); + // $display( + // "time=%0t: mkSendQ 2nd stage recvTotalMetaData", $time, + // ", sqpn=%h", wqe.sqpn, + // ", id=%h", wqe.id, + // ", psn=%h", wqe.psn, + // ", totalLen=%0d", totalLen, + // ", totalPktNum=%0d", totalPktNum, + // ", qpRawPkt=", fshow(qpRawPkt), + // ", hasPayload=", fshow(hasPayload), + // ", isOnlyPkt=", fshow(isOnlyPkt), + // ", needPayloadGen=", fshow(needPayloadGen) + // ); endrule rule updatePSN if (!clearAll); let { - wqe, totalLen, totalPktNum, hasPayload, isOnlyPkt + wqe, totalLen, totalPktNum, hasPayload, isOnlyPkt, qpRawPkt } = psnUpdateQ.first; let curPSN = curPsnReg; @@ -806,10 +827,10 @@ module mkSendQ#( let wqeLastPkt = isOnlyPkt; if (hasPayload) begin let payloadGenResp <- payloadGenerator.srvPort.response.get; - remoteAddr = payloadGenResp.raddr; + remoteAddr = payloadGenResp.raddr; pktPayloadLen = payloadGenResp.pktLen; - padCnt = payloadGenResp.padCnt; - wqeLastPkt = payloadGenResp.isLast; + padCnt = payloadGenResp.padCnt; + wqeLastPkt = payloadGenResp.isLast; end wqeFirstPktReg <= wqeLastPkt; @@ -819,7 +840,6 @@ module mkSendQ#( let isFirstPkt = wqeFirstPktReg && wqe.isFirst; let isLastPkt = wqeLastPkt && wqe.isLast; - if (!isLastPkt) begin immAssert( !isOnlyPkt, @@ -843,6 +863,19 @@ module mkSendQ#( ); end + if (qpRawPkt) begin + immAssert( + hasPayload && isZero(padCnt), + "qpRawPkt assertion @ mkSendQ", + $format( + "hasPayload=", fshow(hasPayload), + " should be true, and pacCnt=%0d", padCnt, + " should be zero when qpRawPkt=", fshow(qpRawPkt), + " and wqe.qpType=", fshow(wqe.qpType) + ) + ); + end + let ackReq = containWorkReqFlag(wqe.flags, IBV_SEND_SIGNALED); let solicited = containWorkReqFlag(wqe.flags, IBV_SEND_SOLICITED); let headerGenInfo = HeaderGenInfo { @@ -856,7 +889,8 @@ module mkSendQ#( solicited : solicited, isFirstPkt: isFirstPkt, isLastPkt : isLastPkt, - isOnlyPkt : isOnlyPkt + isOnlyPkt : isOnlyPkt, + qpRawPkt : qpRawPkt }; headerPrepareQ.enq(tuple2(wqe, headerGenInfo)); // $display( @@ -887,11 +921,16 @@ module mkSendQ#( let isFirstPkt = headerGenInfo.isFirstPkt; let isLastPkt = headerGenInfo.isLastPkt; let isOnlyPkt = headerGenInfo.isOnlyPkt; + let qpRawPkt = headerGenInfo.qpRawPkt; - let maybePktHeaderInfo = dontCareValue; - if (isFirstPkt) begin + let maybePktHeaderInfo = dontCareValue; + if (qpRawPkt) begin + maybePktHeaderInfo = tagged Valid genEmptyPktHeaderInfo(hasPayload); + end + else if (isFirstPkt) begin let maybeFirstOrOnlyPktHeaderInfo = genFirstOrOnlyPktHeader( - wqe, isOnlyPkt, solicited, curPSN, padCnt, ackReq, remoteAddr, totalLen, hasPayload + wqe, isOnlyPkt, solicited, curPSN, padCnt, + ackReq, remoteAddr, totalLen, hasPayload ); immAssert( isValid(maybeFirstOrOnlyPktHeaderInfo), @@ -906,7 +945,8 @@ module mkSendQ#( end else begin let maybeMiddleOrLastPktHeaderInfo = genMiddleOrLastPktHeader( - wqe, isLastPkt, solicited, curPSN, padCnt, ackReq, remoteAddr, pktPayloadLen + wqe, isLastPkt, solicited, curPSN, padCnt, + ackReq, remoteAddr, pktPayloadLen ); immAssert( isValid(maybeMiddleOrLastPktHeaderInfo), @@ -921,18 +961,22 @@ module mkSendQ#( end let pktLenWithPadCnt = pktPayloadLen + zeroExtend(padCnt); + PAD zeroPad = truncate(pktLenWithPadCnt); immAssert( - isZero(pktLenWithPadCnt[1:0]), - "pktLenWithPadCnt assertion @ mkSendQ", + isZero(zeroPad), + "zeroPad assertion @ mkSendQ", $format( - "pktLenWithPadCnt=%0d", pktLenWithPadCnt, - " should be multiple of four, when padCnt=%0d", padCnt, - " pktPayloadLen=%0d", pktPayloadLen + "zeroPad=%0d", zeroPad, + " should be zero, when padCnt=%0d", padCnt, + ", pktPayloadLen=%0d", pktPayloadLen, + " and pktLenWithPadCnt=%0d", pktLenWithPadCnt ) ); - let shouldGenResp = isOnlyPkt || isLastPkt; - pendingHeaderQ.enq(tuple4(wqe, maybePktHeaderInfo, pktLenWithPadCnt, shouldGenResp)); + let isSendDone = isOnlyPkt || isLastPkt; + pendingHeaderQ.enq(tuple6( + wqe.macAddr, wqe.dqpIP, maybePktHeaderInfo, pktLenWithPadCnt, qpRawPkt, isSendDone + )); // $display( // "time=%0t: mkSendQ 4th stage prepareHeader", $time, // ", sqpn=%h", wqe.sqpn, @@ -946,33 +990,37 @@ module mkSendQ#( endrule rule genPktHeader if (!clearAll); - let { wqe, maybePktHeaderInfo, pktLenWithPadCnt, shouldGenResp } = pendingHeaderQ.first; + let { + macAddr, ipAddr, maybePktHeaderInfo, pktLenWithPadCnt, qpRawPkt, isSendDone + } = pendingHeaderQ.first; pendingHeaderQ.deq; if (maybePktHeaderInfo matches tagged Valid .pktHeaderInfo) begin let headerData = pktHeaderInfo.headerData; let headerLen = pktHeaderInfo.headerLen; let hasPayload = pktHeaderInfo.hasPayload; - let pktHeader = genRdmaHeader(headerData, headerLen, hasPayload); + let pktHeader = qpRawPkt ? + genEmptyRdmaHeader(hasPayload) : + genRdmaHeader(headerData, headerLen, hasPayload); let udpPktInfo = PktInfo4UDP { - macAddr: wqe.macAddr, - ipAddr : wqe.dqpIP, + macAddr: macAddr, + ipAddr : ipAddr, pktLen : pktLenWithPadCnt + zeroExtend(headerLen) }; pktHeaderQ.enq(pktHeader); udpPktInfoOutQ.enq(udpPktInfo); - if (shouldGenResp) begin + if (isSendDone) begin let sendResp = SendResp { - hasDmaRespErr: False + rsvd: unpack(0) }; respQ.enq(sendResp); end // $display( // "time=%0t: mkSendQ 5th stage genPktHeader", $time, - // ", sqpn=%h", wqe.sqpn, - // ", id=%h", wqe.id, + // // ", sqpn=%h", wqe.sqpn, + // // ", id=%h", wqe.id, // ", pktLenWithPadCnt=%0d", pktLenWithPadCnt, // ", headerLen=%0d", headerLen, // ", udpPktInfo.macAddr=%h", udpPktInfo.macAddr, diff --git a/src/TransportLayer.bsv b/src/TransportLayer.bsv index 765b2ff..c0ebd9c 100644 --- a/src/TransportLayer.bsv +++ b/src/TransportLayer.bsv @@ -55,264 +55,7 @@ module mkWorkReqAndRecvReqDispatcher#( map(toPipeOut, recvReqOutVec) ); endmodule -/* -// TODO: remove it -module mkSimExtractNormalReqResp#( - MetaDataQPs qpMetaData, - DataStreamPipeOut rdmaPktPipeIn -)(Vector#(MAX_QP, InputRdmaPktBuf)); - // Output FIFO for PipeOut - Vector#(MAX_QP, FIFOF#(BTH)) cnpOutVec <- replicateM(mkFIFOF); - Vector#(MAX_QP, FIFOF#(DataStream)) reqPayloadOutVec <- replicateM(mkFIFOF); - Vector#(MAX_QP, FIFOF#(RdmaPktMetaData)) reqPktMetaDataOutVec <- replicateM(mkFIFOF); - Vector#(MAX_QP, FIFOF#(DataStream)) respPayloadOutVec <- replicateM(mkFIFOF); - Vector#(MAX_QP, FIFOF#(RdmaPktMetaData)) respPktMetaDataOutVec <- replicateM(mkFIFOF); - Reg#(RdmaHeader) rdmaHeaderReg <- mkRegU; - Reg#(PktFragNum) pktFragNumReg <- mkRegU; - Reg#(PktLen) pktLenReg <- mkRegU; - Reg#(Bool) pktValidReg <- mkRegU; - - let headerAndMetaDataAndPayloadPipeOut <- mkExtractHeaderFromRdmaPktPipeOut( - rdmaPktPipeIn - ); - let payloadPipeIn <- mkBuffer(headerAndMetaDataAndPayloadPipeOut.payload); - let rdmaHeaderPipeOut <- mkDataStream2Header( - headerAndMetaDataAndPayloadPipeOut.headerAndMetaData.headerDataStream, - headerAndMetaDataAndPayloadPipeOut.headerAndMetaData.headerMetaData - ); - - rule extractHeader; - let payloadFrag = payloadPipeIn.first; - payloadPipeIn.deq; - - let rdmaHeader = rdmaHeaderReg; - if (payloadFrag.isFirst) begin - rdmaHeader = rdmaHeaderPipeOut.first; - rdmaHeaderPipeOut.deq; - rdmaHeaderReg <= rdmaHeader; - end - let bth = extractBTH(rdmaHeader.headerData); - let aeth = extractAETH(rdmaHeader.headerData); - let deth = extractDETH(rdmaHeader.headerData); - let xrceth = extractXRCETH(rdmaHeader.headerData); - - let isCNP = isCongestionNotificationPkt(bth); - let isRespPkt = isRdmaRespOpCode(bth.opcode); - let isRespPktOrCNP = isRespPkt || isCNP; - - let dqpn = (bth.trans == TRANS_TYPE_XRC && !isRespPktOrCNP) ? xrceth.srqn : bth.dqpn; - let bthPadCnt = bth.padCnt; - let qpIndex = getIndexQP(dqpn); - - let maybeHandlerPD = qpMetaData.getPD(dqpn); - let qp = qpMetaData.getQueuePairByQPN(dqpn); - let isResp = isRespPkt || isCNP; - let cntrlStatus = isResp ? qp.statusSQ : qp.statusRQ; - let isValidStateQP = cntrlStatus.comm.isNonErr || cntrlStatus.comm.isERR; - // $display( - // "time=%0t: extractHeader", $time, - // ", maybeHandlerPD=", fshow(maybeHandlerPD), - // ", isValidStateQP=", fshow(isValidStateQP), - // " should be valid, when dqpn=%h, bth.psn=%h, bth.opcode=", - // dqpn, bth.psn, fshow(bth.opcode) - // ); - immAssert( - isValidStateQP, - "isValidStateQP assertion @ mkSimExtractNormalReqResp", - $format( - "isValidStateQP=", fshow(isValidStateQP), - " should be valid, when bth.trans=", fshow(bth.trans), - " and dqpn=%h", dqpn - ) - ); - // immAssert( - // isValid(maybeHandlerPD), - // "PD valid assertion @ mkSimExtractNormalReqResp", - // $format( - // "isValid(maybeHandlerPD)=", fshow(isValid(maybeHandlerPD)), - // " should be valid, when bth.trans=", fshow(bth.trans), - // " and dqpn=%h", dqpn - // ) - // ); - - let isFirstOrMidPkt = isFirstOrMiddleRdmaOpCode(bth.opcode); - let isLastPkt = isLastRdmaOpCode(bth.opcode); - - let pktLen = pktLenReg; - let pktFragNum = pktFragNumReg; - let pktValid = False; - - let isByteEnAllOne = isAllOnesR(payloadFrag.byteEn); - let payloadFragLen = calcFragByteNumFromByteEn(payloadFrag.byteEn); - immAssert( - isValid(payloadFragLen), - "isValid(payloadFragLen) assertion @ mkSimExtractNormalReqResp", - $format( - "payloadFragLen=", fshow(payloadFragLen), " should be valid" - ) - ); - - let fragLen = unwrapMaybe(payloadFragLen); - let isByteEnNonZero = !isZero(fragLen); - ByteEnBitNum fragLenWithOutPad = fragLen - zeroExtend(bthPadCnt); - PktLen fragLenExtWithOutPad = zeroExtend(fragLenWithOutPad); - case ({ pack(payloadFrag.isFirst), pack(payloadFrag.isLast) }) - 2'b11: begin // payloadFrag.isFirst && payloadFrag.isLast - pktLen = fragLenExtWithOutPad; - pktFragNum = 1; - pktValid = (isFirstOrMidPkt ? False : (isLastPkt ? isByteEnNonZero : True)); - end - 2'b10: begin // payloadFrag.isFirst && !payloadFrag.isLast - pktLen = fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)); - pktFragNum = 1; - pktValid = isByteEnAllOne; - end - 2'b01: begin // !payloadFrag.isFirst && payloadFrag.islast - pktLen = pktLenAddFragLen(pktLenReg, fragLenWithOutPad); - // pktLen = pktLenReg + fragLenExtWithOutPad; - pktFragNum = pktFragNumReg + 1; - pktValid = pktValidReg; - end - 2'b00: begin // !payloadFrag.isFirst && !payloadFrag.islast - pktLen = pktLenAddBusByteWidth(pktLenReg); - // pktLen = pktLenReg + fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)); - pktFragNum = pktFragNumReg + 1; - pktValid = pktValidReg && isByteEnAllOne; - end - endcase - - pktLenReg <= pktLen; - pktValidReg <= pktValid; - pktFragNumReg <= pktFragNum; - - let pktStatus = PKT_ST_VALID; - if (!pktValid) begin - // Invalid packet length - pktStatus = PKT_ST_LEN_ERR; - end - immAssert( - pktValid, - "pktValid assertion @ mkSimExtractNormalReqResp", - $format( - "pktValid=", fshow(pktValid), - " should be valid, when payloadFrag.isFirst=", fshow(payloadFrag.isFirst), - ", payloadFrag.isLast=", fshow(payloadFrag.isLast), - ", isFirstOrMidPkt=", fshow(isFirstOrMidPkt), - ", isLastPkt=", fshow(isLastPkt), - ", isByteEnNonZero=", fshow(isByteEnNonZero), - ", isByteEnAllOne=", fshow(isByteEnAllOne), - ", pktValidReg=", fshow(pktValidReg), - ", bth.opcode=", fshow(bth.opcode), - ", bth.psn=%h", bth.psn - ) - ); - - let isZeroPayloadLen = isZeroR(pktLen); - let pktMetaData = RdmaPktMetaData { - pktPayloadLen : pktLen, - pktFragNum : (isZeroPayloadLen ? 0 : pktFragNum), - isZeroPayloadLen: isZeroPayloadLen, - pktHeader : rdmaHeader, - pdHandler : unwrapMaybe(maybeHandlerPD), - pktValid : pktValid, - pktStatus : pktStatus - }; - if (isValid(maybeHandlerPD)) begin - let bthCheckResult = checkZeroFields4BTH(bth); - let headerCheckResult = - padCntCheckReqHeader(bth) || padCntCheckRespHeader(bth, aeth); - immAssert( - bthCheckResult && headerCheckResult, - "bth valid assertion @ mkSimExtractNormalReqResp", - $format( - "bth=", fshow(bth), - " should be valid, but bthCheckResult=", fshow(bthCheckResult), - " and headerCheckResult=", fshow(headerCheckResult) - ) - ); - - if (isCNP) begin - cnpOutVec[qpIndex].enq(bth); - end - else if (isRespPkt) begin - // Do not use rdmaHeader.headerMetaData.hasPayload here, - // since it is only depend on RdamOpCode - if (!isZeroPayloadLen) begin - respPayloadOutVec[qpIndex].enq(payloadFrag); - end - if (payloadFrag.isLast) begin - respPktMetaDataOutVec[qpIndex].enq(pktMetaData); - end - end - else begin - // Do not use rdmaHeader.headerMetaData.hasPayload here, - // since it is only depend on RdamOpCode - if (!isZeroPayloadLen) begin - reqPayloadOutVec[qpIndex].enq(payloadFrag); - end - if (payloadFrag.isLast) begin - reqPktMetaDataOutVec[qpIndex].enq(pktMetaData); - end - end - end - - if (isZeroPayloadLen) begin - immAssert( - !rdmaHeader.headerMetaData.hasPayload, - "hasPayload assertion @ mkSimExtractNormalReqResp", - $format( - "hasPayload=", fshow(rdmaHeader.headerMetaData.hasPayload), - " should be false when isZeroPayloadLen=", fshow(isZeroPayloadLen) - ) - ); - end - - if (bth.opcode == ACKNOWLEDGE) begin - immAssert( - isZeroPayloadLen && payloadFrag.isLast && payloadFrag.isFirst, - "isZeroPayloadLen assertion @ mkSimExtractNormalReqResp", - $format( - "isZeroPayloadLen=", fshow(isZeroPayloadLen), - ", payloadFrag.isFirst=", fshow(payloadFrag.isFirst), - ", payloadFrag.isLast=", fshow(payloadFrag.isLast), - " should all be true when bth.opcode=", fshow(bth.opcode) - ) - ); - end - // $display( - // "time=%0t: mkSimExtractNormalReqResp recvPktFrag", $time, - // ", bth.opcode=", fshow(bth.opcode), - // ", bth.psn=%h, dqpn=%h, pktFragNum=%0d, pktLen=%0d", - // bth.psn, dqpn, pktFragNum, pktLen - // // ", bthPadCnt=%0d", bthPadCnt, - // // ", fragLen=%0d", fragLen, - // // ", payloadFrag.isFirst=", fshow(payloadFrag.isFirst), - // // ", payloadFrag.isLast=", fshow(payloadFrag.isLast), - // // ", fragLenWithOutPad=%0d", fragLenWithOutPad, - // // ", rdmaHeader=", fshow(rdmaHeader) - // ); - endrule - - function InputRdmaPktBuf genInputRdmaPktBuf(Integer idx); - return interface InputRdmaPktBuf; - interface reqPktPipeOut = interface RdmaPktMetaDataAndPayloadPipeOut; - interface pktMetaData = toPipeOut(reqPktMetaDataOutVec[idx]); - interface payload = toPipeOut(reqPayloadOutVec[idx]); - endinterface; - - interface respPktPipeOut = interface RdmaPktMetaDataAndPayloadPipeOut; - interface pktMetaData = toPipeOut(respPktMetaDataOutVec[idx]); - interface payload = toPipeOut(respPayloadOutVec[idx]); - endinterface; - - interface cnpPipeOut = toPipeOut(cnpOutVec[idx]); - endinterface; - endfunction - - return map(genInputRdmaPktBuf, genVector); -endmodule -*/ interface TransportLayer; interface Put#(RecvReq) recvReqInput; interface Put#(WorkReq) workReqInput; diff --git a/src/Utils.bsv b/src/Utils.bsv index c6062f7..7794693 100644 --- a/src/Utils.bsv +++ b/src/Utils.bsv @@ -310,7 +310,8 @@ function HeaderMetaData genHeaderMetaData( headerLen: headerLen, headerFragNum: headerFragNum, lastFragValidByteNum: lastFragValidByteNum, - hasPayload: hasPayload + hasPayload: hasPayload, + isEmptyHeader : False }; return headerMetaData; endfunction @@ -323,12 +324,27 @@ function RdmaHeader genRdmaHeader( let headerByteEn = genHeaderByteEn(headerLen); let headerMetaData = genHeaderMetaData(headerLen, hasPayload); return RdmaHeader { - headerData: headerData, - headerByteEn: headerByteEn, + headerData : headerData, + headerByteEn : headerByteEn, headerMetaData: headerMetaData }; endfunction +function RdmaHeader genEmptyRdmaHeader(Bool hasPayload); + let emptyHeaderMetaData = HeaderMetaData { + headerLen : 0, + headerFragNum : 0, + lastFragValidByteNum: 0, + hasPayload : hasPayload, + isEmptyHeader : True + }; + return RdmaHeader { + headerData : dontCareValue, + headerByteEn : 0, + headerMetaData: emptyHeaderMetaData + }; +endfunction + function Tuple2#(HeaderByteNum, HeaderBitNum) calcHeaderInvalidFragByteAndBitNum( HeaderFragNum headerValidFragNum ); @@ -723,22 +739,20 @@ function Maybe#(TransType) qpType2TransType(TypeQP qpt); IBV_QPT_UD : tagged Valid TRANS_TYPE_UD; IBV_QPT_XRC_RECV , IBV_QPT_XRC_SEND : tagged Valid TRANS_TYPE_XRC; - IBV_QPT_RAW_PACKET: tagged Valid TRANS_TYPE_RAW; default : tagged Invalid; endcase; endfunction -function Bool transTypeMatchQpType(TransType tt, TypeQP qpt, Bool isRespPkt); +function Bool transTypeMatchQpType(TransType tt, TypeQP qpt, Bool isRecvSide); return case (tt) TRANS_TYPE_CNP: True; TRANS_TYPE_RC : (qpt == IBV_QPT_RC); TRANS_TYPE_UC : (qpt == IBV_QPT_UC); TRANS_TYPE_UD : (qpt == IBV_QPT_UD); TRANS_TYPE_XRC: ( - (!isRespPkt && qpt == IBV_QPT_XRC_RECV) || - (isRespPkt && qpt == IBV_QPT_XRC_SEND) + (!isRecvSide && qpt == IBV_QPT_XRC_RECV) || + (isRecvSide && qpt == IBV_QPT_XRC_SEND) ); - TRANS_TYPE_RAW: (qpt == IBV_QPT_RAW_PACKET); default: False; endcase; endfunction @@ -754,6 +768,10 @@ function Bool qpNeedGenResp(TransType transType); endcase; endfunction +function Bool isRawPktTypeQP(TypeQP qpType); + return qpType == IBV_QPT_RAW_PACKET; +endfunction + function Bool isSupportedReqOpCodeRQ(TypeQP qpt, RdmaOpCode opcode); case (qpt) IBV_QPT_UC: return case (opcode) @@ -874,6 +892,20 @@ function RETH extractRETH(HeaderData headerData, TransType transType); return reth; endfunction +function LETH extractLETH(HeaderData headerData, TransType transType); + let reth = case (transType) + TRANS_TYPE_XRC: unpack(headerData[ + valueOf(HEADER_MAX_DATA_WIDTH) - valueOf(BTH_WIDTH) - valueOf(XRCETH_WIDTH) - valueOf(RETH_WIDTH) -1 : + valueOf(HEADER_MAX_DATA_WIDTH) - valueOf(BTH_WIDTH) - valueOf(XRCETH_WIDTH) - valueOf(RETH_WIDTH) - valueOf(LETH_WIDTH) + ]); + default: unpack(headerData[ + valueOf(HEADER_MAX_DATA_WIDTH) - valueOf(BTH_WIDTH) - valueOf(RETH_WIDTH) -1 : + valueOf(HEADER_MAX_DATA_WIDTH) - valueOf(BTH_WIDTH) - valueOf(RETH_WIDTH) - valueOf(LETH_WIDTH) + ]); + endcase; + return reth; +endfunction + function AtomicEth extractAtomicEth(HeaderData headerData, TransType transType); let atomicEth = case (transType) TRANS_TYPE_XRC: unpack(headerData[ diff --git a/test/TestExtractAndPrependPipeOut.bsv b/test/TestExtractAndPrependPipeOut.bsv index 68523e6..0a2b769 100644 --- a/test/TestExtractAndPrependPipeOut.bsv +++ b/test/TestExtractAndPrependPipeOut.bsv @@ -223,10 +223,11 @@ module mkTestExtractHeaderLongerThanDataStream(Empty); let { headerFragNum, headerLastFragValidByteNum } = calcHeaderFragNumAndLastFragValidByeNum(headerLen); let headerMetaData = HeaderMetaData { - headerLen: headerLen, - headerFragNum: headerFragNum, + headerLen : headerLen, + headerFragNum : headerFragNum, lastFragValidByteNum: headerLastFragValidByteNum, - hasPayload: True + hasPayload : True, + isEmptyHeader : False }; Vector#(2, DataStreamPipeOut) dataStreamPipeOutVec <- diff --git a/test/TestPayloadGen.bsv b/test/TestPayloadGen.bsv index 3dabe4d..546427d 100644 --- a/test/TestPayloadGen.bsv +++ b/test/TestPayloadGen.bsv @@ -564,9 +564,6 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); isLast : True }; let dummySGE = sge; - // dummySGE.isFirst = False; - // dummySGE.isLast = False; - let sgl = vec(sge, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE); let dmaReadCntrlReq = DmaReadCntrlReq { diff --git a/test/TestReqGenSQ.bsv b/test/TestReqGenSQ.bsv index 6ca5d7b..25c7e9d 100644 --- a/test/TestReqGenSQ.bsv +++ b/test/TestReqGenSQ.bsv @@ -151,6 +151,7 @@ module mkTestReqGenNormalAndZeroLenCase#( if (isOnlyRdmaOpCode(rdmaOpCode)) begin pendingWorkReqPipeOut4Ref.deq; + validPsnReg <= False; let isReadWR = isReadWorkReq(refPendingWR.wr.opcode); if (isReadWR) begin @@ -178,6 +179,7 @@ module mkTestReqGenNormalAndZeroLenCase#( end else if (isLastRdmaOpCode(rdmaOpCode)) begin pendingWorkReqPipeOut4Ref.deq; + validPsnReg <= False; immAssert( bth.psn == wrEndPSN, @@ -186,6 +188,7 @@ module mkTestReqGenNormalAndZeroLenCase#( ); end else if (isFirstRdmaOpCode(rdmaOpCode)) begin + validPsnReg <= True; immAssert( bth.psn == wrStartPSN, "bth.psn first request packet assertion @ mkTestReqGenNormalCase", diff --git a/test/TestSendQ.bsv b/test/TestSendQ.bsv index d450878..918c4aa 100644 --- a/test/TestSendQ.bsv +++ b/test/TestSendQ.bsv @@ -222,32 +222,210 @@ module mkRandomWorkQueueElemWithPayload#( return resultPipeOutVec; endmodule /* -module mkRandomWorkQueueElemRawPkt#( - Length minLength, Length maxLength -)(Vector#(vSz, PipeOut#(WorkQueueElem))); - Vector#(6, WorkReqOpCode) workReqOpCodeVec = vec( +module mkRandomWorkQueueElemRawPkt#(Length fixedLen)(Vector#(vSz, PipeOut#(WorkQueueElem))); + Vector#(9, WorkReqOpCode) workReqOpCodeVec = vec( IBV_WR_SEND, IBV_WR_SEND_WITH_IMM, IBV_WR_SEND_WITH_INV, IBV_WR_RDMA_WRITE, IBV_WR_RDMA_WRITE_WITH_IMM, + IBV_WR_RDMA_READ, + IBV_WR_ATOMIC_CMP_AND_SWP, + IBV_WR_ATOMIC_FETCH_AND_ADD, IBV_WR_RDMA_READ_RESP ); let qpType = IBV_QPT_RAW_PACKET; - let wrFlags = IBV_SEND_SIGNALED; + let wrFlags = IBV_SEND_NO_FLAGS; let workReqOpCodePipeOut <- mkRandomItemFromVec(workReqOpCodeVec); let resultPipeOutVec <- mkSimGenWorkQueueElemByOpCode( - workReqOpCodeVec, minLength, maxLength, qpType, wrFlags + workReqOpCodePipeOut, fixedLen, fixedLen, qpType, wrFlags ); return resultPipeOutVec; endmodule */ +(* doc = "testcase" *) +module mkTestSendQueueRawPktCase(Empty); + let pmtu = IBV_MTU_256; + let pmtuLen = calcPmtuLen(pmtu); + let fixedLength = calcPmtuLen(pmtu); + + Reg#(Bool) clearReg <- mkReg(True); + + // WQE generation + PipeOut#(IP) ipAddrPipeOut <- mkGenericRandomPipeOut; + PipeOut#(MAC) macAddrPipeOut <- mkGenericRandomPipeOut; + FIFOF#(WorkQueueElem) wqeQ <- mkFIFOF; + FIFOF#(WorkQueueElem) wqeRefQ <- mkSizedFIFOF(getMaxFragBufSize); + let wqePipeOut4Ref = toPipeOut(wqeRefQ); + // Vector#(2, PipeOut#(WorkQueueElem)) wqePipeOutVec <- mkRandomWorkQueueElemRawPkt(fixedLength); + // let wqePipeOut4Ref <- mkBufferN(getMaxFragBufSize, wqePipeOutVec[1]); + + // Request payload DataStream generation + let simDmaReadSrv <- mkSimDmaReadSrvAndDataStreamPipeOut; + // let dataStreamWithPaddingPipeOut <- mkDataStreamAddPadding( + // simDmaReadSrv.dataStream + // ); + let dataStreamPipeOut4Ref <- mkBufferN(getMaxFragBufSize, simDmaReadSrv.dataStream); + + let dmaReadCntrl <- mkDmaReadCntrl(clearReg, simDmaReadSrv.dmaReadSrv); + let shouldAddPadding = False; + let payloadGenerator <- mkPayloadGenerator(clearReg, shouldAddPadding, dmaReadCntrl); + + let dut <- mkSendQ(clearReg, payloadGenerator); + mkConnection(dut.srvPort.request, toGet(wqeQ)); + + Reg#(PktLen) payloadLenReg <- mkRegU; + let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); + + // mkSink(dut.rdmaDataStreamPipeOut); + // mkSink(dataStreamPipeOut4Ref); + // mkSink(dut.udpInfoPipeOut); + // mkSink(wqePipeOut4Ref); + + rule clearAll if (clearReg); + clearReg <= False; + // $display("time=%0t: clearAll", $time); + endrule + + rule genWQE if (!clearReg); + let ipAddr = ipAddrPipeOut.first; + ipAddrPipeOut.deq; + let macAddr = macAddrPipeOut.first; + macAddrPipeOut.deq; + + let sge = ScatterGatherElem { + laddr : 0, + len : zeroExtend(fixedLength), + lkey : dontCareValue, + isFirst: True, + isLast : True + }; + let dummySGE = sge; + let sgl = vec(sge, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE); + + let wqe = WorkQueueElem { + id : dontCareValue, + opcode : IBV_WR_RDMA_READ, // dontCareValue + flags : enum2Flag(IBV_SEND_NO_FLAGS), + qpType : IBV_QPT_RAW_PACKET, + psn : 0, + pmtu : pmtu, + dqpIP : ipAddr, + macAddr: macAddr, + sgl : sgl, + raddr : dontCareValue, + rkey : dontCareValue, + // pkey : dontCareValue, + sqpn : getDefaultQPN, + dqpn : getDefaultQPN, + comp : tagged Invalid, + swap : tagged Invalid, + immDtOrInvRKey: tagged Invalid, + srqn : tagged Invalid, + qkey : tagged Invalid, + isFirst: True, + isLast : True + }; + wqeQ.enq(wqe); + wqeRefQ.enq(wqe); + endrule + + rule discardSendResp if (!clearReg); + let sendResp <- dut.srvPort.response.get; + // countDown.decr; + endrule + + rule checkAddr if (!clearReg); + let udpPktInfo = dut.udpInfoPipeOut.first; + dut.udpInfoPipeOut.deq; + + let wqe = wqePipeOut4Ref.first; + wqePipeOut4Ref.deq; + + immAssert( + udpPktInfo.macAddr == wqe.macAddr, + "macAddr assertion @ mkTestSendQueueRawPktCase", + $format( + "udpPktInfo.macAddr=%h shoud == wqe.macAddr=%h", + udpPktInfo.macAddr, wqe.macAddr + ) + ); + immAssert( + udpPktInfo.pktLen == pmtuLen, + "udpPktInfo.pktLen assertion @ mkTestSendQueueRawPktCase", + $format( + "udpPktInfo.pktLen=%0d", udpPktInfo.pktLen, + " should == pmtuLen=%0d", pmtuLen + ) + ); + endrule + + rule checkPktFrag if (!clearReg); + let pktFrag = dut.rdmaDataStreamPipeOut.first; + dut.rdmaDataStreamPipeOut.deq; + + let expectedPktFrag = dataStreamPipeOut4Ref.first; + dataStreamPipeOut4Ref.deq; + + immAssert( + pktFrag.data == expectedPktFrag.data && + pktFrag.byteEn == expectedPktFrag.byteEn, + "payload assertion @ mkTestSendQueueRawPktCase", + $format( + "pktFrag.data=%h", pktFrag.data, + " should == expectedPktFrag.data=%h", expectedPktFrag.data, + " and pktFrag.byteEn=%h", pktFrag.byteEn, + " should == expectedPktFrag.byteEn=%h", expectedPktFrag.byteEn + ) + ); + + let maybePktFragLen = calcFragByteNumFromByteEn(pktFrag.byteEn); + immAssert( + isValid(maybePktFragLen), + "maybePktFragLen assertion @ mkTestSendQueueRawPktCase", + $format( + "isValid(maybePktFragLen)=", fshow(isValid(maybePktFragLen)), + " should be valid" + ) + ); + + let pktFragLen = unwrapMaybe(maybePktFragLen); + let totalPayloadLen = payloadLenReg; + if (pktFrag.isFirst) begin + totalPayloadLen = zeroExtend(pktFragLen); + end + else begin + totalPayloadLen = payloadLenReg + zeroExtend(pktFragLen); + end + payloadLenReg <= totalPayloadLen; + + if (pktFrag.isLast) begin + immAssert( + totalPayloadLen == fixedLength, + "totalPayloadLen assertion @ mkTestSendQueueRawPktCase", + $format( + "totalPayloadLen=%0d", totalPayloadLen, + " should == fixedLength=%0d", fixedLength + ) + ); + // $display( + // "time=%0t: checkOutput", $time, + // ", fixedLength=%0d", fixedLength, + // ", totalPayloadLen=%0d", totalPayloadLen, + // ", pktFrag.isFirst=", fshow(pktFrag.isFirst), + // ", pktFrag.isLast=", fshow(pktFrag.isLast) + // ); + end + countDown.decr; + endrule +endmodule + (* doc = "testcase" *) module mkTestSendQueueNormalCase(Empty); let minDmaLength = 1; let maxDmaLength = 10241; let genPayload = True; - let result <- mkTestSendQueueNormalAndZeroLenCase( + let result <- mkTestSendQueueNormalAndNoPayloadCase( genPayload, minDmaLength, maxDmaLength ); endmodule @@ -257,22 +435,14 @@ module mkTestSendQueueNoPayloadCase(Empty); let minDmaLength = 1; let maxDmaLength = 10241; let genPayload = False; - let result <- mkTestSendQueueNormalAndZeroLenCase( + let result <- mkTestSendQueueNormalAndNoPayloadCase( genPayload, minDmaLength, maxDmaLength ); endmodule -module mkTestSendQueueNormalAndZeroLenCase#( +module mkTestSendQueueNormalAndNoPayloadCase#( Bool genPayload, Length minDmaLength, Length maxDmaLength )(Empty); - let pmtuVec = vec( - IBV_MTU_256, - IBV_MTU_512, - IBV_MTU_1024, - IBV_MTU_2048, - IBV_MTU_4096 - ); - Reg#(Bool) clearReg <- mkReg(True); // WQE generation @@ -280,13 +450,6 @@ module mkTestSendQueueNormalAndZeroLenCase#( mkRandomWorkQueueElemWithPayload(minDmaLength, maxDmaLength) : mkRandomWorkQueueElemWithOutPayload(minDmaLength, maxDmaLength); - // Request payload DataStream generation - // let simDmaReadSrv <- mkSimDmaReadSrvAndDataStreamPipeOut; - // let dataStreamWithPaddingPipeOut <- mkDataStreamAddPadding( - // simDmaReadSrv.dataStream - // ); - // let dataStreamWithPaddingPipeOut4Ref <- mkBufferN(getMaxFragBufSize, dataStreamWithPaddingPipeOut); - let simDmaReadSrv <- mkSimDmaReadSrv; let dmaReadCntrl <- mkDmaReadCntrl(clearReg, simDmaReadSrv); let shouldAddPadding = True; @@ -327,43 +490,26 @@ module mkTestSendQueueNormalAndZeroLenCase#( rule clearAll if (clearReg); clearReg <= False; psnIsValidReg <= False; - $display("time=%0t: clearAll", $time); + // $display("time=%0t: clearAll", $time); endrule -/* - rule compareWorkReq; - let pendingWR = pendingWorkReqPipeOut4Comp.first; - pendingWorkReqPipeOut4Comp.deq; - let refWorkReq = workReqPipeOut4Ref.first; - workReqPipeOut4Ref.deq; - - immAssert( - pendingWR.wr.id == refWorkReq.id && - pendingWR.wr.opcode == refWorkReq.opcode, - "pendingWR.wr assertion @ mkTestSendQueueNormalAndZeroLenCase", - $format( - "pendingWR.wr=", fshow(pendingWR.wr), - " should == refWorkReq=", fshow(refWorkReq) - ) - ); - // $display("time=%0t: WR=", $time, fshow(pendingWR.wr)); - endrule -*/ rule compareRdmaReqHeader if (!clearReg); let rdmaHeader = rdmaHeaderPipeOut.first; rdmaHeaderPipeOut.deq; let { transType, rdmaOpCode } = extractTranTypeAndRdmaOpCode(rdmaHeader.headerData); - let bth = extractBTH(rdmaHeader.headerData); - $display("time=%0t: BTH=", $time, fshow(bth)); + let bth = extractBTH(rdmaHeader.headerData); + let reth = extractRETH(rdmaHeader.headerData, transType); + let leth = extractLETH(rdmaHeader.headerData, transType); + // $display("time=%0t: BTH=", $time, fshow(bth)); if (psnIsValidReg) begin curPsnReg <= curPsnReg + 1; immAssert( bth.psn == curPsnReg, - "bth.psn correctness assertion @ mkTestSendQueueNormalAndZeroLenCase", + "bth.psn correctness assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format("bth.psn=%h shoud == curPsnReg=%h", bth.psn, curPsnReg) ); end @@ -382,83 +528,78 @@ module mkTestSendQueueNormalAndZeroLenCase#( wqePipeOut4Ref.deq; psnIsValidReg <= False; - // let isReadWR = isReadWorkReq(refWQE.opcode); - // if (isReadWR) begin immAssert( bth.psn == wrStartPSN, - "bth.psn read request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", + "bth.psn read request packet assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format( "bth.psn=%h should == wrStartPSN=%h when refWQE.opcode=", bth.psn, wrStartPSN, fshow(refWQE.opcode) ) ); - // end - // else begin - // immAssert( - // bth.psn == wrStartPSN && bth.psn == wrEndPSN, - // "bth.psn only request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", - // $format( - // "bth.psn=%h should == wrStartPSN=%h and bth.psn=%h should == wrEndPSN=%h", - // bth.psn, wrStartPSN, bth.psn, wrEndPSN, - // ", when refWQE.opcode=", - // fshow(refWQE.opcode) - // ) - // ); - // end + let isReadWR = isReadWorkReq(refWQE.opcode); + if (isReadWR) begin + let firstIdxSGE = 0; + immAssert( + // TODO: enable LETH check after update calcHeaderLenByTransTypeAndRdmaOpCode() + reth.rkey == refWQE.rkey && reth.va == refWQE.raddr, + // leth.lkey == refWQE.sgl[firstIdxSGE].lkey && + // leth.va == refWQE.sgl[firstIdxSGE].laddr && + // leth.dlen == refWQE.sgl[firstIdxSGE].len, + "read request assertion @ mkTestSendQueueNormalAndNoPayloadCase", + $format( + "reth.rkey=%h should == refWQE.rkey=%h", reth.rkey, refWQE.rkey, + ", reth.va=%h should == refWQE.raddr=%h", reth.va, refWQE.raddr, + ", leth.lkey=%h should == refWQE.sgl[0].lkey=%h", leth.lkey, refWQE.sgl[firstIdxSGE].lkey, + ", leth.va=%h should == refWQE.sgl[0].laddr=%h", leth.va, refWQE.sgl[firstIdxSGE].laddr, + ", leth.dlen=%0d should == refWQE.sgl[0].len=%0d", leth.dlen, refWQE.sgl[firstIdxSGE].len, + ", when refWQE.opcode=", fshow(refWQE.opcode) + ) + ); + // $display( + // "time=%0t: check read request", $time, + // ", RETH=", fshow(reth), + // ", LETH=", fshow(leth), + // ", refWQE=", fshow(refWQE) + // ); + end end else if (isLastRdmaOpCode(rdmaOpCode)) begin let sendResp <- dut.srvPort.response.get; wqePipeOut4Ref.deq; psnIsValidReg <= False; - - // immAssert( - // bth.psn == wrEndPSN, - // "bth.psn last request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", - // $format("bth.psn=%h shoud == wrEndPSN=%h", bth.psn, wrEndPSN) - // ); end else if (isFirstRdmaOpCode(rdmaOpCode)) begin psnIsValidReg <= True; immAssert( bth.psn == wrStartPSN, - "bth.psn first request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", + "bth.psn first request packet assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format("bth.psn=%h shoud == wrStartPSN=%h", bth.psn, wrStartPSN) ); end else begin immAssert( isMiddleRdmaOpCode(rdmaOpCode), - "rdmaOpCode middle request packet assertion @ mkTestSendQueueNormalAndZeroLenCase", + "rdmaOpCode middle request packet assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format( "rdmaOpCode=", fshow(rdmaOpCode), " should be middle RDMA request opcode" ) ); - // immAssert( - // psnInRangeExclusive(bth.psn, wrStartPSN, wrEndPSN), - // "bth.psn between wrStartPSN and wrEndPSN assertion @ mkTestSendQueueNormalAndZeroLenCase", - // $format( - // "bth.psn=%h should > wrStartPSN=%h and bth.psn=%h should < wrEndPSN=%h", - // bth.psn, wrStartPSN, bth.psn, wrEndPSN, - // ", when refWQE.opcode=", fshow(refWQE.opcode), - // " and rdmaOpCode=", fshow(rdmaOpCode) - // ) - // ); end - let isRespPkt = True; + let isRecvSide = True; immAssert( - transTypeMatchQpType(transType, refWQE.qpType, isRespPkt), - "transTypeMatchQpType assertion @ mkTestSendQueueNormalAndZeroLenCase", + transTypeMatchQpType(transType, refWQE.qpType, isRecvSide), + "transTypeMatchQpType assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format( "transType=", fshow(transType), " should match qpType=", fshow(refWQE.qpType), - " and isRespPkt=", fshow(isRespPkt) + " and isRecvSide=", fshow(isRecvSide) ) ); immAssert( rdmaReqOpCodeMatchWorkReqOpCode(rdmaOpCode, refWQE.opcode), - "rdmaReqOpCodeMatchWorkReqOpCode assertion @ mkTestSendQueueNormalAndZeroLenCase", + "rdmaReqOpCodeMatchWorkReqOpCode assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format( "RDMA request opcode=", fshow(rdmaOpCode), " should match workReqOpCode=", fshow(refWQE.opcode) @@ -477,7 +618,7 @@ module mkTestSendQueueNormalAndZeroLenCase#( let maybePktFragLen = calcFragByteNumFromByteEn(pktFrag.byteEn); immAssert( isValid(maybePktFragLen), - "maybePktFragLen assertion @ mkTestSendQueueNormalAndZeroLenCase", + "maybePktFragLen assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format( "isValid(maybePktFragLen)=", fshow(isValid(maybePktFragLen)), " should be valid" @@ -503,7 +644,7 @@ module mkTestSendQueueNormalAndZeroLenCase#( immAssert( udpPktInfo.macAddr == expectedMacAddr, - "macAddr assertion @ mkTestSendQueueNormalAndZeroLenCase", + "macAddr assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format( "udpPktInfo.macAddr=%h shoud == expectedMacAddr=%h", udpPktInfo.macAddr, expectedMacAddr @@ -512,21 +653,21 @@ module mkTestSendQueueNormalAndZeroLenCase#( immAssert( udpPktInfo.pktLen == totalPayloadLen + zeroExtend(headerLen), - "udpPktInfo.pktLen assertion @ mkTestSendQueueNormalAndZeroLenCase", + "udpPktInfo.pktLen assertion @ mkTestSendQueueNormalAndNoPayloadCase", $format( "udpPktInfo.pktLen=%0d", udpPktInfo.pktLen, " should == totalPayloadLen=%0d", totalPayloadLen, " + headerLen=%0d", headerLen ) ); - $display( - "time=%0t: checkPktLen", $time, - ", udpPktInfo.pktLen=%0d", udpPktInfo.pktLen, - ", totalPayloadLen=%0d", totalPayloadLen, - ", headerLen=%0d", headerLen, - ", udpPktInfo.macAddr=%h", udpPktInfo.macAddr, - ", udpPktInfo.ipAddr=", fshow(udpPktInfo.ipAddr) - ); + // $display( + // "time=%0t: checkPktLen", $time, + // ", udpPktInfo.pktLen=%0d", udpPktInfo.pktLen, + // ", totalPayloadLen=%0d", totalPayloadLen, + // ", headerLen=%0d", headerLen, + // ", udpPktInfo.macAddr=%h", udpPktInfo.macAddr, + // ", udpPktInfo.ipAddr=", fshow(udpPktInfo.ipAddr) + // ); end endrule endmodule diff --git a/test/Utils4Test.bsv b/test/Utils4Test.bsv index 9f47e8b..0a3c495 100644 --- a/test/Utils4Test.bsv +++ b/test/Utils4Test.bsv @@ -412,11 +412,12 @@ module mkFixedLenHeaderMetaPipeOut#( let { headerFragNum, headerLastFragValidByteNum } = calcHeaderFragNumAndLastFragValidByeNum(headerLen); let headerMetaData = HeaderMetaData { - headerLen: headerLen, - headerFragNum: headerFragNum, + headerLen : headerLen, + headerFragNum : headerFragNum, lastFragValidByteNum: headerLastFragValidByteNum, // Use the last bit of HeaderLen to randomize hasPayload - hasPayload: alwaysHasPayload || unpack(headerLen[0]) + hasPayload : alwaysHasPayload || unpack(headerLen[0]), + isEmptyHeader : False }; headerMetaDataQ.enq(headerMetaData); // $display("time=%0t: headerMetaData=", $time, fshow(headerMetaData)); From 2e4b966b20a39f222b941fcc0586bf3e9b166c10 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Fri, 2 Feb 2024 21:27:56 +0800 Subject: [PATCH 14/16] add totalLen to WQE --- Makefile.test | 3 +- run.sh | 1 + src/ExtractAndPrependPipeOut.bsv | 11 +- src/PayloadGen.bsv | 195 ++++++++++--------- src/SendQ.bsv | 110 ++++++----- test/TestPayloadGen.bsv | 312 ++++++++++++++++++------------- test/TestSendQ.bsv | 160 ++++++++-------- 7 files changed, 430 insertions(+), 362 deletions(-) diff --git a/Makefile.test b/Makefile.test index 386b69a..df622b0 100644 --- a/Makefile.test +++ b/Makefile.test @@ -121,7 +121,8 @@ TestPayloadGen.bsv = mkTestAddrChunkSrv \ TestSendQ.bsv = mkTestSendQueueRawPktCase \ mkTestSendQueueNormalCase \ - mkTestSendQueueNoPayloadCase + mkTestSendQueueNoPayloadCase \ + mkTestSendQueueZeroPayloadLenCase all: $(TESTBENCHS) diff --git a/run.sh b/run.sh index 99126ae..8343f52 100755 --- a/run.sh +++ b/run.sh @@ -133,6 +133,7 @@ mkdir -p $LOG_DIR # make -j8 TESTFILE=TestSendQ.bsv TOPMODULE=mkTestSendQueueRawPktCase # make -j8 TESTFILE=TestSendQ.bsv TOPMODULE=mkTestSendQueueNormalCase # make -j8 TESTFILE=TestSendQ.bsv TOPMODULE=mkTestSendQueueNoPayloadCase +# make -j8 TESTFILE=TestSendQ.bsv TOPMODULE=mkTestSendQueueZeroPayloadLenCase make -j8 -f Makefile.test all TESTDIR=$TEST_DIR LOGDIR=$LOG_DIR cat $LOG_DIR/*.log | tee $ALL_LOG diff --git a/src/ExtractAndPrependPipeOut.bsv b/src/ExtractAndPrependPipeOut.bsv index 100a9a6..01983d9 100644 --- a/src/ExtractAndPrependPipeOut.bsv +++ b/src/ExtractAndPrependPipeOut.bsv @@ -91,16 +91,11 @@ module mkHeader2DataStream#( // "time=%0t: dataStream.data=%h, dataStream.byteEn=%h, leftShiftHeaderData=%h, leftShiftHeaderByteEn=%h", // $time, dataStream.data, dataStream.byteEn, leftShiftHeaderData, leftShiftHeaderByteEn // ); - // let bth = extractBTH(curHeader.headerData); - // if (bth.opcode == ACKNOWLEDGE) begin - // $display( - // "time=%0t: mkHeader2DataStream outputHeader", $time, - // ", bth.psn=%h, bth.opcode=", bth.psn, fshow(bth.opcode) - // ); - // end + if (!curHeader.headerMetaData.isEmptyHeader) begin headerDataStreamOutQ.enq(dataStream); end + let bth = extractBTH(curHeader.headerData); // $display( // "time=%0t: mkHeader2DataStream outputHeader", $time, @@ -352,7 +347,7 @@ module mkPrependHeader2PipeOut#( preDataStreamReg <= headerLastFragDataStream; // $display( // "time=%0t: mkPrependHeader2PipeOut outputHeader", $time, - // ", headerHasPayloadReg=%b", headerHasPayloadReg, + // ", headerHasPayloadReg=", fshow(headerHasPayloadReg), // ", headerLastFragValidByteNum=%0d", headerLastFragValidByteNumReg, // ", headerLastFragValidBitNum=%0d", headerLastFragValidBitNumReg, // ", headerLastFragInvalidByteNum=%0d", headerLastFragInvalidByteNumReg, diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index 4377a6b..6a96461 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -57,12 +57,11 @@ typedef struct { IP dqpIP; MAC macAddr; ScatterGatherList sgl; + Length totalLen; ADDR raddr; RKEY rkey; - // PKEY pkey; QPN sqpn; // TODO: remove it QPN dqpn; - // Bool solicited; // Relevant only for the Send and RDMA Write with immediate data Maybe#(Long) comp; Maybe#(Long) swap; Maybe#(ImmOrRKey) immDtOrInvRKey; @@ -83,6 +82,7 @@ instance FShow#(WorkQueueElem); ", dqpIP=", fshow(wqe.dqpIP), ", macAddr=%h", wqe.macAddr, ", sgl=", fshow(wqe.sgl), + ", totalLen=%0d", wqe.totalLen, ", rkey=%h", wqe.rkey, ", raddr=%h", wqe.raddr, ", sqpn=%h", wqe.sqpn, @@ -136,19 +136,14 @@ typedef struct { Bool isLast; } MergedMetaDataSGE deriving(Bits, FShow); -typedef struct { - QPN sqpn; // TODO: remove it - WorkReqID wrID; // TODO: remove it - Length totalLen; - PMTU pmtu; - // NumSGE sgeNum; - // PktFragNum pmtuFragNum; - // PktLen pmtuLen; -} TotalPayloadLenMetaDataSGL deriving(Bits, FShow); +// typedef struct { +// QPN sqpn; // TODO: remove it +// WorkReqID wrID; // TODO: remove it +// Length totalLen; +// PMTU pmtu; +// } TotalPayloadLenMetaDataSGL deriving(Bits, FShow); typedef struct { - // QPN sqpn; // TODO: remove it - // WorkReqID wrID; // TODO: remove it PktLen firstPktLen; PktFragNum firstPktFragNum; ByteEnBitNum firstPktLastFragValidByteNum; @@ -164,15 +159,12 @@ typedef struct { PktNum adjustedPktNum; PktNum origPktNum; PMTU pmtu; - // Length totalLen; } AdjustedTotalPayloadMetaData deriving(Bits, FShow); typedef struct { - // DmaReqSrcType initiator; ScatterGatherList sgl; + Length totalLen; QPN sqpn; // TODO: remove it - // ADDR startAddr; - // Length len; WorkReqID wrID; // TODO: remove it } DmaReadMetaDataSGL deriving(Bits, FShow); @@ -613,15 +605,14 @@ module mkAddrChunkSrv#(Bool clearAll)(AddrChunkSrv); endmodule */ typedef struct { - // DmaReqSrcType initiator; DmaReadMetaDataSGL sglDmaReadMetaData; PMTU pmtu; } DmaReadCntrlReq deriving(Bits, FShow); typedef struct { DmaReadResp dmaReadResp; - Bool isOrigFirst; - Bool isOrigLast; + Bool isFirstFragInSGL; + Bool isLastFragInSGL; } DmaReadCntrlResp deriving(Bits, FShow); typedef Server#(DmaReadCntrlReq, DmaReadCntrlResp) DmaCntrlReadSrv; @@ -636,7 +627,7 @@ interface DmaReadCntrl; interface DmaCntrlReadSrv srvPort; interface DmaCntrl dmaCntrl; interface PipeOut#(PktMetaDataSGE) sgePktMetaDataPipeOut; - interface PipeOut#(TotalPayloadLenMetaDataSGL) sglTotalPayloadLenMetaDataPipeOut; + // interface PipeOut#(TotalPayloadLenMetaDataSGL) sglTotalPayloadLenMetaDataPipeOut; interface PipeOut#(MergedMetaDataSGE) sgeMergedMetaDataPipeOut; endinterface @@ -645,12 +636,13 @@ module mkDmaReadCntrl#( )(DmaReadCntrl); FIFOF#(DmaReadCntrlReq) reqQ <- mkFIFOF; FIFOF#(DmaReadCntrlResp) respQ <- mkFIFOF; - FIFOF#(MergedMetaDataSGE) sgeMergedMetaDataOutQ <- mkSizedFIFOF(valueOf(MAX_SGE)); - FIFOF#(TotalPayloadLenMetaDataSGL) sglTotalPayloadLenMetaDataOutQ <- mkFIFOF; + FIFOF#(MergedMetaDataSGE) sgeMergedMetaDataOutQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + // FIFOF#(TotalPayloadLenMetaDataSGL) sglTotalPayloadLenMetaDataOutQ <- mkFIFOF; + FIFOF#(Tuple2#(ScatterGatherElem, PMTU)) pendingScatterGatherElemQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + FIFOF#(LKEY) pendingLKeyQ <- mkFIFOF; FIFOF#(Tuple2#(QPN, WorkReqID)) pendingDmaCntrlReqQ <- mkFIFOF; // TODO: remove it FIFOF#(Tuple2#(Bool, Bool)) pendingDmaReadReqQ <- mkFIFOF; - FIFOF#(Tuple2#(ScatterGatherElem, PMTU)) pendingScatterGatherElemQ <- mkSizedFIFOF(valueOf(MAX_SGE)); let addrChunkSrv <- mkAddrChunkSrv(clearAll); @@ -664,9 +656,12 @@ module mkDmaReadCntrl#( rule resetAndClear if (clearAll); reqQ.clear; respQ.clear; + sgeMergedMetaDataOutQ.clear; + // sglTotalPayloadLenMetaDataOutQ.clear; - pendingDmaCntrlReqQ.clear; pendingScatterGatherElemQ.clear; + pendingLKeyQ.clear; + pendingDmaCntrlReqQ.clear; pendingDmaReadReqQ.clear; cancelReg[1] <= False; @@ -689,8 +684,8 @@ module mkDmaReadCntrl#( !isZeroR(sge.len), "zero SGE assertion @ mkDmaReadCntrl", $format( - "sge.len=%d", sge.len, - " should not be zero when sglIdxReg=%d", sglIdxReg + "sge.len=%0d", sge.len, + " should not be zero when sglIdxReg=%0d", sglIdxReg ) ); @@ -703,14 +698,6 @@ module mkDmaReadCntrl#( }; sgeMergedMetaDataOutQ.enq(sgeMergedMetaData); pendingScatterGatherElemQ.enq(tuple2(sge, dmaReadCntrlReq.pmtu)); - // let addrChunkReq = AddrChunkReq { - // startAddr: sge.laddr, - // len : sge.len, - // pmtu : dmaReadCntrlReq.pmtu, - // isFirst : sge.isFirst, - // isLast : sge.isLast - // }; - // addrChunkSrv.srvPort.request.put(addrChunkReq); let curSQPN = dmaReadCntrlReq.sglDmaReadMetaData.sqpn; let curWorkReqID = dmaReadCntrlReq.sglDmaReadMetaData.wrID; @@ -735,18 +722,17 @@ module mkDmaReadCntrl#( "first SGE assertion @ mkDmaReadCntrl", $format( "sge.isFirst=", fshow(sge.isFirst), - " should be true when sglIdxReg=%d", sglIdxReg + " should be true when sglIdxReg=%0d", sglIdxReg ) ); end - // if (sglIdxReg == valueOf(TSub#(MAX_SGE, 1))) begin if (isAllOnesR(sglIdxReg)) begin immAssert( sge.isLast, "last SGE assertion @ mkDmaReadCntrl", $format( "sge.isLast=", fshow(sge.isLast), - " should be true when sglIdxReg=%d", sglIdxReg + " should be true when sglIdxReg=%0d", sglIdxReg ) ); end @@ -755,14 +741,22 @@ module mkDmaReadCntrl#( reqQ.deq; sglIdxReg <= 0; - let sglTotalPayloadLenMetaData = TotalPayloadLenMetaDataSGL { - sqpn : curSQPN, - wrID : curWorkReqID, - totalLen: totalLen, - // sgeNum : sgeNum, - pmtu : dmaReadCntrlReq.pmtu - }; - sglTotalPayloadLenMetaDataOutQ.enq(sglTotalPayloadLenMetaData); + // let sglTotalPayloadLenMetaData = TotalPayloadLenMetaDataSGL { + // sqpn : curSQPN, + // wrID : curWorkReqID, + // totalLen: totalLen, + // pmtu : dmaReadCntrlReq.pmtu + // }; + // sglTotalPayloadLenMetaDataOutQ.enq(sglTotalPayloadLenMetaData); + immAssert( + totalLen == dmaReadCntrlReq.sglDmaReadMetaData.totalLen, + "totalLen assertion @ mkDmaReadCntrl", + $format( + "dmaReadCntrlReq.sglDmaReadMetaData.totalLen=%0d", + dmaReadCntrlReq.sglDmaReadMetaData.totalLen, + " should == totalLen=%0d", totalLen + ) + ); // $display( // "time=%0t: mkDmaReadCntrl recvReq", $time, // ", sqpn=%h", curSQPN, @@ -802,13 +796,19 @@ module mkDmaReadCntrl#( isLast : sge.isLast }; addrChunkSrv.srvPort.request.put(addrChunkReq); + + pendingLKeyQ.enq(sge.lkey); + // $display( + // "time=%0t: mkDmaReadCntrl issueChunkReq", $time, + // ", addrChunkReq=", fshow(addrChunkReq) + // ); endrule rule issueDmaReq if (!clearAll && !cancelReg[1]); let addrChunkResp <- addrChunkSrv.srvPort.response.get; + let lkey = pendingLKeyQ.first; let { curSQPN, curWorkReqID } = pendingDmaCntrlReqQ.first; - // let pendingSGE = pendingScatterGatherElemQ.first; let dmaReadReq = DmaReadReq { initiator: DMA_SRC_SQ_RD, @@ -817,19 +817,16 @@ module mkDmaReadCntrl#( len : addrChunkResp.chunkLen, wrID : curWorkReqID }; - dmaReadSrv.request.put(dmaReadReq); - // let curSGE = pendingScatterGatherElemQ.first; - // let isFirstDmaReqChunk = addrChunkResp.isFirst && curSGE.isFirst; - // let isLastDmaReqChunk = addrChunkResp.isLast && curSGE.isLast; + let isLastChunkInSGE = addrChunkResp.isLast; let isFirstDmaReqChunk = addrChunkResp.isFirst && addrChunkResp.isOrigFirst; - let isLastDmaReqChunk = addrChunkResp.isLast && addrChunkResp.isOrigLast; + let isLastDmaReqChunk = addrChunkResp.isLast && addrChunkResp.isOrigLast; pendingDmaReadReqQ.enq(tuple2(isFirstDmaReqChunk, isLastDmaReqChunk)); - // if (addrChunkResp.isLast) begin - // pendingScatterGatherElemQ.deq; - // end + if (isLastChunkInSGE) begin + pendingLKeyQ.deq; + end if (isLastDmaReqChunk) begin pendingDmaCntrlReqQ.deq; end @@ -847,13 +844,13 @@ module mkDmaReadCntrl#( let { isFirstDmaReqChunk, isLastDmaReqChunk } = pendingDmaReadReqQ.first; - let isOrigFirst = dmaResp.dataStream.isFirst && isFirstDmaReqChunk; - let isOrigLast = dmaResp.dataStream.isLast && isLastDmaReqChunk; + let isFirstFragInSGL = dmaResp.dataStream.isFirst && isFirstDmaReqChunk; + let isLastFragInSGL = dmaResp.dataStream.isLast && isLastDmaReqChunk; let dmaReadCntrlResp = DmaReadCntrlResp { - dmaReadResp: dmaResp, - isOrigFirst: isOrigFirst, - isOrigLast : isOrigLast + dmaReadResp : dmaResp, + isFirstFragInSGL: isFirstFragInSGL, + isLastFragInSGL : isLastFragInSGL }; respQ.enq(dmaReadCntrlResp); @@ -893,7 +890,7 @@ module mkDmaReadCntrl#( method Bool isIdle() = gracefulStopReg[0]; endinterface; - interface sglTotalPayloadLenMetaDataPipeOut = toPipeOut(sglTotalPayloadLenMetaDataOutQ); + // interface sglTotalPayloadLenMetaDataPipeOut = toPipeOut(sglTotalPayloadLenMetaDataOutQ); interface sgeMergedMetaDataPipeOut = toPipeOut(sgeMergedMetaDataOutQ); interface sgePktMetaDataPipeOut = addrChunkSrv.sgePktMetaDataPipeOut; endmodule @@ -1782,6 +1779,7 @@ typedef struct { WorkReqID wrID; // TODO: remote it QPN sqpn; // TODO: remote it ScatterGatherList sgl; + Length totalLen; ADDR raddr; PMTU pmtu; } PayloadGenReqSG deriving(Bits, FShow); @@ -1795,9 +1793,8 @@ typedef struct { } PayloadGenRespSG deriving(Bits, FShow); typedef struct { - Length totalLen; + // Length totalLen; PktNum totalPktNum; - // PMTU pmtu; Bool isOnlyPkt; Bool isZeroPayloadLen; } PayloadGenTotalMetaData deriving(Bits, FShow); @@ -1895,24 +1892,27 @@ module mkPayloadGenerator#( let sglIdx = 0; let sge = payloadGenReq.sgl[sglIdx]; - // If first SGE has zero length, then whole SGL has no payload - let isZeroPayloadLen = isZeroR(sge.len); - // if (isZeroPayloadLen) begin - // immAssert( - // sge.isLast, - // "last SGE assertion @ mkDmaReadCntrl", - // $format( - // "sge.isLast=", fshow(sge.isLast), - // " should be true when sglIdx=%d", sglIdx, - // ", and sge.len=%d", sge.len - // ) - // ); - // end + // // If first SGE has zero length, then whole SGL has no payload + // let isZeroPayloadLen = isZeroR(sge.len); + let isZeroPayloadLen = isZeroR(payloadGenReq.totalLen); + if (isZeroPayloadLen) begin + immAssert( + sge.isLast, + "last SGE assertion @ mkDmaReadCntrl", + $format( + "sge.isLast=", fshow(sge.isLast), + " should be true when sglIdx=%0d", sglIdx, + ", sge.len=%0d", sge.len, + ", and totalLen=%0d", payloadGenReq.totalLen + ) + ); + end if (!isZeroPayloadLen) begin let dmaReadCntrlReq = DmaReadCntrlReq { sglDmaReadMetaData: DmaReadMetaDataSGL { sgl : payloadGenReq.sgl, + totalLen : payloadGenReq.totalLen, sqpn : payloadGenReq.sqpn, wrID : payloadGenReq.wrID }, @@ -1936,13 +1936,12 @@ module mkPayloadGenerator#( let { payloadGenReq, isZeroPayloadLen } = adjustReqPktLenQ.first; adjustReqPktLenQ.deq; - let totalLen = 0; - if (!isZeroPayloadLen) begin - let sglTotalPayloadLenMetaData = dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.first; - dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.deq; - totalLen = sglTotalPayloadLenMetaData.totalLen; - end - + let totalLen = payloadGenReq.totalLen; + // if (!isZeroPayloadLen) begin + // let sglTotalPayloadLenMetaData = dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.first; + // dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.deq; + // totalLen = sglTotalPayloadLenMetaData.totalLen; + // end let { truncatedPktNum, residue } = truncateLenByPMTU(totalLen, payloadGenReq.pmtu); let origPktNum = truncatedPktNum + (isZeroR(residue) ? 0 : 1); @@ -2008,7 +2007,6 @@ module mkPayloadGenerator#( adjustedPktNum : totalPktNum, origPktNum : origPktNum, pmtu : pmtu - // totalLen : totalLen }; if (!isZeroPayloadLen) begin adjustedTotalPayloadMetaDataQ.enq(adjustedTotalPayloadMetaData); @@ -2032,9 +2030,8 @@ module mkPayloadGenerator#( addPadCntQ.enq(paddingMetaData); let totalMetaData = PayloadGenTotalMetaData { - totalLen : totalLen, + // totalLen : totalLen, totalPktNum : totalPktNum, - // pmtu : pmtu, isOnlyPkt : isOnlyPkt, isZeroPayloadLen: isZeroPayloadLen }; @@ -2071,12 +2068,12 @@ module mkPayloadGenerator#( (fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)) - zeroExtend(lastPktPadCnt) >= lastPktLastFragValidByteNum), "zero SGE assertion @ mkPayloadGenerator", $format( - "firstPktLastFragValidByteNum=%d", firstPktLastFragValidByteNum, - " + firstPktPadCnt=%d", firstPktPadCnt, - " should not > DATA_BUS_BYTE_WIDTH=%d", valueOf(DATA_BUS_BYTE_WIDTH), - ", and lastPktLastFragValidByteNum=%d", lastPktLastFragValidByteNum, - " + lastPktPadCnt=%d", lastPktPadCnt, - " should not > DATA_BUS_BYTE_WIDTH=%d", valueOf(DATA_BUS_BYTE_WIDTH) + "firstPktLastFragValidByteNum=%0d", firstPktLastFragValidByteNum, + " + firstPktPadCnt=%0d", firstPktPadCnt, + " should not > DATA_BUS_BYTE_WIDTH=%0d", valueOf(DATA_BUS_BYTE_WIDTH), + ", and lastPktLastFragValidByteNum=%0d", lastPktLastFragValidByteNum, + " + lastPktPadCnt=%0d", lastPktPadCnt, + " should not > DATA_BUS_BYTE_WIDTH=%0d", valueOf(DATA_BUS_BYTE_WIDTH) ) ); let oneAsPSN = 1; @@ -2123,24 +2120,24 @@ module mkPayloadGenerator#( let curPayloadFrag = adjustedPayloadPipeOut.first; adjustedPayloadPipeOut.deq; - let isFirstPktLastFrag = curPayloadFrag.isLast && isFirstPkt; - let isLastPktLastFrag = curPayloadFrag.isLast && isLastPkt; + let isLastFragInFirstPkt = curPayloadFrag.isLast && isFirstPkt; + let isLastFragInLastPkt = curPayloadFrag.isLast && isLastPkt; - if (isLastPktLastFrag) begin + if (isLastFragInLastPkt) begin addPadCntQ.deq; if (shouldAddPadding) begin curPayloadFrag.byteEn = lastPktLastFragByteEnWithPadding; end end - if (isFirstPktLastFrag) begin + if (isLastFragInFirstPkt) begin if (shouldAddPadding) begin curPayloadFrag.byteEn = firstPktLastFragByteEnWithPadding; end end if (curPayloadFrag.isLast) begin - isFirstPktReg <= isLastPktLastFrag; + isFirstPktReg <= isLastFragInLastPkt; pktRemoteAddrReg <= nextRemoteAddr; remainingPktNumReg <= remainingPktNum; end @@ -2167,8 +2164,8 @@ module mkPayloadGenerator#( // ", remainingPktNumReg=%0d", remainingPktNumReg, // ", isFirstPktReg=", fshow(isFirstPktReg), // ", isOnlyPkt=", fshow(isOnlyPkt), - // ", isFirstPktLastFrag=", fshow(isFirstPktLastFrag), - // ", isLastPktLastFrag=", fshow(isLastPktLastFrag), + // ", isLastFragInFirstPkt=", fshow(isLastFragInFirstPkt), + // ", isLastFragInLastPkt=", fshow(isLastFragInLastPkt), // ", isFirstPkt=", fshow(isFirstPkt), // ", isLastPkt=", fshow(isLastPkt), // ", curPayloadFrag.isFirst=", fshow(curPayloadFrag.isFirst), diff --git a/src/SendQ.bsv b/src/SendQ.bsv index fb432c7..eeb0a8a 100644 --- a/src/SendQ.bsv +++ b/src/SendQ.bsv @@ -584,7 +584,7 @@ module mkSendQ#( // Pipeline FIFOF FIFOF#(Tuple3#(WorkQueueElem, Bool, Bool)) totalMetaDataQ <- mkFIFOF; - FIFOF#(Tuple6#(WorkQueueElem, Length, PktNum, Bool, Bool, Bool)) psnUpdateQ <- mkFIFOF; + FIFOF#(Tuple7#(WorkQueueElem, Length, PktNum, Bool, Bool, Bool, Bool)) psnUpdateQ <- mkFIFOF; FIFOF#(Tuple2#(WorkQueueElem, HeaderGenInfo)) headerPrepareQ <- mkFIFOF; FIFOF#(Tuple6#(MAC, IP, Maybe#(PktHeaderInfo), PktLen, Bool, Bool)) pendingHeaderQ <- mkFIFOF; FIFOF#(RdmaHeader) pktHeaderQ <- mkFIFOF; @@ -719,23 +719,24 @@ module mkSendQ#( let qpRawPkt = isRawPktTypeQP(wqe.qpType); let isSendWR = isSendWorkReq(wqe.opcode); - let needPayloadGen = qpRawPkt || workReqNeedPayloadGen(wqe.opcode); + let shouldGenPayload = qpRawPkt || workReqNeedPayloadGen(wqe.opcode); let remoteAddr = wqe.raddr; if (qpRawPkt || isSendWR) begin remoteAddr = 0; end - if (needPayloadGen) begin + if (shouldGenPayload) begin let payloadGenReq = PayloadGenReqSG { - wrID : wqe.id, - sqpn : wqe.sqpn, - sgl : wqe.sgl, - raddr: remoteAddr, - pmtu : wqe.pmtu + wrID : wqe.id, + sqpn : wqe.sqpn, + sgl : wqe.sgl, + totalLen: wqe.totalLen, + raddr : remoteAddr, + pmtu : wqe.pmtu }; payloadGenerator.srvPort.request.put(payloadGenReq); end - totalMetaDataQ.enq(tuple3(wqe, qpRawPkt, needPayloadGen)); + totalMetaDataQ.enq(tuple3(wqe, qpRawPkt, shouldGenPayload)); // TODO: handle pending read/atomic request number limit // $display( @@ -744,57 +745,58 @@ module mkSendQ#( // ", id=%h", wqe.id, // ", macAddr=%h", wqe.macAddr, // ", pmtu=", fshow(wqe.pmtu), - // ", needPayloadGen=", fshow(needPayloadGen) + // ", shouldGenPayload=", fshow(shouldGenPayload) // ); endrule rule recvTotalMetaData if (!clearAll); - let { wqe, qpRawPkt, needPayloadGen } = totalMetaDataQ.first; + let { wqe, qpRawPkt, shouldGenPayload } = totalMetaDataQ.first; totalMetaDataQ.deq; let sglZeroIdx = 0; - let hasPayload = needPayloadGen; - let isOnlyPkt = !needPayloadGen; - let totalLen = wqe.sgl[sglZeroIdx].len; + let hasPayload = shouldGenPayload; + let isOnlyPkt = !shouldGenPayload; + let totalLen = wqe.totalLen; // wqe.sgl[sglZeroIdx].len; let totalPktNum = 1; - if (needPayloadGen) begin + + if (!wqe.isFirst && !wqe.isLast) begin + immAssert( + !isOnlyPkt && + totalLen == fromInteger(valueOf(WQE_SLICE_MAX_SIZE)), + "wqe slice length assertion @ mkSendQ", + $format( + "totalLen=%0d", totalLen, + " should == WQE_SLICE_MAX_SIZE=%0d", valueOf(WQE_SLICE_MAX_SIZE), + ", and isOnlyPkt=", fshow(isOnlyPkt), + " should be false when wqe.isFirst=", fshow(wqe.isFirst), + " and wqe.isLast=", fshow(wqe.isLast) + ) + ); + end + else if (!wqe.isFirst && wqe.isLast) begin + immAssert( + fromInteger(valueOf(WQE_SLICE_MAX_SIZE)) >= totalLen, + "wqe slice length assertion @ mkSendQ", + $format( + "totalLen=%0d", totalLen, + " should be no more than WQE_SLICE_MAX_SIZE=%0d", valueOf(WQE_SLICE_MAX_SIZE), + " when wqe.isFirst=", fshow(wqe.isFirst), + " and wqe.isLast=", fshow(wqe.isLast) + ) + ); + end + + if (shouldGenPayload) begin let payloadTotalMetaData = payloadGenerator.totalMetaDataPipeOut.first; payloadGenerator.totalMetaDataPipeOut.deq; hasPayload = !payloadTotalMetaData.isZeroPayloadLen; isOnlyPkt = payloadTotalMetaData.isOnlyPkt; - totalLen = payloadTotalMetaData.totalLen; + // totalLen = payloadTotalMetaData.totalLen; totalPktNum = payloadTotalMetaData.totalPktNum; - - if (!wqe.isFirst && !wqe.isLast) begin - immAssert( - !isOnlyPkt && - payloadTotalMetaData.totalLen == fromInteger(valueOf(WQE_SLICE_MAX_SIZE)), - "wqe slice length assertion @ mkSendQ", - $format( - "payloadTotalMetaData.totalLen=%0d", payloadTotalMetaData.totalLen, - " should == WQE_SLICE_MAX_SIZE=%0d", valueOf(WQE_SLICE_MAX_SIZE), - ", and isOnlyPkt=", fshow(isOnlyPkt), - " should be false when wqe.isFirst=", fshow(wqe.isFirst), - " and wqe.isLast=", fshow(wqe.isLast) - ) - ); - end - else if (!wqe.isFirst && wqe.isLast) begin - immAssert( - fromInteger(valueOf(WQE_SLICE_MAX_SIZE)) >= payloadTotalMetaData.totalLen, - "wqe slice length assertion @ mkSendQ", - $format( - "payloadTotalMetaData.totalLen=%0d", payloadTotalMetaData.totalLen, - " should be no more than WQE_SLICE_MAX_SIZE=%0d", valueOf(WQE_SLICE_MAX_SIZE), - " when wqe.isFirst=", fshow(wqe.isFirst), - " and wqe.isLast=", fshow(wqe.isLast) - ) - ); - end end - psnUpdateQ.enq(tuple6( - wqe, totalLen, totalPktNum, hasPayload, isOnlyPkt, qpRawPkt + psnUpdateQ.enq(tuple7( + wqe, totalLen, totalPktNum, shouldGenPayload, hasPayload, isOnlyPkt, qpRawPkt )); // $display( // "time=%0t: mkSendQ 2nd stage recvTotalMetaData", $time, @@ -806,13 +808,13 @@ module mkSendQ#( // ", qpRawPkt=", fshow(qpRawPkt), // ", hasPayload=", fshow(hasPayload), // ", isOnlyPkt=", fshow(isOnlyPkt), - // ", needPayloadGen=", fshow(needPayloadGen) + // ", shouldGenPayload=", fshow(shouldGenPayload) // ); endrule rule updatePSN if (!clearAll); let { - wqe, totalLen, totalPktNum, hasPayload, isOnlyPkt, qpRawPkt + wqe, totalLen, totalPktNum, shouldGenPayload, hasPayload, isOnlyPkt, qpRawPkt } = psnUpdateQ.first; let curPSN = curPsnReg; @@ -825,13 +827,18 @@ module mkSendQ#( let pktPayloadLen = 0; let padCnt = 0; let wqeLastPkt = isOnlyPkt; - if (hasPayload) begin + if (shouldGenPayload) begin let payloadGenResp <- payloadGenerator.srvPort.response.get; remoteAddr = payloadGenResp.raddr; pktPayloadLen = payloadGenResp.pktLen; padCnt = payloadGenResp.padCnt; wqeLastPkt = payloadGenResp.isLast; end + // let payloadGenResp <- payloadGenerator.srvPort.response.get; + // let remoteAddr = payloadGenResp.raddr; + // let pktPayloadLen = payloadGenResp.pktLen; + // let padCnt = payloadGenResp.padCnt; + // let wqeLastPkt = payloadGenResp.isLast; wqeFirstPktReg <= wqeLastPkt; if (wqeLastPkt) begin @@ -900,6 +907,7 @@ module mkSendQ#( // ", curPSN=%h", curPSN, // ", pktPayloadLen=%0d", pktPayloadLen, // ", padCnt=%0d", padCnt, + // ", hasPayload=", fshow(hasPayload), // ", isFirstPkt=", fshow(isFirstPkt), // ", isLastPkt=", fshow(isLastPkt), // ", isOnlyPkt=", fshow(isOnlyPkt) @@ -985,7 +993,8 @@ module mkSendQ#( // ", isFirstPkt=", fshow(isFirstPkt), // ", isLastPkt=", fshow(isLastPkt), // ", isOnlyPkt=", fshow(isOnlyPkt), - // ", isValid(maybePktHeaderInfo)=", fshow(isValid(maybePktHeaderInfo)) + // ", isValid(maybePktHeaderInfo)=", fshow(isValid(maybePktHeaderInfo)), + // ", hasPayload=", fshow(hasPayload) // ); endrule @@ -1025,7 +1034,8 @@ module mkSendQ#( // ", headerLen=%0d", headerLen, // ", udpPktInfo.macAddr=%h", udpPktInfo.macAddr, // ", udpPktInfo.ipAddr=", fshow(udpPktInfo.ipAddr), - // ", udpPktInfo.pktLen=%0d", udpPktInfo.pktLen + // ", udpPktInfo.pktLen=%0d", udpPktInfo.pktLen, + // ", hasPayload=", fshow(hasPayload) // ); end endrule diff --git a/test/TestPayloadGen.bsv b/test/TestPayloadGen.bsv index 546427d..7958065 100644 --- a/test/TestPayloadGen.bsv +++ b/test/TestPayloadGen.bsv @@ -259,11 +259,12 @@ module mkTestDmaReadCntrlScatterGatherListCase(Empty); Reg#(Length) totalLenReg <- mkRegU; Reg#(ScatterGatherList) sglReg <- mkRegU; - FIFOF#(Tuple2#(ScatterGatherList, PMTU)) sglQ <- mkFIFOF; + FIFOF#(Tuple3#(ScatterGatherList, Length, PMTU)) sglQ <- mkFIFOF; FIFOF#(TotalFragNum) sglTotalFragNumQ <- mkFIFOF; + let sgeMinNum = 1; Vector#(1, PipeOut#(NumSGE)) sgeNumPipeOutVec <- - mkRandomValueInRangePipeOut(fromInteger(1), fromInteger(valueOf(MAX_SGE))); + mkRandomValueInRangePipeOut(fromInteger(sgeMinNum), fromInteger(valueOf(MAX_SGE))); let sgeNumPipeOut = sgeNumPipeOutVec[0]; PipeOut#(Bool) randomCancelPipeOut <- mkGenericRandomPipeOut; @@ -277,7 +278,7 @@ module mkTestDmaReadCntrlScatterGatherListCase(Empty); let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); mkSink(dut.sgePktMetaDataPipeOut); - mkSink(dut.sglTotalPayloadLenMetaDataPipeOut); + // mkSink(dut.sglTotalPayloadLenMetaDataPipeOut); mkSink(dut.sgeMergedMetaDataPipeOut); rule clearAll if (clearReg); @@ -301,14 +302,14 @@ module mkTestDmaReadCntrlScatterGatherListCase(Empty); payloadLenPipeOut.deq; let sgl = sglReg; - let isFirst = isZero(sglIdxReg); - let isLast = isAllOnesR(sglIdxReg) || (sgeNum - 1 == zeroExtend(sglIdxReg)); + let isFirstSGE = isZero(sglIdxReg); + let isLastSGE = isAllOnesR(sglIdxReg) || (sgeNum - 1 == zeroExtend(sglIdxReg)); let sge = ScatterGatherElem { laddr : startAddr, len : payloadLen, lkey : dontCareValue, - isFirst: isFirst, - isLast : isLast + isFirst: isFirstSGE, + isLast : isLastSGE }; sgl[sglIdxReg] = sge; sglReg <= sgl; @@ -336,7 +337,7 @@ module mkTestDmaReadCntrlScatterGatherListCase(Empty); let totalLen = totalLenReg; let sglTotalfragNum = sglTotalFragNumReg; - if (isFirst) begin + if (isFirstSGE) begin totalLen = payloadLen; sglTotalfragNum = sgeTotalFragNum; end @@ -347,10 +348,10 @@ module mkTestDmaReadCntrlScatterGatherListCase(Empty); totalLenReg <= totalLen; sglTotalFragNumReg <= sglTotalfragNum; - if (isLast) begin + if (isLastSGE) begin sgeNumPipeOut.deq; pmtuPipeOut.deq; - sglQ.enq(tuple2(sgl, pmtu)); + sglQ.enq(tuple3(sgl, totalLen, pmtu)); sglTotalFragNumQ.enq(sglTotalfragNum); sglIdxReg <= 0; @@ -372,13 +373,14 @@ module mkTestDmaReadCntrlScatterGatherListCase(Empty); endrule rule issueReq if (!clearReg); - let { sgl, pmtu } = sglQ.first; + let { sgl, totalLen, pmtu } = sglQ.first; sglQ.deq; let dmaReadCntrlReq = DmaReadCntrlReq { pmtu : pmtu, sglDmaReadMetaData: DmaReadMetaDataSGL { sgl : sgl, + totalLen : totalLen, sqpn : getDefaultQPN, wrID : dontCareValue // initiator: DMA_SRC_RQ_RD, @@ -416,8 +418,8 @@ module mkTestDmaReadCntrlScatterGatherListCase(Empty); rule checkResp if (!clearReg); let dmaReadCntrlResp <- dut.srvPort.response.get; - let sglFirstFrag = dmaReadCntrlResp.isOrigFirst; - let sglLastFrag = dmaReadCntrlResp.isOrigLast; + let sglFirstFrag = dmaReadCntrlResp.isFirstFragInSGL; + let sglLastFrag = dmaReadCntrlResp.isLastFragInSGL; let isFirstFrag = dmaReadCntrlResp.dmaReadResp.dataStream.isFirst; let isLastFrag = dmaReadCntrlResp.dmaReadResp.dataStream.isLast; @@ -510,7 +512,7 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); mkSink(dut.sgePktMetaDataPipeOut); - mkSink(dut.sglTotalPayloadLenMetaDataPipeOut); + // mkSink(dut.sglTotalPayloadLenMetaDataPipeOut); mkSink(dut.sgeMergedMetaDataPipeOut); rule clearAll if (stateReg == TEST_DMA_CNTRL_INIT); @@ -570,6 +572,7 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); pmtu : pmtu, sglDmaReadMetaData: DmaReadMetaDataSGL { sgl : sgl, + totalLen : sge.len, sqpn : getDefaultQPN, wrID : dontCareValue // initiator: DMA_SRC_RQ_RD, @@ -607,17 +610,15 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); rule checkResp if (stateReg == TEST_DMA_CNTRL_RUN && !dut.dmaCntrl.isIdle); let dmaReadCntrlResp <- dut.srvPort.response.get; - // let isOrigFirst = dmaReadCntrlResp.isOrigFirst; - // let isOrigLast = dmaReadCntrlResp.isOrigLast; isFinalRespLastFragReg <= dmaReadCntrlResp.dmaReadResp.dataStream.isLast; - let isFirstFrag = dmaReadCntrlResp.isOrigFirst && dmaReadCntrlResp.dmaReadResp.dataStream.isFirst; - let isLastFrag = dmaReadCntrlResp.isOrigLast && dmaReadCntrlResp.dmaReadResp.dataStream.isLast; + let isFirstFragInSGL = dmaReadCntrlResp.isFirstFragInSGL; + let isLastFragInSGL = dmaReadCntrlResp.isLastFragInSGL; let totalFragNum = totalFragNumQ.first; let remainingFragNum = remainingFragNumReg; if (!isLessOrEqOne(totalFragNum)) begin - if (isFirstFrag) begin + if (isFirstFragInSGL) begin remainingFragNum = totalFragNum - 2; end else begin @@ -626,7 +627,7 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); end remainingFragNumReg <= remainingFragNum; - if (isLastFrag) begin + if (isLastFragInSGL) begin totalFragNumQ.deq; immAssert( @@ -634,8 +635,8 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); "remaining fragNum assertion @ mkTestDmaReadCntrlNormalOrCancelCase", $format( "remainingFragNumReg=%0d should be zero when", remainingFragNumReg, - " isOrigFirst=", fshow(isFirstFrag), - " and isLastFrag=", fshow(isLastFrag) + " isFirstFragInSGL=", fshow(isFirstFragInSGL), + " and isLastFragInSGL=", fshow(isLastFragInSGL) ) ); end @@ -655,8 +656,8 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); // ", remainingFragNumReg=%0d", remainingFragNumReg, // ", isFirst=", fshow(dmaReadCntrlResp.dmaReadResp.dataStream.isFirst), // ", isLast=", fshow(dmaReadCntrlResp.dmaReadResp.dataStream.isLast), - // ", isOrigFirst=", fshow(dmaReadCntrlResp.isOrigFirst), - // ", isOrigLast=", fshow(dmaReadCntrlResp.isOrigLast) + // ", isFirstFragInSGL=", fshow(dmaReadCntrlResp.isFirstFragInSGL), + // ", isLastFragInSGL=", fshow(dmaReadCntrlResp.isLastFragInSGL) // ); endrule @@ -717,12 +718,18 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( Reg#(Bool) clearReg <- mkReg(True); Reg#(TotalFragNum) remainingFragNumReg <- mkReg(0); Reg#(IdxSGL) sglIdxReg <- mkReg(0); + Reg#(NumSGE) sgeNumReg <- mkRegU; Reg#(Length) totalLenReg <- mkRegU; Vector#(MAX_SGE, Reg#(ScatterGatherElem)) sglRegVec <- replicateM(mkRegU); FIFOF#(DataStream) sgePayloadOutQ <- mkFIFOF; FIFOF#(Tuple2#(TotalFragNum, ByteEn)) sgeRefQ <- mkSizedFIFOF(valueOf(MAX_SGE)); + let sgeMinNum = 1; + Vector#(1, PipeOut#(NumSGE)) sgeNumPipeOutVec <- + mkRandomValueInRangePipeOut(fromInteger(sgeMinNum), fromInteger(valueOf(MAX_SGE))); + let sgeNumPipeOut = sgeNumPipeOutVec[0]; + PipeOut#(ADDR) startAddrPipeOut <- mkGenericRandomPipeOut; let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); @@ -737,8 +744,10 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( Reg#(TestMergePayloadStateSGE) stateReg <- mkReg(TEST_MERGE_EACH_SGE_INIT); let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); - // mkSink(dut.sgePktMetaDataPipeOut); - mkSink(dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut); + // mkSink(toPipeOut(sgeRefQ)); + // mkSink(toPipeOut(sgePayloadOutQ)); + // mkSink(dmaReadCntrl.sgePktMetaDataPipeOut); + // mkSink(dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut); mkSink(dmaReadCntrl.sgeMergedMetaDataPipeOut); rule clearAll if (stateReg == TEST_MERGE_EACH_SGE_INIT); @@ -751,25 +760,28 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( endrule rule genSGE if (stateReg == TEST_MERGE_EACH_SGE_PREPARE); + let sgeNum = sgeNumPipeOut.first; + sgeNumReg <= sgeNum; + let startAddr = startAddrPipeOut.first; startAddrPipeOut.deq; let payloadLen = payloadLenPipeOut.first; payloadLenPipeOut.deq; - let isFirst = isZero(sglIdxReg); - let isLast = isAllOnesR(sglIdxReg); + let isFirstSGE = isZero(sglIdxReg); + let isLastSGE = isAllOnesR(sglIdxReg) || (sgeNum - 1 == zeroExtend(sglIdxReg)); let sge = ScatterGatherElem { laddr : startAddr, len : payloadLen, lkey : dontCareValue, - isFirst: isFirst, - isLast : isLast + isFirst: isFirstSGE, + isLast : isLastSGE }; sglRegVec[sglIdxReg] <= sge; let totalLen = totalLenReg; - if (isFirst) begin + if (isFirstSGE) begin totalLen = payloadLen; end else begin @@ -777,7 +789,8 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( end totalLenReg <= totalLen; - if (isLast) begin + if (isLastSGE) begin + sgeNumPipeOut.deq; sglIdxReg <= 0; stateReg <= TEST_MERGE_EACH_SGE_ISSUE; end @@ -793,6 +806,7 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( countDown.decr; // $display( // "time=%0t: genSGE", $time, + // ", sgeNum=%0d", sgeNum, // ", sglIdxReg=%0d", sglIdxReg, // ", startAddr=%h", startAddr, // ", payloadLen=%0d", payloadLen, @@ -811,6 +825,7 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( pmtu : pmtu, sglDmaReadMetaData: DmaReadMetaDataSGL { sgl : sgl, + totalLen : totalLenReg, sqpn : getDefaultQPN, wrID : dontCareValue // initiator: DMA_SRC_RQ_RD, @@ -825,38 +840,41 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( // "time=%0t: issueDmaReadCntrlReq", $time, // ", pmtu=", fshow(pmtu) // ); - // for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin - // let sge = sglRegVec[idx]; - // let { - // pmtuLen, firstPktLen, lastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt - // } = calcPktNumAndPktLenByAddrAndPMTU(sge.laddr, sge.len, pmtu); - // let firstPktFragNum = calcFragNumByPktLen(firstPktLen); - // let lastPktFragNum = calcFragNumByPktLen(lastPktLen); + for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin + let sge = sglRegVec[idx]; + let { + pmtuLen, firstPktLen, lastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU(sge.laddr, sge.len, pmtu); + let firstPktFragNum = calcFragNumByPktLen(firstPktLen); + let lastPktFragNum = calcFragNumByPktLen(lastPktLen); - // $display( - // "time=%0t: issueDmaReadCntrlReq", $time, - // ", SGE idx=%0d", idx, - // ", sge.laddr=%h", sge.laddr, - // ", sgePktNum=%0d", sgePktNum, - // ", firstPktLen=%0d", firstPktLen, - // ", lastPktLen=%0d", lastPktLen, - // ", pmtuLen=%0d", pmtuLen, - // ", firstPktFragNum=%0d", firstPktFragNum, - // ", lastPktFragNum=%0d", lastPktFragNum - // ); - // end + // $display( + // "time=%0t: issueDmaReadCntrlReq", $time, + // ", SGE idx=%0d", idx, + // ", sge.laddr=%h", sge.laddr, + // ", sgePktNum=%0d", sgePktNum, + // ", firstPktLen=%0d", firstPktLen, + // ", lastPktLen=%0d", lastPktLen, + // ", pmtuLen=%0d", pmtuLen, + // ", firstPktFragNum=%0d", firstPktFragNum, + // ", lastPktFragNum=%0d", lastPktFragNum, + // ", isFirst=", fshow(sge.isFirst), + // ", isLast=", fshow(sge.isLast) + // ); + end endrule rule recvDmaReadCntrlResp; let dmaReadCntrlResp <- dmaReadCntrl.srvPort.response.get; sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); + // $display( + // "time=%0t: recvDmaReadCntrlResp", $time, + // ", dmaReadCntrlResp=", fshow(dmaReadCntrlResp) + // ); endrule rule checkMergedPayloadSGE if (stateReg == TEST_MERGE_EACH_SGE_RUN); let { sgeFragNum, lastFragByteEn } = sgeRefQ.first; - // let { - // sgeFragNum, lastFragByteEn, lastFragValidByteNum - // } = calcTotalFragNumByLength(refSGE.len); let payloadFrag = dut.first; dut.deq; @@ -894,10 +912,11 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( ); end + let isLastSGE = isAllOnesR(sglIdxReg) || (sgeNumReg - 1 == zeroExtend(sglIdxReg)); if (payloadFrag.isLast) begin sgeRefQ.deq; - if (isAllOnesR(sglIdxReg)) begin + if (isLastSGE) begin sglIdxReg <= 0; stateReg <= TEST_MERGE_EACH_SGE_INIT; end @@ -937,9 +956,9 @@ module mkTestMergeNormalOrSmallPayloadEachSGE#( end // $display( // "time=%0t: checkMergedPayloadSGE", $time, - // ", refSGE.len=%0d", refSGE.len, + // // ", refSGE.len=%0d", refSGE.len, + // // ", lastFragValidByteNum=%0d", lastFragValidByteNum, // ", sgeFragNum=%0d", sgeFragNum, - // ", lastFragValidByteNum=%0d", lastFragValidByteNum, // ", lastFragByteEn=%h", lastFragByteEn, // ", remainingFragNumReg=%0d", remainingFragNumReg, // ", payloadFrag.isFirst=", fshow(payloadFrag.isFirst), @@ -992,6 +1011,11 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( FIFOF#(DataStream) sgePayloadOutQ <- mkFIFOF; FIFOF#(Tuple2#(TotalFragNum, ByteEn)) sglRefQ <- mkFIFOF; + let sgeMinNum = 1; + Vector#(1, PipeOut#(NumSGE)) sgeNumPipeOutVec <- + mkRandomValueInRangePipeOut(fromInteger(sgeMinNum), fromInteger(valueOf(MAX_SGE))); + let sgeNumPipeOut = sgeNumPipeOutVec[0]; + PipeOut#(ADDR) startAddrPipeOut <- mkGenericRandomPipeOut; let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); let pmtuPipeOut <- mkRandomItemFromVec(pmtuVec); @@ -1012,7 +1036,7 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( // mkSink(dmaReadCntrl.sgePktMetaDataPipeOut); // mkSink(dmaReadCntrl.sgeMergedMetaDataPipeOut); - mkSink(dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut); + // mkSink(dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut); rule clearAll if (stateReg == TEST_MERGE_ALL_SGE_INIT); clearReg <= False; @@ -1024,25 +1048,27 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( endrule rule genSGE if (stateReg == TEST_MERGE_ALL_SGE_PREPARE); + let sgeNum = sgeNumPipeOut.first; + let startAddr = startAddrPipeOut.first; startAddrPipeOut.deq; let payloadLen = payloadLenPipeOut.first; payloadLenPipeOut.deq; - let isFirst = isZero(sglIdxReg); - let isLast = isAllOnesR(sglIdxReg); + let isFirstSGE = isZero(sglIdxReg); + let isLastSGE = isAllOnesR(sglIdxReg) || (sgeNum - 1 == zeroExtend(sglIdxReg)); let sge = ScatterGatherElem { laddr : startAddr, len : payloadLen, lkey : dontCareValue, - isFirst: isFirst, - isLast : isLast + isFirst: isFirstSGE, + isLast : isLastSGE }; sglRegVec[sglIdxReg] <= sge; let totalLen = totalLenReg; - if (isFirst) begin + if (isFirstSGE) begin totalLen = payloadLen; end else begin @@ -1050,7 +1076,8 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( end totalLenReg <= totalLen; - if (isLast) begin + if (isLastSGE) begin + sgeNumPipeOut.deq; sglIdxReg <= 0; stateReg <= TEST_MERGE_ALL_SGE_ISSUE; end @@ -1082,6 +1109,7 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( pmtu : pmtu, sglDmaReadMetaData: DmaReadMetaDataSGL { sgl : sgl, + totalLen : totalLenReg, sqpn : getDefaultQPN, wrID : dontCareValue // initiator: DMA_SRC_RQ_RD, @@ -1105,13 +1133,13 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( // ", sglLastFragByteEn=%h", sglLastFragByteEn, // ", sglLastFragValidByteNum=%0d", sglLastFragValidByteNum // ); - // for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin - // let sge = sglRegVec[idx]; - // let { - // pmtuLen, sgeFirstPktLen, sgeLastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt - // } = calcPktNumAndPktLenByAddrAndPMTU(sge.laddr, sge.len, pmtu); - // let sgeFirstPktFragNum = calcFragNumByPktLen(sgeFirstPktLen); - // let sgeLastPktFragNum = calcFragNumByPktLen(sgeLastPktLen); + for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin + let sge = sglRegVec[idx]; + let { + pmtuLen, sgeFirstPktLen, sgeLastPktLen, sgePktNum, secondChunkStartAddr //, isSinglePkt + } = calcPktNumAndPktLenByAddrAndPMTU(sge.laddr, sge.len, pmtu); + let sgeFirstPktFragNum = calcFragNumByPktLen(sgeFirstPktLen); + let sgeLastPktFragNum = calcFragNumByPktLen(sgeLastPktLen); // $display( // "time=%0t: issueDmaReadCntrlReq", $time, @@ -1124,7 +1152,7 @@ module mkTestMergeNormalOrSmallPayloadAllSGE#( // ", sgeFirstPktFragNum=%0d", sgeFirstPktFragNum, // ", sgeLastPktFragNum=%0d", sgeLastPktFragNum // ); - // end + end endrule rule recvDmaReadCntrlResp; @@ -1240,7 +1268,7 @@ typedef enum { TEST_ADJUST_PAYLOAD_INIT, TEST_ADJUST_PAYLOAD_PREPARE, TEST_ADJUST_PAYLOAD_ISSUE, - TEST_ADJUST_PAYLOAD_RECV, + TEST_ADJUST_PAYLOAD_CALC, TEST_ADJUST_PAYLOAD_RUN } TestAdjustPayloadState deriving(Bits, Eq, FShow); @@ -1269,6 +1297,11 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( FIFOF#(AdjustedTotalPayloadMetaData) adjustedTotalPayloadMetaDataQ <- mkFIFOF; FIFOF#(Tuple3#(AdjustedTotalPayloadMetaData, PktFragNum, ByteEnBitNum)) adjustedTotalPayloadMetaDataRefQ <- mkFIFOF; + let sgeMinNum = 1; + Vector#(1, PipeOut#(NumSGE)) sgeNumPipeOutVec <- + mkRandomValueInRangePipeOut(fromInteger(sgeMinNum), fromInteger(valueOf(MAX_SGE))); + let sgeNumPipeOut = sgeNumPipeOutVec[0]; + PipeOut#(ADDR) remoteAddrPipeOut <- mkGenericRandomPipeOut; PipeOut#(ADDR) localAddrPipeOut <- mkGenericRandomPipeOut; let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); @@ -1287,6 +1320,7 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( clearReg, toPipeOut(adjustedTotalPayloadMetaDataQ), sglMergedPayloadPipeOut ); + Reg#(Length) totalLenReg <- mkRegU; Reg#(TestAdjustPayloadState) stateReg <- mkReg(TEST_ADJUST_PAYLOAD_INIT); let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); @@ -1304,24 +1338,36 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( endrule rule genSGE if (stateReg == TEST_ADJUST_PAYLOAD_PREPARE); + let sgeNum = sgeNumPipeOut.first; + let localAddr = localAddrPipeOut.first; localAddrPipeOut.deq; let payloadLen = payloadLenPipeOut.first; payloadLenPipeOut.deq; - let isFirst = isZero(sglIdxReg); - let isLast = isAllOnesR(sglIdxReg); + let isFirstSGE = isZero(sglIdxReg); + let isLastSGE = isAllOnesR(sglIdxReg) || (sgeNum - 1 == zeroExtend(sglIdxReg)); let sge = ScatterGatherElem { laddr : localAddr, len : payloadLen, lkey : dontCareValue, - isFirst: isFirst, - isLast : isLast + isFirst: isFirstSGE, + isLast : isLastSGE }; sglRegVec[sglIdxReg] <= sge; - if (isLast) begin + let totalLen = totalLenReg; + if (isFirstSGE) begin + totalLen = payloadLen; + end + else begin + totalLen = totalLenReg + payloadLen; + end + totalLenReg <= totalLen; + + if (isLastSGE) begin + sgeNumPipeOut.deq; sglIdxReg <= 0; stateReg <= TEST_ADJUST_PAYLOAD_ISSUE; end @@ -1354,16 +1400,14 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( pmtu : pmtu, sglDmaReadMetaData: DmaReadMetaDataSGL { sgl : sgl, + totalLen : totalLenReg, sqpn : getDefaultQPN, wrID : dontCareValue - // initiator: DMA_SRC_RQ_RD, - // startAddr: startAddr, - // len : payloadLen, } }; dmaReadCntrl.srvPort.request.put(dmaReadCntrlReq); - stateReg <= TEST_ADJUST_PAYLOAD_RECV; + stateReg <= TEST_ADJUST_PAYLOAD_CALC; for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin let sge = sglRegVec[idx]; let { @@ -1391,27 +1435,27 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( sgePayloadOutQ.enq(dmaReadCntrlResp.dmaReadResp.dataStream); endrule - rule recvTotalPayloadLenMetaData if (stateReg == TEST_ADJUST_PAYLOAD_RECV); + rule calcAdjustedTotalPayloadMetaData if (stateReg == TEST_ADJUST_PAYLOAD_CALC); let remoteAddr = remoteAddrPipeOut.first; remoteAddrPipeOut.deq; let pmtu = pmtuQ.first; pmtuQ.deq; - let totalPayloadMetaData = dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.first; - dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.deq; - let totalLen = totalPayloadMetaData.totalLen; + // let totalPayloadMetaData = dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.first; + // dmaReadCntrl.sglTotalPayloadLenMetaDataPipeOut.deq; + // let totalLen = totalPayloadMetaData.totalLen; let { pmtuLen, firstPktLen, lastPktLen, totalPktNum, secondChunkStartAddr //, isSinglePkt } = calcPktNumAndPktLenByAddrAndPMTU( - remoteAddr, totalLen, pmtu + remoteAddr, totalLenReg, pmtu ); remainingPktNumReg <= totalPktNum - 1; isFirstPktReg <= True; - let origLastFragValidByteNum = calcLastFragValidByteNum(totalLen); - let origPktNum = calcPktNumByLenOnly(totalLen, pmtu); + let origLastFragValidByteNum = calcLastFragValidByteNum(totalLenReg); + let origPktNum = calcPktNumByLenOnly(totalLenReg, pmtu); let firstPktLastFragValidByteNum = calcLastFragValidByteNum(firstPktLen); let lastPktLastFragValidByteNum = calcLastFragValidByteNum(lastPktLen); @@ -1431,7 +1475,6 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( adjustedPktNum : totalPktNum, origPktNum : origPktNum, pmtu : pmtu - // totalLen : totalLen }; adjustedTotalPayloadMetaDataQ.enq(adjustedTotalPayloadMetaData); adjustedTotalPayloadMetaDataRefQ.enq(tuple3( @@ -1440,9 +1483,9 @@ module mkTestAdjustNormalOrSmallPayloadSegment#( stateReg <= TEST_ADJUST_PAYLOAD_RUN; // $display( - // "time=%0t: recvTotalPayloadLenMetaData", $time, + // "time=%0t: calcAdjustedTotalPayloadMetaData", $time, // ", pmtu=", fshow(pmtu), - // ", totalLen=%0d", totalLen, + // ", totalLenReg=%0d", totalLenReg, // ", firstPktLen=%0d", firstPktLen, // ", lastPktLen=%0d", lastPktLen, // ", pmtuLen=%0d", pmtuLen, @@ -1619,6 +1662,14 @@ module mkTestPayloadGenZeroCase(Empty); let result <- mkTestNormalOrSmallPayloadGen(minPayloadLen, maxPayloadLen); endmodule +typedef enum { + TEST_PAYLOAD_GEN_INIT, + TEST_PAYLOAD_GEN_PREPARE, + TEST_PAYLOAD_GEN_ISSUE, + TEST_PAYLOAD_GEN_RECV, + TEST_PAYLOAD_GEN_RUN +} TestPayloadGenState deriving(Bits, Eq, FShow); + module mkTestNormalOrSmallPayloadGen#( // Both min and max are inclusive Length minPayloadLen, Length maxPayloadLen @@ -1637,6 +1688,11 @@ module mkTestNormalOrSmallPayloadGen#( Reg#(Length) totalLenReg <- mkRegU; Vector#(MAX_SGE, Reg#(ScatterGatherElem)) sglRegVec <- replicateM(mkRegU); + let sgeMinNum = 1; + Vector#(1, PipeOut#(NumSGE)) sgeNumPipeOutVec <- + mkRandomValueInRangePipeOut(fromInteger(sgeMinNum), fromInteger(valueOf(MAX_SGE))); + let sgeNumPipeOut = sgeNumPipeOutVec[0]; + PipeOut#(ADDR) remoteAddrPipeOut <- mkGenericRandomPipeOut; PipeOut#(ADDR) localAddrPipeOut <- mkGenericRandomPipeOut; let payloadLenPipeOut <- mkRandomLenPipeOut(minPayloadLen, maxPayloadLen); @@ -1657,47 +1713,52 @@ module mkTestNormalOrSmallPayloadGen#( Reg#(PktFragNum) lastPktFragNumReg <- mkRegU; Reg#(PktFragNum) pmtuFragNumReg <- mkRegU; Reg#(ADDR) expectedRemoteAddrReg <- mkRegU; - Reg#(TestAdjustPayloadState) stateReg <- mkReg(TEST_ADJUST_PAYLOAD_INIT); + Reg#(TestPayloadGenState) stateReg <- mkReg(TEST_PAYLOAD_GEN_INIT); let countDown <- mkCountDown(valueOf(MAX_CMP_CNT)); - rule clearAll if (stateReg == TEST_ADJUST_PAYLOAD_INIT); + rule clearAll if (stateReg == TEST_PAYLOAD_GEN_INIT); clearReg <= False; sglIdxReg <= 0; - stateReg <= TEST_ADJUST_PAYLOAD_PREPARE; + stateReg <= TEST_PAYLOAD_GEN_PREPARE; // $display("time=%0t: clearAll", $time); endrule - rule genSGE if (stateReg == TEST_ADJUST_PAYLOAD_PREPARE); + rule genSGE if (stateReg == TEST_PAYLOAD_GEN_PREPARE); + let sgeNum = sgeNumPipeOut.first; + let localAddr = localAddrPipeOut.first; localAddrPipeOut.deq; let payloadLen = payloadLenPipeOut.first; payloadLenPipeOut.deq; - let isFirst = isZero(sglIdxReg); - let isLast = isAllOnesR(sglIdxReg); + let isZeroPayloadLen = isZero(payloadLen); + let isFirstSGE = isZeroPayloadLen || isZero(sglIdxReg); + let isLastSGE = isZeroPayloadLen || isAllOnesR(sglIdxReg) || + (sgeNum - 1 == zeroExtend(sglIdxReg)); let sge = ScatterGatherElem { laddr : localAddr, len : payloadLen, lkey : dontCareValue, - isFirst: isFirst, - isLast : isLast + isFirst: isFirstSGE, + isLast : isLastSGE }; sglRegVec[sglIdxReg] <= sge; let totalLen = totalLenReg; - if (isFirst) begin + if (isFirstSGE) begin totalLen = payloadLen; end else begin totalLen = totalLenReg + payloadLen; end - totalLenReg <= totalLen; + totalLenReg <= totalLen; - if (isLast) begin + if (isLastSGE) begin + sgeNumPipeOut.deq; sglIdxReg <= 0; - stateReg <= TEST_ADJUST_PAYLOAD_ISSUE; + stateReg <= TEST_PAYLOAD_GEN_ISSUE; end else begin sglIdxReg <= sglIdxReg + 1; @@ -1718,7 +1779,7 @@ module mkTestNormalOrSmallPayloadGen#( // ); endrule - rule issuePayloadGenReq if (stateReg == TEST_ADJUST_PAYLOAD_ISSUE); + rule issuePayloadGenReq if (stateReg == TEST_PAYLOAD_GEN_ISSUE); let sgl = readVReg(sglRegVec); let pmtu = pmtuPipeOut.first; @@ -1727,11 +1788,12 @@ module mkTestNormalOrSmallPayloadGen#( remoteAddrPipeOut.deq; let payloadGenReq = PayloadGenReqSG { - wrID : dontCareValue, - sqpn : getDefaultQPN, - sgl : sgl, - raddr: remoteAddr, - pmtu : pmtu + wrID : dontCareValue, + sqpn : getDefaultQPN, + sgl : sgl, + totalLen: totalLenReg, + raddr : remoteAddr, + pmtu : pmtu }; dut.srvPort.request.put(payloadGenReq); @@ -1762,7 +1824,7 @@ module mkTestNormalOrSmallPayloadGen#( // ", remoteAddr=%h", remoteAddr // ); - stateReg <= TEST_ADJUST_PAYLOAD_RECV; + stateReg <= TEST_PAYLOAD_GEN_RECV; for (Integer idx = 0; idx < valueOf(MAX_SGE); idx = idx + 1) begin let sge = sglRegVec[idx]; let { @@ -1785,19 +1847,19 @@ module mkTestNormalOrSmallPayloadGen#( end endrule - rule recvTotalMetaData if (stateReg == TEST_ADJUST_PAYLOAD_RECV); + rule recvTotalMetaData if (stateReg == TEST_PAYLOAD_GEN_RECV); let totalMetaData = dut.totalMetaDataPipeOut.first; dut.totalMetaDataPipeOut.deq; - immAssert( - totalLenReg == totalMetaData.totalLen, - "totalLen assertion @ mkTestNormalOrSmallPayloadGen", - $format( - "totalLenReg=%0d", totalLenReg, - " should == totalMetaData.totalLen", - totalMetaData.totalLen - ) - ); + // immAssert( + // totalLenReg == totalMetaData.totalLen, + // "totalLen assertion @ mkTestNormalOrSmallPayloadGen", + // $format( + // "totalLenReg=%0d", totalLenReg, + // " should == totalMetaData.totalLen", + // totalMetaData.totalLen + // ) + // ); immAssert( remainingPktNumReg == totalMetaData.totalPktNum, "totalPktNum assertion @ mkTestNormalOrSmallPayloadGen", @@ -1809,11 +1871,11 @@ module mkTestNormalOrSmallPayloadGen#( ); isZeroPayloadLenReg <= totalMetaData.isZeroPayloadLen; remainingPktNumReg <= remainingPktNumReg - 1; - stateReg <= TEST_ADJUST_PAYLOAD_RUN; + stateReg <= TEST_PAYLOAD_GEN_RUN; // $display("time=%0t: recvTotalMetaData", $time); endrule - rule checkResp if (stateReg == TEST_ADJUST_PAYLOAD_RUN); + rule checkResp if (stateReg == TEST_PAYLOAD_GEN_RUN); let payloadGenResp = payloadGenRespReg; let isFirstFrag = isFirstFragReg; @@ -1837,7 +1899,7 @@ module mkTestNormalOrSmallPayloadGen#( ); if (isZeroPayloadLenReg) begin - stateReg <= TEST_ADJUST_PAYLOAD_INIT; + stateReg <= TEST_PAYLOAD_GEN_INIT; isFirstFrag = True; end else begin @@ -1856,7 +1918,7 @@ module mkTestNormalOrSmallPayloadGen#( expectedRemoteAddrReg <= nextRemoteAddr; if (isLastPkt) begin - stateReg <= TEST_ADJUST_PAYLOAD_INIT; + stateReg <= TEST_PAYLOAD_GEN_INIT; end else begin remainingPktNum = remainingPktNumReg - 1; diff --git a/test/TestSendQ.bsv b/test/TestSendQ.bsv index 918c4aa..0182de5 100644 --- a/test/TestSendQ.bsv +++ b/test/TestSendQ.bsv @@ -34,8 +34,10 @@ module mkSimGenWorkQueueElemByOpCode#( IBV_MTU_4096 ); + let sgeMinNum = 1; Vector#(1, PipeOut#(NumSGE)) sgeNumPipeOutVec <- - mkRandomValueInRangePipeOut(fromInteger(1), fromInteger(valueOf(MAX_SGE))); + mkRandomValueInRangePipeOut(fromInteger(sgeMinNum), fromInteger(valueOf(MAX_SGE))); + let sgeNumPipeOut = sgeNumPipeOutVec[0]; PipeOut#(WorkReqID) workReqIdPipeOut <- mkGenericRandomPipeOut; PipeOut#(PSN) psnPipeOut <- mkGenericRandomPipeOut; @@ -55,8 +57,9 @@ module mkSimGenWorkQueueElemByOpCode#( Vector#(vSz, PipeOut#(WorkQueueElem)) resultPipeOutVec <- mkForkVector(toPipeOut(wqeOutQ)); - Reg#(IdxSGL) sglIdxReg <- mkReg(0); Vector#(MAX_SGE, Reg#(ScatterGatherElem)) sglRegVec <- replicateM(mkRegU); + Reg#(IdxSGL) sglIdxReg <- mkReg(0); + Reg#(Length) totalLenReg <- mkRegU; Reg#(Bool) busyReg <- mkReg(True); rule genSGE if (busyReg); @@ -68,18 +71,27 @@ module mkSimGenWorkQueueElemByOpCode#( let payloadLen = payloadLenPipeOut.first; payloadLenPipeOut.deq; - let isFirst = isZero(sglIdxReg); - let isLast = isAllOnesR(sglIdxReg); // || (sgeNum - 1 == zeroExtend(sglIdxReg)); + let isZeroPayloadLen = isZero(payloadLen); + let isFirstSGE = isZeroPayloadLen || isZero(sglIdxReg); + let isLastSGE = isZeroPayloadLen || isAllOnesR(sglIdxReg) || + (sgeNum - 1 == zeroExtend(sglIdxReg)); let sge = ScatterGatherElem { laddr : localAddr, len : payloadLen, lkey : dontCareValue, - isFirst: isFirst, - isLast : isLast + isFirst: isFirstSGE, + isLast : isLastSGE }; sglRegVec[sglIdxReg] <= sge; - if (isLast) begin + if (isFirstSGE) begin + totalLenReg <= payloadLen; + end + else begin + totalLenReg <= totalLenReg + payloadLen; + end + + if (isLastSGE) begin sgeNumPipeOutVec[0].deq; sglIdxReg <= 0; busyReg <= False; @@ -94,8 +106,8 @@ module mkSimGenWorkQueueElemByOpCode#( // ", sglIdxReg=%0d", sglIdxReg, // ", localAddr=%h", localAddr, // ", payloadLen=%0d", payloadLen, - // ", isFirst=", fshow(isFirst), - // ", isLast=", fshow(isLast) + // ", isFirstSGE=", fshow(isFirstSGE), + // ", isLastSGE=", fshow(isLastSGE) // ); endrule @@ -155,27 +167,27 @@ module mkSimGenWorkQueueElemByOpCode#( end let wqe = WorkQueueElem { - id : wrID, - opcode : wrOpCode, - flags : enum2Flag(flags), - qpType : qpType, - psn : psn, - pmtu : pmtu, - dqpIP : ipAddr, - macAddr: macAddr, - sgl : sgl, - raddr : remoteAddr, - rkey : dontCareValue, - // pkey : dontCareValue, - sqpn : sqpn, - dqpn : dqpn, - comp : hasComp ? (tagged Valid comp) : (tagged Invalid), - swap : hasSwap ? (tagged Valid swap) : (tagged Invalid), + id : wrID, + opcode : wrOpCode, + flags : enum2Flag(flags), + qpType : qpType, + psn : psn, + pmtu : pmtu, + dqpIP : ipAddr, + macAddr : macAddr, + sgl : sgl, + totalLen : totalLenReg, + raddr : remoteAddr, + rkey : dontCareValue, + sqpn : sqpn, + dqpn : dqpn, + comp : hasComp ? (tagged Valid comp) : (tagged Invalid), + swap : hasSwap ? (tagged Valid swap) : (tagged Invalid), immDtOrInvRKey: hasImmDt ? tagged Valid tagged Imm immDt : (hasInv ? tagged Valid tagged RKey rkey2Inv : tagged Invalid), - srqn : tagged Valid srqn, // for XRC - qkey : tagged Valid qkey, // for UD - isFirst: True, - isLast : True + srqn : tagged Valid srqn, // for XRC + qkey : tagged Valid qkey, // for UD + isFirst : True, + isLast : True }; wqeOutQ.enq(wqe); busyReg <= True; @@ -221,28 +233,7 @@ module mkRandomWorkQueueElemWithPayload#( ); return resultPipeOutVec; endmodule -/* -module mkRandomWorkQueueElemRawPkt#(Length fixedLen)(Vector#(vSz, PipeOut#(WorkQueueElem))); - Vector#(9, WorkReqOpCode) workReqOpCodeVec = vec( - IBV_WR_SEND, - IBV_WR_SEND_WITH_IMM, - IBV_WR_SEND_WITH_INV, - IBV_WR_RDMA_WRITE, - IBV_WR_RDMA_WRITE_WITH_IMM, - IBV_WR_RDMA_READ, - IBV_WR_ATOMIC_CMP_AND_SWP, - IBV_WR_ATOMIC_FETCH_AND_ADD, - IBV_WR_RDMA_READ_RESP - ); - let qpType = IBV_QPT_RAW_PACKET; - let wrFlags = IBV_SEND_NO_FLAGS; - let workReqOpCodePipeOut <- mkRandomItemFromVec(workReqOpCodeVec); - let resultPipeOutVec <- mkSimGenWorkQueueElemByOpCode( - workReqOpCodePipeOut, fixedLen, fixedLen, qpType, wrFlags - ); - return resultPipeOutVec; -endmodule -*/ + (* doc = "testcase" *) module mkTestSendQueueRawPktCase(Empty); let pmtu = IBV_MTU_256; @@ -257,8 +248,6 @@ module mkTestSendQueueRawPktCase(Empty); FIFOF#(WorkQueueElem) wqeQ <- mkFIFOF; FIFOF#(WorkQueueElem) wqeRefQ <- mkSizedFIFOF(getMaxFragBufSize); let wqePipeOut4Ref = toPipeOut(wqeRefQ); - // Vector#(2, PipeOut#(WorkQueueElem)) wqePipeOutVec <- mkRandomWorkQueueElemRawPkt(fixedLength); - // let wqePipeOut4Ref <- mkBufferN(getMaxFragBufSize, wqePipeOutVec[1]); // Request payload DataStream generation let simDmaReadSrv <- mkSimDmaReadSrvAndDataStreamPipeOut; @@ -304,27 +293,27 @@ module mkTestSendQueueRawPktCase(Empty); let sgl = vec(sge, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE, dummySGE); let wqe = WorkQueueElem { - id : dontCareValue, - opcode : IBV_WR_RDMA_READ, // dontCareValue - flags : enum2Flag(IBV_SEND_NO_FLAGS), - qpType : IBV_QPT_RAW_PACKET, - psn : 0, - pmtu : pmtu, - dqpIP : ipAddr, - macAddr: macAddr, - sgl : sgl, - raddr : dontCareValue, - rkey : dontCareValue, - // pkey : dontCareValue, - sqpn : getDefaultQPN, - dqpn : getDefaultQPN, - comp : tagged Invalid, - swap : tagged Invalid, + id : dontCareValue, + opcode : IBV_WR_RDMA_READ, // dontCareValue + flags : enum2Flag(IBV_SEND_NO_FLAGS), + qpType : IBV_QPT_RAW_PACKET, + psn : 0, + pmtu : pmtu, + dqpIP : ipAddr, + macAddr : macAddr, + sgl : sgl, + totalLen : zeroExtend(fixedLength), + raddr : dontCareValue, + rkey : dontCareValue, + sqpn : getDefaultQPN, + dqpn : getDefaultQPN, + comp : tagged Invalid, + swap : tagged Invalid, immDtOrInvRKey: tagged Invalid, - srqn : tagged Invalid, - qkey : tagged Invalid, - isFirst: True, - isLast : True + srqn : tagged Invalid, + qkey : tagged Invalid, + isFirst : True, + isLast : True }; wqeQ.enq(wqe); wqeRefQ.enq(wqe); @@ -440,6 +429,16 @@ module mkTestSendQueueNoPayloadCase(Empty); ); endmodule +(* doc = "testcase" *) +module mkTestSendQueueZeroPayloadLenCase(Empty); + let minDmaLength = 0; + let maxDmaLength = 0; + let genPayload = True; + let result <- mkTestSendQueueNormalAndNoPayloadCase( + genPayload, minDmaLength, maxDmaLength + ); +endmodule + module mkTestSendQueueNormalAndNoPayloadCase#( Bool genPayload, Length minDmaLength, Length maxDmaLength )(Empty); @@ -468,11 +467,7 @@ module mkTestSendQueueNormalAndNoPayloadCase#( headerAndMetaDataAndPayloadPipeOut.headerAndMetaData.headerDataStream, headerAndMetaDataAndPayloadPipeOut.headerAndMetaData.headerMetaData ); - // // Remove empty payload DataStream - // let filteredPayloadDataStreamPipeOut <- mkPipeFilter( - // filterEmptyDataStream, - // headerAndMetaDataAndPayloadPipeOut.payload - // ); + let udpInfoPipeOut4Ref <- mkBufferN(getMaxFragBufSize, dut.udpInfoPipeOut); FIFOF#(Tuple2#(HeaderByteNum, MAC)) headerLenAndMacAddrQ <- mkFIFOF; @@ -485,7 +480,14 @@ module mkTestSendQueueNormalAndNoPayloadCase#( // mkSink(wqePipeOut4Ref); // mkSink(dut.udpInfoPipeOut); // mkSink(dut.rdmaDataStreamPipeOut); - // mkSink(filteredPayloadDataStreamPipeOut); + // mkSink(rdmaHeaderPipeOut); + // mkSink(toPipeOut(headerLenAndMacAddrQ)); + // mkSink(headerAndMetaDataAndPayloadPipeOut.payload); + // mkSink(udpInfoPipeOut4Ref); + // rule discardSendResp if (!clearReg); + // let sendResp <- dut.srvPort.response.get; + // $display("time=%0t: discardSendResp", $time); + // endrule rule clearAll if (clearReg); clearReg <= False; From 30b16901262f1c798e8e42e8380fe31a50b7eb0a Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Sat, 3 Feb 2024 12:13:03 +0800 Subject: [PATCH 15/16] add IndexMR to DmaReadReq --- run_one.sh | 2 +- src/DataTypes.bsv | 17 +++++++++++++---- src/MetaData.bsv | 11 ----------- src/PayloadConAndGen.bsv | 6 ++---- src/PayloadGen.bsv | 3 ++- src/ReqGenSQ.bsv | 3 ++- src/ReqHandleRQ.bsv | 3 ++- src/Utils.bsv | 6 ++++++ test/SimDma.bsv | 6 ++++-- test/SimGenRdmaReqResp.bsv | 3 ++- test/TestPayloadConAndGen.bsv | 9 ++++++--- test/TestQueuePair.bsv | 3 ++- 12 files changed, 42 insertions(+), 30 deletions(-) diff --git a/run_one.sh b/run_one.sh index 6eb1759..989bd4b 100755 --- a/run_one.sh +++ b/run_one.sh @@ -13,7 +13,7 @@ TEST_LOG=run.log TEST_DIR=test cd $TEST_DIR truncate -s 0 $TEST_LOG -FILES=`ls TestPayloadGen.bsv` +FILES=`ls SimDma.bsv` for FILE in $FILES; do # echo $FILE TESTCASES=`grep -Phzo 'doc.*?\nmodule\s+\S+(?=\()' $FILE | xargs -0 -I {} echo "{}" | grep module | cut -d ' ' -f 2` diff --git a/src/DataTypes.bsv b/src/DataTypes.bsv index 73b0082..907e1b2 100644 --- a/src/DataTypes.bsv +++ b/src/DataTypes.bsv @@ -81,6 +81,10 @@ typedef TLog#(TLB_CACHE_SIZE) TLB_CACHE_INDEX_WIDTH; // 14 typedef TSub#(PHYSICAL_ADDR_WIDTH, PAGE_OFFSET_WIDTH) TLB_CACHE_PA_DATA_WIDTH; // 48-21=27 typedef TSub#(TSub#(ADDR_WIDTH, TLB_CACHE_INDEX_WIDTH), PAGE_OFFSET_WIDTH) TLB_CACHE_TAG_WIDTH; // 64-14-21=29 +typedef TDiv#(MAX_MR, MAX_PD) MAX_MR_PER_PD; +typedef TLog#(MAX_MR_PER_PD) MR_INDEX_WIDTH; +typedef TSub#(KEY_WIDTH, MR_INDEX_WIDTH) MR_KEY_PART_WIDTH; + // Derived types typedef Bit#(DATA_BUS_WIDTH) DATA; typedef Bit#(DATA_BUS_BYTE_WIDTH) ByteEn; @@ -140,6 +144,9 @@ typedef struct { typedef SizeOf#(PayloadTLB) TLB_PAYLOAD_WIDTH; +typedef UInt#(MR_INDEX_WIDTH) IndexMR; +typedef Bit#(MR_KEY_PART_WIDTH) KeyPartMR; + // Common types typedef Server#(DmaReadReq, DmaReadResp) DmaReadSrv; @@ -247,19 +254,21 @@ typedef struct { } PermCheckReq deriving(Bits, FShow); typedef struct { - DmaReqSrcType initiator; - QPN sqpn; + DmaReqSrcType initiator; // TODO: remove it + QPN sqpn; // TODO: remove it + WorkReqID wrID; // TODO: remove it ADDR startAddr; Length len; - WorkReqID wrID; + IndexMR mrIdx; } DmaReadMetaData deriving(Bits, FShow); typedef struct { DmaReqSrcType initiator; // TODO: remove it QPN sqpn; // TODO: remove it + WorkReqID wrID; // TODO: remove it ADDR startAddr; PktLen len; - WorkReqID wrID; // TODO: remove it + IndexMR mrIdx; } DmaReadReq deriving(Bits, FShow); typedef struct { diff --git a/src/MetaData.bsv b/src/MetaData.bsv index 400f1c0..c6aed8f 100644 --- a/src/MetaData.bsv +++ b/src/MetaData.bsv @@ -152,13 +152,6 @@ endmodule // MR related -typedef TDiv#(MAX_MR, MAX_PD) MAX_MR_PER_PD; -typedef TLog#(MAX_MR_PER_PD) MR_INDEX_WIDTH; -typedef TSub#(KEY_WIDTH, MR_INDEX_WIDTH) MR_KEY_PART_WIDTH; - -typedef UInt#(MR_INDEX_WIDTH) IndexMR; -typedef Bit#(MR_KEY_PART_WIDTH) KeyPartMR; - typedef struct { ADDR laddr; Length len; @@ -205,10 +198,6 @@ module mkMetaDataMRs(MetaDataMRs) provisos( return tuple2(lkey, rkey); endfunction - // function IndexMR lkey2IndexMR(LKEY lkey) = unpack(truncateLSB(lkey)); - // function IndexMR rkey2IndexMR(RKEY rkey) = unpack(truncateLSB(rkey)); - function IndexMR key2IndexMR(Bit#(KEY_WIDTH) key) = unpack(truncateLSB(key)); - interface srvPort = interface SrvPortMR; interface request = interface Put#(ReqMR); method Action put(ReqMR mrReq); diff --git a/src/PayloadConAndGen.bsv b/src/PayloadConAndGen.bsv index d6bdbf8..cdce073 100644 --- a/src/PayloadConAndGen.bsv +++ b/src/PayloadConAndGen.bsv @@ -252,16 +252,14 @@ module mkDmaReadCntrl#( let pendingDmaReadCntrlReq = pendingDmaCntrlReqQ.first; - // let dmaReadReq = pendingDmaReadCntrlReq.dmaReadReq; - // dmaReadReq.startAddr = addrChunkResp.chunkAddr; - // dmaReadReq.len = zeroExtend(addrChunkResp.chunkLen); let dmaReadMetaData = pendingDmaReadCntrlReq.dmaReadMetaData; let dmaReadReq = DmaReadReq { initiator: dmaReadMetaData.initiator, sqpn : dmaReadMetaData.sqpn, startAddr: addrChunkResp.chunkAddr, len : addrChunkResp.chunkLen, - wrID : dmaReadMetaData.wrID + wrID : dmaReadMetaData.wrID, + mrIdx : dmaReadMetaData.mrIdx }; dmaReadSrv.request.put(dmaReadReq); diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index 6a96461..75871bd 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -815,7 +815,8 @@ module mkDmaReadCntrl#( sqpn : curSQPN, startAddr: addrChunkResp.chunkAddr, len : addrChunkResp.chunkLen, - wrID : curWorkReqID + wrID : curWorkReqID, + mrIdx : key2IndexMR(lkey) }; dmaReadSrv.request.put(dmaReadReq); diff --git a/src/ReqGenSQ.bsv b/src/ReqGenSQ.bsv index af93907..731b440 100644 --- a/src/ReqGenSQ.bsv +++ b/src/ReqGenSQ.bsv @@ -731,7 +731,8 @@ module mkReqGenSQ#( sqpn : cntrlStatus.comm.getSQPN, startAddr: curPendingWR.wr.laddr, len : curPendingWR.wr.len, - wrID : curPendingWR.wr.id + wrID : curPendingWR.wr.id, + mrIdx : key2IndexMR(curPendingWR.wr.lkey) } }; diff --git a/src/ReqHandleRQ.bsv b/src/ReqHandleRQ.bsv index 413d825..1b8a918 100644 --- a/src/ReqHandleRQ.bsv +++ b/src/ReqHandleRQ.bsv @@ -2353,7 +2353,8 @@ module mkReqHandleRQ#( sqpn : cntrlStatus.comm.getSQPN, startAddr: permCheckReq.reqAddr, // reth.va len : permCheckReq.totalLen, // reth.dlen - wrID : dontCareValue + wrID : dontCareValue, + mrIdx : key2IndexMR(permCheckReq.rkey) } }; diff --git a/src/Utils.bsv b/src/Utils.bsv index 7794693..e110bc0 100644 --- a/src/Utils.bsv +++ b/src/Utils.bsv @@ -413,6 +413,12 @@ function Bool isDefaultPKEY(PKEY pkey); return isAllOnesR(pkey); endfunction +function IndexMR key2IndexMR(Bit#(nSz) key) provisos( + Add#(KEY_WIDTH, 0, nSz) +); + return unpack(truncateLSB(key)); +endfunction + function ADDR addrAddPsnMultiplyPMTU(ADDR addr, PSN psn, PMTU pmtu); return case (pmtu) IBV_MTU_256 : begin diff --git a/test/SimDma.bsv b/test/SimDma.bsv index 31cc11d..9744ef2 100644 --- a/test/SimDma.bsv +++ b/test/SimDma.bsv @@ -393,7 +393,8 @@ module mkFixedPktLenDataStreamPipeOut#( sqpn : getDefaultQPN, startAddr: dontCareValue, len : pktLen, - wrID : dontCareValue + wrID : dontCareValue, + mrIdx : dontCareValue }; simDmaReadSrv.request.put(dmaReq); // $display("time=%0t: pktLen=%0d", $time, pktLen); @@ -489,7 +490,8 @@ module mkTestDmaReadAndWriteSrv(Empty); sqpn : getDefaultQPN, startAddr: dontCareValue, len : pktLen, - wrID : dontCareValue + wrID : dontCareValue, + mrIdx : dontCareValue }; simDmaReadSrv.dmaReadSrv.request.put(dmaReadReq); diff --git a/test/SimGenRdmaReqResp.bsv b/test/SimGenRdmaReqResp.bsv index a21fe18..0673d19 100644 --- a/test/SimGenRdmaReqResp.bsv +++ b/test/SimGenRdmaReqResp.bsv @@ -374,7 +374,8 @@ module mkSimGenRdmaRespHeaderAndDataStream#( sqpn : cntrlStatus.comm.getSQPN, startAddr: curPendingWR.wr.laddr, len : curPendingWR.wr.len, - wrID : curPendingWR.wr.id + wrID : curPendingWR.wr.id, + mrIdx : key2IndexMR(curPendingWR.wr.lkey) } }; payloadGenerator.srvPort.request.put(payloadGenReq); diff --git a/test/TestPayloadConAndGen.bsv b/test/TestPayloadConAndGen.bsv index e1e064f..8a0e30e 100644 --- a/test/TestPayloadConAndGen.bsv +++ b/test/TestPayloadConAndGen.bsv @@ -231,7 +231,8 @@ module mkTestDmaReadCntrlNormalOrCancelCase#(Bool normalOrCancelCase)(Empty); sqpn : cntrlStatus.comm.getSQPN, startAddr: startAddr, len : payloadLen, - wrID : dontCareValue + wrID : dontCareValue, + mrIdx : dontCareValue } }; @@ -663,7 +664,8 @@ module mkTestPayloadConAndGenNormalCase(Empty); sqpn : cntrlStatus.comm.getSQPN, startAddr: dontCareValue, len : zeroExtend(pktLen), - wrID : dontCareValue + wrID : dontCareValue, + mrIdx : dontCareValue } }; payloadGenerator.srvPort.request.put(payloadGenReq); @@ -805,7 +807,8 @@ module mkTestPayloadGenSegmentAndPaddingCase(Empty); sqpn : cntrlStatus.comm.getSQPN, startAddr: dontCareValue, len : payloadLen, - wrID : dontCareValue + wrID : dontCareValue, + mrIdx : dontCareValue } }; payloadGenerator.srvPort.request.put(payloadGenReq); diff --git a/test/TestQueuePair.bsv b/test/TestQueuePair.bsv index c311ba0..c45b31a 100644 --- a/test/TestQueuePair.bsv +++ b/test/TestQueuePair.bsv @@ -108,7 +108,8 @@ module mkSimDmaReadClt(SimDmaReadClt) provisos( sqpn : dontCareValue, startAddr: dontCareValue, len : 1025, - wrID : dontCareValue + wrID : dontCareValue, + mrIdx : dontCareValue }; reqQ.enq(dmaReadReq); From d7867ba7c5c3f55218272cb9741814ca2e659bf8 Mon Sep 17 00:00:00 2001 From: Pu Wang Date: Sat, 3 Feb 2024 23:53:32 +0800 Subject: [PATCH 16/16] clean up commented code --- run_one.sh | 4 +-- src/PayloadGen.bsv | 67 +++------------------------------------------- src/SendQ.bsv | 6 ----- 3 files changed, 6 insertions(+), 71 deletions(-) diff --git a/run_one.sh b/run_one.sh index 989bd4b..efd11bc 100755 --- a/run_one.sh +++ b/run_one.sh @@ -9,11 +9,11 @@ if [ -f "$BASH_PROFILE" ]; then source $BASH_PROFILE fi -TEST_LOG=run.log +TEST_LOG=test.log TEST_DIR=test cd $TEST_DIR truncate -s 0 $TEST_LOG -FILES=`ls SimDma.bsv` +FILES=`ls TestSendQ.bsv` for FILE in $FILES; do # echo $FILE TESTCASES=`grep -Phzo 'doc.*?\nmodule\s+\S+(?=\()' $FILE | xargs -0 -I {} echo "{}" | grep module | cut -d ' ' -f 2` diff --git a/src/PayloadGen.bsv b/src/PayloadGen.bsv index 75871bd..0297957 100644 --- a/src/PayloadGen.bsv +++ b/src/PayloadGen.bsv @@ -87,13 +87,13 @@ instance FShow#(WorkQueueElem); ", raddr=%h", wqe.raddr, ", sqpn=%h", wqe.sqpn, ", dqpn=%h", wqe.dqpn, - // ", solicited=", fshow(wqe.solicited), ", comp=", fshow(wqe.comp), ", swap=", fshow(wqe.swap), ", immDtOrInvRKey=", fshow(wqe.immDtOrInvRKey), ", srqn=", fshow(wqe.srqn), - ", dqpn=", fshow(wqe.dqpn), - ", qkey=", fshow(wqe.qkey), " }" + ", qkey=", fshow(wqe.qkey), + ", isFirst=", fshow(wqe.isFirst), + ", isLast=", fshow(wqe.isLast), " }" ); endfunction endinstance @@ -108,18 +108,6 @@ typedef struct { typedef Vector#(MAX_SGE, ScatterGatherElem) ScatterGatherList; -// typedef struct { -// // The last fragment ByteEn for each packet of the SGE -// ByteEnBitNum curPktLastFragValidByteNum; -// PktLen pktLen; -// PMTU pmtu; -// Bool sgeHasJustTwoPkts; -// Bool isFirst; -// Bool isLast; -// Bool isFirstSGE; -// Bool isLastSGE; -// } PktMetaDataSGE deriving(Bits, FShow); - typedef struct { PktLen firstPktLen; PktLen lastPktLen; @@ -128,9 +116,6 @@ typedef struct { } PktMetaDataSGE deriving(Bits, FShow); typedef struct { - // PktLen lastPktLen; - // PktFragNum lastPktFragNum; - // PktNum sgePktNum; ByteEnBitNum lastFragValidByteNum; Bool isFirst; Bool isLast; @@ -147,15 +132,7 @@ typedef struct { PktLen firstPktLen; PktFragNum firstPktFragNum; ByteEnBitNum firstPktLastFragValidByteNum; - // PAD firstPktPadCnt; - // ByteEn firstPktLastFragByteEn; - // PktLen lastPktLen; - // PktFragNum lastPktFragNum; - // ByteEnBitNum lastPktLastFragValidByteNum; - // PAD lastPktPadCnt; - // ByteEn lastPktLastFragByteEn; ByteEnBitNum origLastFragValidByteNum; - // PktLen pmtuLen; PktNum adjustedPktNum; PktNum origPktNum; PMTU pmtu; @@ -270,9 +247,7 @@ function Tuple5#(PktLen, PktLen, PktLen, PktNum, ADDR) calcPktNumAndPktLenByAddr let notFullPkt = isZeroR(truncatedPktNum); let totalPktNum = truncatedPktNum + zeroExtend(residuePktNum) + zeroExtend(extraPktNum); - // let firstPktLen = notFullPkt ? (hasExtraPkt ? maxFirstPktLen : lenLowPart) : maxFirstPktLen; let firstPktLen = (notFullPkt && !hasExtraPkt) ? lenLowPart : maxFirstPktLen; - // let lastPktLen = notFullPkt ? (hasResidue ? tmpLastPktLen : 0) : (hasResidue ? tmpLastPktLen : pmtuLen); let lastPktLen = notFullPkt ? (hasExtraPkt ? tmpLastPktLen : lenLowPart) : (hasResidue ? tmpLastPktLen : pmtuLen); // let isSinglePkt = isLessOrEqOneR(totalPktNum); @@ -283,7 +258,6 @@ function PktFragNum calcFragNumByPktLen(PktLen pktLen) provisos( Add#(PMTU_FRAG_NUM_WIDTH, DATA_BUS_BYTE_NUM_WIDTH, PKT_LEN_WIDTH) ); BusByteWidthMask lastFragByteNumResidue = truncate(pktLen); - // Bit#(TSub#(PKT_LEN_WIDTH, DATA_BUS_BYTE_NUM_WIDTH)) truncatedPktLen = PktFragNum truncatedPktLen = truncateLSB(pktLen); let pktFragNum = truncatedPktLen + zeroExtend(pack(!isZeroR(lastFragByteNumResidue))); return pktFragNum; @@ -898,10 +872,6 @@ endmodule typedef enum { MERGE_SGE_PAYLOAD_INIT, - // MERGE_SGE_PAYLOAD_ONLY_PKT, - // MERGE_SGE_PAYLOAD_FIRST_PKT, - // MERGE_SGE_PAYLOAD_MID_PKT, - // MERGE_SGE_PAYLOAD_LAST_PKT, MERGE_SGE_PAYLOAD_FIRST_OR_MID_PKT, MERGE_SGE_PAYLOAD_LAST_OR_ONLY_PKT } MergePayloadStateEachSGE deriving(Bits, Eq, FShow); @@ -1151,10 +1121,6 @@ typedef enum { MERGE_SGL_PAYLOAD_INIT, MERGE_SGL_PAYLOAD_FIRST_OR_MID_SGE, MERGE_SGL_PAYLOAD_LAST_OR_ONLY_SGE - // MERGE_SGL_PAYLOAD_FIRST_SGE, - // MERGE_SGL_PAYLOAD_MID_SGE, - // MERGE_SGL_PAYLOAD_LAST_SGE, - // MERGE_SGL_PAYLOAD_ONLY_SGE } MergePayloadStateAllSGE deriving(Bits, Eq, FShow); module mkMergePayloadAllSGE#( @@ -1269,8 +1235,6 @@ module mkMergePayloadAllSGE#( let sgeMergedMetaData = sgeMergedMetaDataPipeIn.first; sgeMergedMetaDataPipeIn.deq; - // let isOnlySGE = sgeMergedMetaData.isFirst && sgeMergedMetaData.isLast; - // sgeIsOnlyReg <= isOnlySGE; sgeIsLastReg <= sgeMergedMetaData.isLast; let lastFragValidByteNum = sgeMergedMetaData.lastFragValidByteNum; @@ -1367,7 +1331,6 @@ module mkMergePayloadAllSGE#( prePayloadFragReg <= nextPrePayloadFrag; let outPayloadFrag = mergedFrag; - // outPayloadFrag.isFirst = isFirstFragReg; outPayloadFrag.isLast = False; if (shouldOutput) begin isFirstFragReg <= False; @@ -1456,11 +1419,6 @@ typedef enum { ADJUST_PAYLOAD_SEGMENT_INIT, ADJUST_PAYLOAD_SEGMENT_FIRST_OR_MID_PKT, ADJUST_PAYLOAD_SEGMENT_LAST_OR_ONLY_PKT - // ADJUST_PAYLOAD_SEGMENT_FIRST_PKT, - // ADJUST_PAYLOAD_SEGMENT_MID_PKT, - // ADJUST_PAYLOAD_SEGMENT_LAST_PKT, - // ADJUST_PAYLOAD_SEGMENT_ONLY_PKT, - // ADJUST_PAYLOAD_SEGMENT_LAST_PKT_EXTRA_FRAG } AdjustPayloadSegmentState deriving(Bits, Eq, FShow); // TODO: output target address, isFirst, isLast of each packet @@ -1501,7 +1459,6 @@ module mkAdjustPayloadSegment#( }; rule resetAndClear if (clearAll); - // pktLenOutQ.clear; pktPayloadOutQ.clear; stateReg <= ADJUST_PAYLOAD_SEGMENT_INIT; endrule @@ -1542,10 +1499,6 @@ module mkAdjustPayloadSegment#( let sglHasOnlyPkt = isOneR(adjustedTotalPayloadMeta.adjustedPktNum); sglHasOnlyPktReg <= sglHasOnlyPkt; let hasExtraFrag = firstPktLastFragValidByteNum < origLastFragValidByteNum; - // let hasExtraFrag = ( - // { 1'b0, firstPktLastFragInvalidByteNum } + - // { 1'b0, origLastFragValidByteNum } - // ) > fromInteger(valueOf(DATA_BUS_BYTE_WIDTH)); hasExtraFragReg <= hasExtraFrag; if (sglHasOnlyPkt) begin @@ -1562,7 +1515,6 @@ module mkAdjustPayloadSegment#( let sglHasOnlyFragOnlyPkt = isOrigOnlyPkt && curPayloadFrag.isLast; sglHasOnlyFragOnlyPktReg <= sglHasOnlyFragOnlyPkt; - // isFirstFragReg <= True; isFirstPktReg <= True; // $display( // "time=%0t: prepareNextSGL", $time, @@ -1629,11 +1581,9 @@ module mkAdjustPayloadSegment#( isFirstPktReg <= False; end - // isFirstFragReg <= True; sglRemainingPktNumReg <= sglRemainingPktNumReg - 1; if (isTwoR(sglRemainingPktNumReg)) begin - // pktRemainingFragNumReg <= lastPktFragNumReg; stateReg <= ADJUST_PAYLOAD_SEGMENT_LAST_OR_ONLY_PKT; end else begin @@ -1642,12 +1592,10 @@ module mkAdjustPayloadSegment#( end end else begin - // isFirstFragReg <= False; pktRemainingFragNumReg <= pktRemainingFragNumReg - 1; end - // outPayloadFrag.isFirst = isFirstFragReg; - outPayloadFrag.isLast = isLastFrag; + outPayloadFrag.isLast = isLastFrag; pktPayloadOutQ.enq(outPayloadFrag); // $display( // "time=%0t: adjustFirstOrMidPkt", $time, @@ -1716,8 +1664,6 @@ module mkAdjustPayloadSegment#( let outPayloadFrag = prePayloadFragReg; if (!sglHasOnlyPktReg) begin - // isFirstFragReg <= False; - // outPayloadFrag.isFirst = isFirstFragReg; outPayloadFrag = mergedFrag; outPayloadFrag.isLast = isLastFrag; end @@ -1999,11 +1945,6 @@ module mkPayloadGenerator#( firstPktLen : firstPktLen, firstPktFragNum : firstPktFragNum, firstPktLastFragValidByteNum : firstPktLastFragValidByteNum, - // firstPktPadCnt : firstPktPadCnt, - // lastPktLen : lastPktLen, - // lastPktFragNum : lastPktFragNum, - // lastPktLastFragValidByteNum : lastPktLastFragValidByteNum, - // lastPktPadCnt : lastPktPadCnt, origLastFragValidByteNum : origLastFragValidByteNum, adjustedPktNum : totalPktNum, origPktNum : origPktNum, diff --git a/src/SendQ.bsv b/src/SendQ.bsv index eeb0a8a..8b3e09b 100644 --- a/src/SendQ.bsv +++ b/src/SendQ.bsv @@ -834,11 +834,6 @@ module mkSendQ#( padCnt = payloadGenResp.padCnt; wqeLastPkt = payloadGenResp.isLast; end - // let payloadGenResp <- payloadGenerator.srvPort.response.get; - // let remoteAddr = payloadGenResp.raddr; - // let pktPayloadLen = payloadGenResp.pktLen; - // let padCnt = payloadGenResp.padCnt; - // let wqeLastPkt = payloadGenResp.isLast; wqeFirstPktReg <= wqeLastPkt; if (wqeLastPkt) begin @@ -1041,7 +1036,6 @@ module mkSendQ#( endrule interface srvPort = toGPServer(reqQ, respQ); - // interface wqeInPut = toPut(reqQ); interface rdmaDataStreamPipeOut = rdmaPktDataStreamPipeOut; interface udpInfoPipeOut = toPipeOut(udpPktInfoOutQ); method Bool isEmpty() = !(