ch4/ofi: move the util.iov into MPIDI_OFI_request_t.u

The util.iov field is used by fi_trecvmsg with FI_CLAIM flag or the huge recv path for threshold checking and fi_read.
pmodels · Feb 6, 2024 · 0cf1638 · 0cf1638
1 parent 1e8f257
commit 0cf1638
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 11 deletions.
diff --git a/src/mpid/ch4/netmod/ofi/ofi_huge.c b/src/mpid/ch4/netmod/ofi/ofi_huge.c
@@ -23,7 +23,7 @@ static int get_huge(MPIR_Request * rreq)
         cur_offset = MPIDI_OFI_global.max_msg_size;
     }
 
-    MPI_Aint data_sz = MPIDI_OFI_REQUEST(rreq, util.iov.iov_len);
+    MPI_Aint data_sz = MPIDI_OFI_REQUEST(rreq, u.recv.msg_iov.iov_len);
 
     if (data_sz < info->msgsize) {
         rreq->status.MPI_ERROR = MPI_ERR_TRUNCATE;
@@ -69,7 +69,7 @@ static int get_huge_issue_read(MPIR_Request * rreq)
     }
     bytesLeft = info->msgsize - cur_offset;
 
-    void *recv_buf = MPIDI_OFI_REQUEST(rreq, util.iov.iov_base);
+    void *recv_buf = MPIDI_OFI_REQUEST(rreq, u.recv.msg_iov.iov_base);
 
     MPI_Aint chunk_size;
     if (MPIDI_OFI_COMM(comm).enable_striping) {

diff --git a/src/mpid/ch4/netmod/ofi/ofi_pre.h b/src/mpid/ch4/netmod/ofi/ofi_pre.h
@@ -234,6 +234,10 @@ typedef struct {
             void *buf;
             MPI_Aint count;
             MPI_Datatype datatype;
+            struct iovec msg_iov;       /* FI_CLAIM require fi_trecvmsg which require usage of iov.
+                                         * We always set it with {recv_buf, data_sz} since they are
+                                         * useful for the huge recv path as well.
+                                         */
         } recv;
         struct {
             struct iovec *iovs;
@@ -254,9 +258,6 @@ typedef struct {
             MPI_Datatype datatype;
         } pipeline_recv;
     } u;
-    union {
-        struct iovec iov;
-    } util;
 } MPIDI_OFI_request_t;
 
 typedef struct {

diff --git a/src/mpid/ch4/netmod/ofi/ofi_recv.h b/src/mpid/ch4/netmod/ofi/ofi_recv.h
@@ -263,9 +263,6 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_do_irecv(void *buf,
 #endif
         MPIDI_OFI_REQUEST(rreq, u.recv.count) = count;
         MPIDI_OFI_REQUEST(rreq, u.recv.datatype) = datatype;
-    } else {
-        /* MPIDI_OFI_EVENT_RECV_PACK may get overwritten with MPIDI_OFI_EVENT_RECV_HUGE */
-        MPIDI_OFI_REQUEST(rreq, u.recv.pack_buffer) = NULL;
     }
 
     if (rreq->comm == NULL) {
@@ -274,8 +271,9 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_do_irecv(void *buf,
     }
     /* Read ordering unnecessary for context_id, so use relaxed load */
     MPL_atomic_relaxed_store_int(&MPIDI_OFI_REQUEST(rreq, util_id), context_id);
-    MPIDI_OFI_REQUEST(rreq, util.iov.iov_base) = recv_buf;
-    MPIDI_OFI_REQUEST(rreq, util.iov.iov_len) = data_sz;
+    /* msg_iov is needed to use fi_trecvmsg (with FI_CLAIM) or the huge recv path */
+    MPIDI_OFI_REQUEST(rreq, u.recv.msg_iov.iov_base) = recv_buf;
+    MPIDI_OFI_REQUEST(rreq, u.recv.msg_iov.iov_len) = data_sz;
 
     if (unlikely(data_sz >= MPIDI_OFI_global.max_msg_size) && !MPIDI_OFI_COMM(comm).enable_striping) {
         MPIDI_OFI_REQUEST(rreq, event_id) = MPIDI_OFI_EVENT_RECV_HUGE;
@@ -303,7 +301,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_do_irecv(void *buf,
                                       (void *) &(MPIDI_OFI_REQUEST(rreq, context))), vci_local,
                              trecv);
     } else {
-        msg.msg_iov = &MPIDI_OFI_REQUEST(rreq, util.iov);
+        msg.msg_iov = &MPIDI_OFI_REQUEST(rreq, u.recv.msg_iov);
         msg.desc = desc;
         msg.iov_count = 1;
         msg.tag = match_bits;