ch4/ofi: enable variable av entry size

Declare MPIDI_OFI_addr_t as dest[1] and set av_entry_size at init time based on runtime settings.
pmodels · Nov 9, 2024 · ca6244f · ca6244f
1 parent b1e9ba1
commit ca6244f
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 7 deletions.
diff --git a/src/mpid/ch4/netmod/ofi/ofi_init.c b/src/mpid/ch4/netmod/ofi/ofi_init.c
@@ -779,7 +779,8 @@ int MPIDI_OFI_init_local(int *tag_bits)
     ofi_am_init(0);
     ofi_am_post_recv(0, 0);
 
-    MPIDI_global.av_entry_size = sizeof(MPIDI_av_entry_t);
+    MPIDI_global.av_entry_size = sizeof(MPIDI_av_entry_t) +
+        sizeof(fi_addr_t) * (MPIDI_OFI_global.num_nics * MPIDI_OFI_global.max_vcis - 1);
 
   fn_exit:
     *tag_bits = MPIDI_OFI_TAG_BITS;

diff --git a/src/mpid/ch4/netmod/ofi/ofi_pre.h b/src/mpid/ch4/netmod/ofi/ofi_pre.h
@@ -313,11 +313,7 @@ typedef struct {
 
 /* The actual addr array is dest[MPIDI_OFI_global.num_nics][MPIDI_OFI_global.max_vcis] */
 typedef struct {
-#ifdef MPIDI_OFI_VNI_USE_DOMAIN
-    fi_addr_t dest[MPIDI_OFI_MAX_NICS][MPIDI_CH4_MAX_VCIS];     /* [nic][vci] */
-#else
-    fi_addr_t dest[MPIDI_OFI_MAX_NICS][1];
-#endif
+    fi_addr_t dest[1];
 } MPIDI_OFI_addr_t;
 
 #endif /* OFI_PRE_H_INCLUDED */
diff --git a/src/mpid/ch4/netmod/ofi/ofi_types.h b/src/mpid/ch4/netmod/ofi/ofi_types.h
@@ -170,7 +170,7 @@ static inline uint32_t MPIDI_OFI_idata_get_gpuchunk_bits(uint64_t idata)
 #define MPIDI_OFI_AM_RREQ_HDR(req,field) ((req)->dev.ch4.am.netmod_am.ofi.rreq_hdr->field)
 #define MPIDI_OFI_REQUEST(req,field)       ((req)->dev.ch4.netmod.ofi.field)
 #define MPIDI_OFI_AV(av)                   ((av)->netmod.ofi)
-#define MPIDI_OFI_AV_ADDR(av, nic, vci)    ((av)->netmod.ofi.dest[nic][vci])
+#define MPIDI_OFI_AV_ADDR(av, nic, vci)    ((av)->netmod.ofi.dest[nic * MPIDI_OFI_global.max_vcis + vci])
 
 #define MPIDI_OFI_COMM(comm)     ((comm)->dev.ch4.netmod.ofi)