Skip to content

Commit

Permalink
posix: add numa support to RMA shared memory window code
Browse files Browse the repository at this point in the history
Similarly to pt2pt fastbox integration this patch decomposes current
single shared segment into multiple segments, one per NUMA node, that
can then be separately bound using hwloc. Moreover, when using symheap
either all single segment allocations succeed or none of them does. If a
symheap segment allocation fails all the previous should be reverted. In
order to accomplish this the new function:
`MPIDI_CH4R_release_shm_symheap` has been introduced.
  • Loading branch information
Giuseppe Congiu committed Feb 12, 2019
1 parent f99604f commit 6b11e8d
Show file tree
Hide file tree
Showing 3 changed files with 267 additions and 59 deletions.
7 changes: 4 additions & 3 deletions src/mpid/ch4/include/mpidpre.h
Original file line number Diff line number Diff line change
Expand Up @@ -366,10 +366,11 @@ typedef struct MPIDI_CH4U_win_target {

typedef struct MPIDI_CH4U_win_t {
uint64_t win_id;
void *mmap_addr;
int64_t mmap_sz;
void **mmap_addr;
int64_t *mmap_sz;
int num_seg;

MPL_shm_hnd_t shm_segment_handle;
MPL_shm_hnd_t *shm_segment_handle;

/* per-window OP completion for fence */
MPIR_cc_t local_cmpl_cnts; /* increase at OP issuing, decrease at local completion */
Expand Down
31 changes: 28 additions & 3 deletions src/mpid/ch4/src/ch4r_symheap.h
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,8 @@ static inline int MPIDI_CH4I_allreduce_maxloc(size_t mysz, int myloc, MPIR_Comm
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
static inline int MPIDI_CH4R_get_shm_symheap(MPI_Aint shm_size, MPI_Aint * shm_offsets,
MPIR_Comm * comm, MPIR_Win * win, int *fail_flag)
MPIR_Comm * comm, MPIR_Win * win, int seg_num,
int *fail_flag)
{
int mpi_errno = MPI_SUCCESS;
unsigned any_mapfail_flag = 1;
Expand All @@ -398,8 +399,8 @@ static inline int MPIDI_CH4R_get_shm_symheap(MPI_Aint shm_size, MPI_Aint * shm_o

#ifdef USE_SYM_HEAP
int iter = MPIR_CVAR_CH4_SHM_SYMHEAP_RETRY;
MPL_shm_hnd_t *shm_segment_hdl_ptr = &MPIDI_CH4U_WIN(win, shm_segment_handle);
void **base_ptr = &MPIDI_CH4U_WIN(win, mmap_addr);
MPL_shm_hnd_t *shm_segment_hdl_ptr = &MPIDI_CH4U_WIN(win, shm_segment_handle[seg_num]);
void **base_ptr = &MPIDI_CH4U_WIN(win, mmap_addr[seg_num]);

size_t mapsize = 0, page_sz = 0, maxsz = 0;
int maxsz_loc = 0;
Expand Down Expand Up @@ -505,4 +506,28 @@ static inline int MPIDI_CH4R_get_shm_symheap(MPI_Aint shm_size, MPI_Aint * shm_o
return mpi_errno;
}

#undef FUNCNAME
#define FUNCNAME MPIDI_CH4R_release_shm_symheap
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
static inline int MPIDI_CH4R_release_shm_symheap(MPI_Aint shm_size, MPIR_Win * win, int seg_num)
{
int mpi_errno = MPI_SUCCESS;

#ifdef USE_SYM_HEAP
MPL_shm_hnd_t *shm_segment_hdl_ptr = &MPIDI_CH4U_WIN(win, shm_segment_handle[seg_num]);
void *base_ptr = MPIDI_CH4U_WIN(win, mmap_addr[seg_num]);

/* destroy successful shm segment */
mpi_errno = MPIDI_CH4U_destroy_shm_segment(shm_size, shm_segment_hdl_ptr, base_ptr);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
#endif

fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}

#endif /* CH4R_SYMHEAP_H_INCLUDED */
Loading

0 comments on commit 6b11e8d

Please sign in to comment.