Skip to content

Commit

Permalink
ch4: implement MPIR_Data send/recv in netmod/shm/mpidig (WIP)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hzhou
hzhou committed Sep 4, 2024
1 parent 824a6f6 commit 3cc7ccb
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 17 deletions.
7 changes: 7 additions & 0 deletions src/mpid/ch4/ch4_api.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,12 +138,18 @@ Non Native API:
SHM*: req

Native API:
send_data : int
NM*: data-2, rank, tag, comm, attr-2, addr, req_p
SHM*: data-2, rank, tag, comm, attr-2, addr, req_p
mpi_isend : int
NM*: buf, count, datatype, rank, tag, comm, attr-2, addr, req_p
SHM*: buf, count, datatype, rank, tag, comm, attr-2, addr, req_p
mpi_cancel_send : int
NM*: sreq
SHM*: sreq
recv_data : int
NM*: data-2, rank, tag, comm, attr-2, addr, req_p, partner
SHM*: data-2, rank, tag, comm, attr-2, req_p
mpi_irecv : int
NM*: buf-2, count, datatype, rank, tag, comm, attr-2, addr, req_p, partner
SHM*: buf-2, count, datatype, rank, tag, comm, attr-2, req_p
Expand Down Expand Up @@ -446,6 +452,7 @@ PARAM:
context_id: MPIR_Context_id_t
count: MPI_Aint
data: const void *
data-2: MPIR_Data *
data_sz: MPI_Aint
datatype: MPI_Datatype
datatype_p: MPIR_Datatype *
Expand Down
57 changes: 40 additions & 17 deletions src/mpid/ch4/src/mpidig_send.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,7 @@ MPL_STATIC_INLINE_PREFIX bool MPIDIG_check_eager(int is_local, MPI_Aint am_hdr_s
#endif
}

MPL_STATIC_INLINE_PREFIX int MPIDIG_isend_impl(const void *buf, MPI_Aint count,
MPI_Datatype datatype, int rank, int tag,
MPL_STATIC_INLINE_PREFIX int MPIDIG_isend_impl(MPIR_Data * data, int rank, int tag,
MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, uint8_t flags,
int src_vci, int dst_vci,
Expand Down Expand Up @@ -78,29 +77,35 @@ MPL_STATIC_INLINE_PREFIX int MPIDIG_isend_impl(const void *buf, MPI_Aint count,
MPIR_cc_inc(sreq->cc_ptr); /* expecting SSEND_ACK */
}
MPI_Aint data_sz;
MPIDI_Datatype_check_size(datatype, count, data_sz);
if (data->length > 0) {
data_sz = data->length;
} else {
MPIDI_Datatype_check_size(data->datatype, data->count, data_sz);
}
am_hdr.data_sz = data_sz;
am_hdr.rndv_hdr_sz = 0;

#ifdef HAVE_DEBUGGER_SUPPORT
MPIDIG_REQUEST(sreq, datatype) = datatype;
MPIDIG_REQUEST(sreq, buffer) = (void *) buf;
MPIDIG_REQUEST(sreq, count) = count;
/* FIXME: partial data support in debugger */
MPIDIG_REQUEST(sreq, datatype) = data->datatype;
MPIDIG_REQUEST(sreq, buffer) = data->buf;
MPIDIG_REQUEST(sreq, count) = data->count;
#endif

int is_local = MPIDI_av_is_local(addr);
MPI_Aint am_hdr_sz = (MPI_Aint) sizeof(am_hdr);
if (MPIDIG_check_eager(is_local, am_hdr_sz, data_sz, sreq)) {
/* EAGER send */
CH4_CALL(am_isend(rank, comm, MPIDIG_SEND, &am_hdr, am_hdr_sz, buf, count, datatype,
src_vci, dst_vci, sreq), is_local, mpi_errno);
CH4_CALL(am_isend(rank, comm, MPIDIG_SEND, &am_hdr, am_hdr_sz,
data->buf, data->count, data->datatype, src_vci, dst_vci, sreq),
is_local, mpi_errno);
} else {
/* RNDV send */
MPIDIG_REQUEST(sreq, buffer) = (void *) buf;
MPIDIG_REQUEST(sreq, count) = count;
MPIDIG_REQUEST(sreq, datatype) = datatype;
MPIDIG_REQUEST(sreq, buffer) = data->buf;
MPIDIG_REQUEST(sreq, count) = data->count;
MPIDIG_REQUEST(sreq, datatype) = data->datatype;
MPIDIG_REQUEST(sreq, u.send.dest) = rank;
MPIR_Datatype_add_ref_if_not_builtin(datatype);
MPIR_Datatype_add_ref_if_not_builtin(data->datatype);
MPIDIG_AM_SEND_SET_RNDV(am_hdr.flags, MPIDIG_RNDV_GENERIC);

CH4_CALL(am_send_hdr(rank, comm, MPIDIG_SEND, &am_hdr, am_hdr_sz, src_vci, dst_vci),
Expand All @@ -115,10 +120,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDIG_isend_impl(const void *buf, MPI_Aint count,
goto fn_exit;
}

MPL_STATIC_INLINE_PREFIX int MPIDIG_mpi_isend(const void *buf,
MPI_Aint count,
MPI_Datatype datatype,
int rank,
MPL_STATIC_INLINE_PREFIX int MPIDIG_send_data(MPIR_Data * data, int rank,
int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, int src_vci, int dst_vci,
MPIR_Request ** request,
Expand All @@ -128,13 +130,34 @@ MPL_STATIC_INLINE_PREFIX int MPIDIG_mpi_isend(const void *buf,
MPIR_FUNC_ENTER;

uint8_t flags = syncflag ? MPIDIG_AM_SEND_FLAGS_SYNC : MPIDIG_AM_SEND_FLAGS_NONE;
mpi_errno = MPIDIG_isend_impl(buf, count, datatype, rank, tag, comm, context_offset, addr,
mpi_errno = MPIDIG_isend_impl(data, rank, tag, comm, context_offset, addr,
flags, src_vci, dst_vci, request, errflag);

MPIR_FUNC_EXIT;
return mpi_errno;
}

MPL_STATIC_INLINE_PREFIX int MPIDIG_mpi_isend(const void *buf,
MPI_Aint count,
MPI_Datatype datatype,
int rank,
int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, int src_vci, int dst_vci,
MPIR_Request ** request,
bool syncflag, MPIR_Errflag_t errflag)
{
MPIR_Data data = {
.buf = (void *) buf,
.count = count,
.datatype = datatype,
.offset = 0,
.length = -1,
};

return MPIDIG_send_data(&data, rank, tag, comm, context_offset, addr, src_vci, dst_vci,
request, syncflag, errflag);
}

MPL_STATIC_INLINE_PREFIX int MPIDIG_mpi_cancel_send(MPIR_Request * sreq)
{
int mpi_errno = MPI_SUCCESS;
Expand Down

0 comments on commit 3cc7ccb

Please sign in to comment.