From f55fb09f5d40d70044a18781ee5b613ebdc37bad Mon Sep 17 00:00:00 2001 From: Thomas Naughton Date: Thu, 15 Oct 2020 15:56:47 -0400 Subject: [PATCH 1/3] add timer 'OMPI_SPC_TIME_ALLTOALL' for alltoall Signed-off-by: Thomas Naughton --- ompi/mpi/c/alltoall.c | 6 ++++++ ompi/runtime/ompi_spc.c | 2 ++ ompi/runtime/ompi_spc.h | 1 + 3 files changed, 9 insertions(+) diff --git a/ompi/mpi/c/alltoall.c b/ompi/mpi/c/alltoall.c index 303b82a2e81..41030f9470a 100644 --- a/ompi/mpi/c/alltoall.c +++ b/ompi/mpi/c/alltoall.c @@ -51,6 +51,7 @@ int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, { int err; size_t recvtype_size; + opal_timer_t timer = 0; /* SPC */ SPC_RECORD(OMPI_SPC_ALLTOALL, 1); @@ -106,10 +107,15 @@ int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, OPAL_CR_ENTER_LIBRARY(); + SPC_TIMER_START(OMPI_SPC_TIME_ALLTOALL, &timer); + /* Invoke the coll component to perform the back-end operation */ err = comm->c_coll->coll_alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, comm->c_coll->coll_alltoall_module); + + SPC_TIMER_STOP(OMPI_SPC_TIME_ALLTOALL, &timer); + OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index 099934c658c..d4e70165bac 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -120,6 +120,7 @@ static ompi_spc_event_t ompi_spc_events_names[OMPI_SPC_NUM_COUNTERS] = { SET_COUNTER_ARRAY(OMPI_SPC_TESTALL, "The number of times MPI_Testall was called."), SET_COUNTER_ARRAY(OMPI_SPC_TESTANY, "The number of times MPI_Testany was called."), SET_COUNTER_ARRAY(OMPI_SPC_TESTSOME, "The number of times MPI_Testsome was called."), + SET_COUNTER_ARRAY(OMPI_SPC_TIME_ALLTOALL, "The number microseconds spent performing the MPI_Alltoall operation. Note: The timer used on the back end is in cycles, which could potentially be problematic on a system where the clock frequency can change. On such a system, this counter could be inaccurate since we assume a fixed clock rate."), SET_COUNTER_ARRAY(OMPI_SPC_WAIT, "The number of times MPI_Wait was called."), SET_COUNTER_ARRAY(OMPI_SPC_WAITALL, "The number of times MPI_Waitall was called."), SET_COUNTER_ARRAY(OMPI_SPC_WAITANY, "The number of times MPI_Waitany was called."), @@ -331,6 +332,7 @@ void ompi_spc_init(void) /* If this is a timer event, set the corresponding timer_event entry */ SET_SPC_BIT(ompi_spc_timer_event, OMPI_SPC_MATCH_TIME); + SET_SPC_BIT(ompi_spc_timer_event, OMPI_SPC_TIME_ALLTOALL); opal_argv_free(arg_strings); } diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h index 5d040511c34..5ba88acc6cc 100644 --- a/ompi/runtime/ompi_spc.h +++ b/ompi/runtime/ompi_spc.h @@ -138,6 +138,7 @@ typedef enum ompi_spc_counters { OMPI_SPC_TESTALL, OMPI_SPC_TESTANY, OMPI_SPC_TESTSOME, + OMPI_SPC_TIME_ALLTOALL, OMPI_SPC_WAIT, OMPI_SPC_WAITALL, OMPI_SPC_WAITANY, From 8951c7f62c6f441c03e60ca865c3931b2af08e29 Mon Sep 17 00:00:00 2001 From: Thomas Naughton Date: Tue, 27 Oct 2020 15:38:20 -0400 Subject: [PATCH 2/3] add ompi_spc_value_diff() helper SPC function Signed-off-by: Thomas Naughton --- ompi/runtime/ompi_spc.c | 68 +++++++++++++++++++++++++++++++++++++++++ ompi/runtime/ompi_spc.h | 2 ++ 2 files changed, 70 insertions(+) diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index d4e70165bac..02fdb2fd32e 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -405,6 +405,74 @@ static void ompi_spc_dump(void) ompi_spc_comm->c_coll->coll_barrier(ompi_spc_comm, ompi_spc_comm->c_coll->coll_barrier_module); } + +/* + * Congestion - helper function for checking diff w/ SPCs + * + * Given a specific SPC name and prior value, we + * get the new value and return the difference between + * the prior and new values (diff = new - prev). + * If do not care about the diff you can pass NULL for spc_diff, + * and will simply get the new_value. + * + * Note: Return the value as-is (do not convert cycles, etc.) + * + * On success, return MPI_SUCCESS, otherwise return -1. + */ +int ompi_spc_value_diff(char *spc_name, + long long spc_prev_value, + long long *spc_new_value, + long long *spc_diff) +{ + int i; + long long value = -1; + int found = 0; + + if (NULL == ompi_spc_events) { + //fprintf(stderr, " #-- DBG: WARN: SPC system not available\n"); + return -1; + } + + /* Find the index of given SPC. */ + for(i = 0; i < OMPI_SPC_NUM_COUNTERS; i++) { + if( 0 == strcmp(ompi_spc_events[i].name, spc_name) ) { + + //OPAL_THREAD_LOCK(&_spc_mutex); + + /* + * TJN: Not using SPC_CYCLES_TO_USECS() macro b/c it + * appears to have side-effects. :-/ + */ + if( IS_SPC_BIT_SET(ompi_spc_timer_event, i) ) { + value = (long long)ompi_spc_events[i].value; + //fprintf(stderr, " #-- DBG: %s (tmp) value = %d sys_clock_freq_mhz = %d\n", spc_name, value, sys_clock_freq_mhz); + value = value / sys_clock_freq_mhz; + } + + //fprintf(stderr, " #-- DBG: %s value = %d\n", spc_name, value); + + //OPAL_THREAD_UNLOCK(&_spc_mutex); + + found = 1; + break; + } + } + + if (found != 1) { + printf("Error: Failed to find SPC counter '%s'\n", spc_name); + return -1; + } + + *spc_new_value = value; + + if (NULL != spc_diff) { + *spc_diff = value - spc_prev_value; + } + + return MPI_SUCCESS; +} + + /* Frees any dynamically alocated OMPI SPC data structures */ void ompi_spc_fini(void) { diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h index 5ba88acc6cc..66b304fc7a7 100644 --- a/ompi/runtime/ompi_spc.h +++ b/ompi/runtime/ompi_spc.h @@ -188,6 +188,8 @@ void ompi_spc_user_or_mpi(int tag, ompi_spc_value_t value, unsigned int user_enu void ompi_spc_cycles_to_usecs(ompi_spc_value_t *cycles); void ompi_spc_update_watermark(unsigned int watermark_enum, unsigned int value_enum); +int ompi_spc_value_diff(char *spc_name, long long spc_prev_value, long long *spc_new_value, long long *spc_diff); + /* Macros for using the SPC utility functions throughout the codebase. * If SPC_ENABLE is not 1, the macros become no-ops. */ From 6c0a31e70d5e8ad88ad6ea4c93508eb111f79bfa Mon Sep 17 00:00:00 2001 From: Thomas Naughton Date: Wed, 30 Jun 2021 20:13:33 -0400 Subject: [PATCH 3/3] add spc for Alltoallv (OMPI_SPC_TIME_ALLTOALLV) Signed-off-by: Thomas Naughton --- ompi/mpi/c/alltoallv.c | 5 +++++ ompi/runtime/ompi_spc.c | 2 ++ ompi/runtime/ompi_spc.h | 1 + 3 files changed, 8 insertions(+) diff --git a/ompi/mpi/c/alltoallv.c b/ompi/mpi/c/alltoallv.c index 70719b63883..2a4646a3d3d 100644 --- a/ompi/mpi/c/alltoallv.c +++ b/ompi/mpi/c/alltoallv.c @@ -49,6 +49,7 @@ int MPI_Alltoallv(const void *sendbuf, const int sendcounts[], MPI_Datatype recvtype, MPI_Comm comm) { int i, size, err; + opal_timer_t timer = 0; /* SPC */ SPC_RECORD(OMPI_SPC_ALLTOALLV, 1); @@ -125,10 +126,14 @@ int MPI_Alltoallv(const void *sendbuf, const int sendcounts[], OPAL_CR_ENTER_LIBRARY(); + SPC_TIMER_START(OMPI_SPC_TIME_ALLTOALLV, &timer); + /* Invoke the coll component to perform the back-end operation */ err = comm->c_coll->coll_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, comm->c_coll->coll_alltoallv_module); + SPC_TIMER_STOP(OMPI_SPC_TIME_ALLTOALLV, &timer); + OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index 02fdb2fd32e..2ca66a9b34a 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -121,6 +121,7 @@ static ompi_spc_event_t ompi_spc_events_names[OMPI_SPC_NUM_COUNTERS] = { SET_COUNTER_ARRAY(OMPI_SPC_TESTANY, "The number of times MPI_Testany was called."), SET_COUNTER_ARRAY(OMPI_SPC_TESTSOME, "The number of times MPI_Testsome was called."), SET_COUNTER_ARRAY(OMPI_SPC_TIME_ALLTOALL, "The number microseconds spent performing the MPI_Alltoall operation. Note: The timer used on the back end is in cycles, which could potentially be problematic on a system where the clock frequency can change. On such a system, this counter could be inaccurate since we assume a fixed clock rate."), + SET_COUNTER_ARRAY(OMPI_SPC_TIME_ALLTOALLV, "The number microseconds spent performing the MPI_Alltoallv operation. Note: The timer used on the back end is in cycles, which could potentially be problematic on a system where the clock frequency can change. On such a system, this counter could be inaccurate since we assume a fixed clock rate."), SET_COUNTER_ARRAY(OMPI_SPC_WAIT, "The number of times MPI_Wait was called."), SET_COUNTER_ARRAY(OMPI_SPC_WAITALL, "The number of times MPI_Waitall was called."), SET_COUNTER_ARRAY(OMPI_SPC_WAITANY, "The number of times MPI_Waitany was called."), @@ -333,6 +334,7 @@ void ompi_spc_init(void) /* If this is a timer event, set the corresponding timer_event entry */ SET_SPC_BIT(ompi_spc_timer_event, OMPI_SPC_MATCH_TIME); SET_SPC_BIT(ompi_spc_timer_event, OMPI_SPC_TIME_ALLTOALL); + SET_SPC_BIT(ompi_spc_timer_event, OMPI_SPC_TIME_ALLTOALLV); opal_argv_free(arg_strings); } diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h index 66b304fc7a7..69814f233e7 100644 --- a/ompi/runtime/ompi_spc.h +++ b/ompi/runtime/ompi_spc.h @@ -139,6 +139,7 @@ typedef enum ompi_spc_counters { OMPI_SPC_TESTANY, OMPI_SPC_TESTSOME, OMPI_SPC_TIME_ALLTOALL, + OMPI_SPC_TIME_ALLTOALLV, OMPI_SPC_WAIT, OMPI_SPC_WAITALL, OMPI_SPC_WAITANY,