From fd77bb2d0a584eba81e0abda3088c0d4082145ea Mon Sep 17 00:00:00 2001 From: Thomas Naughton Date: Tue, 2 Oct 2018 11:10:57 -0400 Subject: [PATCH 1/4] WIP: carry Tau memory_instrumentation bits to master See https://github.com/ParaToolsInc/ompi.git --- ompi/communicator/comm_init.c | 2 ++ ompi/group/group_init.c | 17 ++++++++++++++++- ompi/mca/pml/ob1/pml_ob1_comm.c | 4 ++++ ompi/mca/pml/yalla/pml_yalla.c | 4 ++++ ompi/win/win.c | 2 ++ opal/class/opal_object.c | 24 ++++++++++++++++++++++++ opal/class/opal_object.h | 11 +++++++++++ oshmem/proc/proc.c | 7 +++++++ 8 files changed, 70 insertions(+), 1 deletion(-) diff --git a/ompi/communicator/comm_init.c b/ompi/communicator/comm_init.c index 824c0680019..9c1732c58d6 100644 --- a/ompi/communicator/comm_init.c +++ b/ompi/communicator/comm_init.c @@ -242,6 +242,7 @@ ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size ) /* create new communicator element */ new_comm = OBJ_NEW(ompi_communicator_t); + Tau_start_class_allocation(new_comm->c_base.obj_class->cls_name, 0, 0); new_comm->super.s_info = NULL; new_comm->c_local_group = ompi_group_allocate ( local_size ); if ( 0 < remote_size ) { @@ -258,6 +259,7 @@ ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size ) /* fill in the inscribing hyper-cube dimensions */ new_comm->c_cube_dim = opal_cube_dim(local_size); + Tau_stop_class_allocation(new_comm->c_base.obj_class->cls_name, 0); return new_comm; } diff --git a/ompi/group/group_init.c b/ompi/group/group_init.c index 674e4749eda..dd10141e5f0 100644 --- a/ompi/group/group_init.c +++ b/ompi/group/group_init.c @@ -57,7 +57,11 @@ ompi_predefined_group_t *ompi_mpi_group_null_addr = &ompi_mpi_group_null; ompi_group_t *ompi_group_allocate(int group_size) { /* local variables */ + Tau_start_class_allocation("ompi_group_t", 0, 0); ompi_proc_t **procs = calloc (group_size, sizeof (ompi_proc_t *)); + Tau_start_class_allocation("ompi_proc_t **", group_size * sizeof(ompi_proc_t *), 0); + Tau_stop_class_allocation("ompi_proc_t **", 1); + Tau_stop_class_allocation("ompi_group_t", 0); ompi_group_t *new_group; if (NULL == procs) { @@ -81,6 +85,7 @@ ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_ /* create new group group element */ new_group = OBJ_NEW(ompi_group_t); + Tau_start_class_allocation(new_group->super.obj_class->cls_name, 0, 0); if (NULL == new_group) { return NULL; @@ -105,7 +110,7 @@ ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_ OMPI_GROUP_SET_DENSE(new_group); ompi_group_increment_proc_count (new_group); - + Tau_stop_class_allocation(new_group->super.obj_class->cls_name, 0); return new_group; } @@ -126,11 +131,14 @@ ompi_group_t *ompi_group_allocate_sporadic(int group_size) new_group = NULL; goto error_exit; } + Tau_start_class_allocation(new_group->super.obj_class->cls_name, 0, 0); /* allocate array of (grp_sporadic_list )'s */ if (0 < group_size) { new_group->sparse_data.grp_sporadic.grp_sporadic_list = (struct ompi_group_sporadic_list_t *)malloc (sizeof(struct ompi_group_sporadic_list_t ) * group_size); + Tau_start_class_allocation("ompi_group_sporadic_list_t *", sizeof(struct ompi_group_sporadic_list_t ) * group_size, 0); + Tau_stop_class_allocation("ompi_group_sporadic_list_t *", 1); /* non-empty group */ if ( NULL == new_group->sparse_data.grp_sporadic.grp_sporadic_list) { @@ -151,6 +159,7 @@ ompi_group_t *ompi_group_allocate_sporadic(int group_size) OMPI_GROUP_SET_SPORADIC(new_group); error_exit: + Tau_stop_class_allocation(new_group->super.obj_class->cls_name, 0); return new_group; } @@ -168,6 +177,7 @@ ompi_group_t *ompi_group_allocate_strided(void) new_group = NULL; goto error_exit; } + Tau_start_class_allocation(new_group->super.obj_class->cls_name, 0, 0); /* initialize our rank to MPI_UNDEFINED */ new_group->grp_my_rank = MPI_UNDEFINED; new_group->grp_proc_pointers = NULL; @@ -177,6 +187,7 @@ ompi_group_t *ompi_group_allocate_strided(void) new_group->sparse_data.grp_strided.grp_strided_last_element = -1; error_exit: /* return */ + Tau_stop_class_allocation(new_group->super.obj_class->cls_name, 0); return new_group; } ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size) @@ -196,8 +207,11 @@ ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size) goto error_exit; } /* allocate the unsigned char list */ + Tau_start_class_allocation(new_group->super.obj_class->cls_name, 0, 0); new_group->sparse_data.grp_bitmap.grp_bitmap_array = (unsigned char *)malloc (sizeof(unsigned char) * ompi_group_div_ceil(orig_group_size,BSIZE)); + Tau_start_class_allocation("unsigned char *", sizeof(struct ompi_group_sporadic_list_t ) * group_size, 0); + Tau_stop_class_allocation("unsigned char *", 1); new_group->sparse_data.grp_bitmap.grp_bitmap_array_len = ompi_group_div_ceil(orig_group_size,BSIZE); @@ -211,6 +225,7 @@ ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size) error_exit: /* return */ + Tau_stop_class_allocation(new_group->super.obj_class->cls_name, 0); return new_group; } diff --git a/ompi/mca/pml/ob1/pml_ob1_comm.c b/ompi/mca/pml/ob1/pml_ob1_comm.c index 9eeedd6b05d..51d2328eb2a 100644 --- a/ompi/mca/pml/ob1/pml_ob1_comm.c +++ b/ompi/mca/pml/ob1/pml_ob1_comm.c @@ -108,7 +108,11 @@ OBJ_CLASS_INSTANCE( int mca_pml_ob1_comm_init_size (mca_pml_ob1_comm_t* comm, size_t size) { /* send message sequence-number support - sender side */ + Tau_start_class_allocation(comm->super.obj_class->cls_name, 0, 0); comm->procs = (mca_pml_ob1_comm_proc_t **) calloc(size, sizeof (mca_pml_ob1_comm_proc_t *)); + Tau_start_class_allocation("mca_pml_ob1_comm_proc_t", size * sizeof(mca_pml_ob1_comm_proc_t *), 0); + Tau_stop_class_allocation("mca_pml_ob1_comm_proc_t", 1); + Tau_stop_class_allocation(comm->super.obj_class->cls_name, 0); if(NULL == comm->procs) { return OMPI_ERR_OUT_OF_RESOURCE; } diff --git a/ompi/mca/pml/yalla/pml_yalla.c b/ompi/mca/pml/yalla/pml_yalla.c index 03bb65d420d..c0724c0810b 100644 --- a/ompi/mca/pml/yalla/pml_yalla.c +++ b/ompi/mca/pml/yalla/pml_yalla.c @@ -256,7 +256,11 @@ int mca_pml_yalla_add_procs(struct ompi_proc_t **procs, size_t nprocs) return OMPI_ERROR; } + Tau_start_class_allocation(procs[i]->super.super.super.obj_class->cls_name, 0, 0); + Tau_start_class_allocation("mxm_conn_h", sizeof(mxm_conn_h), 0); procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML] = conn; + Tau_stop_class_allocation("mxm_conn_h", 1); + Tau_stop_class_allocation(procs[i]->super.super.super.obj_class->cls_name, 0); } return OMPI_SUCCESS; diff --git a/ompi/win/win.c b/ompi/win/win.c index b4bc150d893..160b7147bd4 100644 --- a/ompi/win/win.c +++ b/ompi/win/win.c @@ -148,6 +148,7 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int /* create the object */ win = OBJ_NEW(ompi_win_t); + Tau_start_class_allocation(win->w_base.obj_class->cls_name, 0, 0); if (NULL == win) { return OMPI_ERR_OUT_OF_RESOURCE; } @@ -187,6 +188,7 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int } *win_out = win; + Tau_stop_class_allocation(win->w_base.obj_class->cls_name, 0); return OMPI_SUCCESS; } diff --git a/opal/class/opal_object.c b/opal/class/opal_object.c index 64fef5712fc..108b310716a 100644 --- a/opal/class/opal_object.c +++ b/opal/class/opal_object.c @@ -33,6 +33,30 @@ #include "opal/class/opal_object.h" #include "opal/constants.h" +void __attribute__((weak)) Tau_track_class_allocation(const char * name, size_t size) { + +} + +void __attribute__((weak)) Tau_track_class_deallocation(const char * name, size_t size) { + +} + +void __attribute__((weak)) Tau_start_class_allocation(const char * name, size_t size, int include_in_parent) { + +} + +void __attribute__((weak)) Tau_stop_class_allocation(const char * name, int record) { + +} + +void __attribute__((weak)) Tau_start_class_deallocation(const char * name, size_t size, int include_in_parent) { + +} + +void __attribute__((weak)) Tau_stop_class_deallocation(const char * name, int record) { + +} + /* * Instantiation of class descriptor for the base class. This is * special, since be mark it as already initialized, with no parent diff --git a/opal/class/opal_object.h b/opal/class/opal_object.h index 4d3380f0da0..789f7c2e568 100644 --- a/opal/class/opal_object.h +++ b/opal/class/opal_object.h @@ -132,6 +132,13 @@ BEGIN_C_DECLS #define OPAL_OBJ_MAGIC_ID ((0xdeafbeedULL << 32) + 0xdeafbeedULL) #endif +void __attribute__((weak)) Tau_track_class_allocation(const char * name, size_t size); +void __attribute__((weak)) Tau_track_class_deallocation(const char * name, size_t size); +void __attribute__((weak)) Tau_start_class_allocation(const char * name, size_t size, int include_in_parent); +void __attribute__((weak)) Tau_stop_class_allocation(const char * name, int record); +void __attribute__((weak)) Tau_start_class_deallocation(const char * name, size_t size, int include_in_parent); +void __attribute__((weak)) Tau_stop_class_deallocation(const char * name, int record); + /* typedefs ***********************************************************/ typedef struct opal_object_t opal_object_t; @@ -457,6 +464,7 @@ static inline void opal_obj_run_destructors(opal_object_t * object) assert(NULL != object->obj_class); + Tau_track_class_deallocation(object->obj_class->cls_name, object->obj_class->cls_sizeof); cls_destruct = object->obj_class->cls_destruct_array; while( NULL != *cls_destruct ) { (*cls_destruct)(object); @@ -480,6 +488,8 @@ static inline opal_object_t *opal_obj_new(opal_class_t * cls) opal_object_t *object; assert(cls->cls_sizeof >= sizeof(opal_object_t)); + Tau_start_class_allocation(cls->cls_name, cls->cls_sizeof, 0); + #if OPAL_WANT_MEMCHECKER object = (opal_object_t *) calloc(1, cls->cls_sizeof); #else @@ -493,6 +503,7 @@ static inline opal_object_t *opal_obj_new(opal_class_t * cls) object->obj_reference_count = 1; opal_obj_run_constructors(object); } + Tau_stop_class_allocation(cls->cls_name, 1); return object; } diff --git a/oshmem/proc/proc.c b/oshmem/proc/proc.c index 23cbbc685c5..6bd6a913de9 100644 --- a/oshmem/proc/proc.c +++ b/oshmem/proc/proc.c @@ -161,6 +161,8 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz } group = OBJ_NEW(oshmem_group_t); + Tau_start_class_allocation(group->base.obj_class->cls_name, 0, 0); + if (NULL == group) { return NULL; } @@ -172,7 +174,10 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz /* allocate an array */ proc_array = (ompi_proc_t**) malloc(pe_size * sizeof(ompi_proc_t*)); + Tau_start_class_allocation("ompi_proc_t **", pe_size * sizeof(ompi_proc_t*), 0); + Tau_stop_class_allocation("ompi_proc_t **", 1); if (NULL == proc_array) { + Tau_stop_class_allocation(group->base.obj_class->cls_name, 0); OBJ_RELEASE(group); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return NULL ; @@ -224,6 +229,7 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz if (OSHMEM_SUCCESS != mca_scoll_base_select(group)) { opal_output(0, "Error: No collective modules are available: group is not created, returning NULL"); + Tau_stop_class_allocation(group->base.obj_class->cls_name, 0); oshmem_proc_group_destroy_internal(group, 0); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return NULL; @@ -237,6 +243,7 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz } OPAL_THREAD_UNLOCK(&oshmem_proc_lock); + Tau_stop_class_allocation(group->base.obj_class->cls_name, 0); return group; } From f070e1c3169439ed7364956fbfd34347641c5d83 Mon Sep 17 00:00:00 2001 From: Thomas Naughton Date: Fri, 4 Jan 2019 17:50:49 -0500 Subject: [PATCH 2/4] update to new opal_infosubscriber_t structure There was a restructuring of the comm/win/group objects that shifted the fields down for getting at the class name. This appears to relate to opal_infosubscriber_t changes from 50aa143. NOTE: The opal_infosubscriber_t changes were not applied to the oshmem_group_t structure, so no need for changes there. --- ompi/communicator/comm_init.c | 4 ++-- ompi/win/win.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ompi/communicator/comm_init.c b/ompi/communicator/comm_init.c index 9c1732c58d6..59b559ad450 100644 --- a/ompi/communicator/comm_init.c +++ b/ompi/communicator/comm_init.c @@ -242,7 +242,7 @@ ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size ) /* create new communicator element */ new_comm = OBJ_NEW(ompi_communicator_t); - Tau_start_class_allocation(new_comm->c_base.obj_class->cls_name, 0, 0); + Tau_start_class_allocation(new_comm->super.s_base.obj_class->cls_name, 0, 0); new_comm->super.s_info = NULL; new_comm->c_local_group = ompi_group_allocate ( local_size ); if ( 0 < remote_size ) { @@ -259,7 +259,7 @@ ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size ) /* fill in the inscribing hyper-cube dimensions */ new_comm->c_cube_dim = opal_cube_dim(local_size); - Tau_stop_class_allocation(new_comm->c_base.obj_class->cls_name, 0); + Tau_stop_class_allocation(new_comm->super.s_base.obj_class->cls_name, 0); return new_comm; } diff --git a/ompi/win/win.c b/ompi/win/win.c index 160b7147bd4..6f36c68ec56 100644 --- a/ompi/win/win.c +++ b/ompi/win/win.c @@ -148,7 +148,7 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int /* create the object */ win = OBJ_NEW(ompi_win_t); - Tau_start_class_allocation(win->w_base.obj_class->cls_name, 0, 0); + Tau_start_class_allocation(win->super.s_base.obj_class->cls_name, 0, 0); if (NULL == win) { return OMPI_ERR_OUT_OF_RESOURCE; } @@ -188,7 +188,7 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int } *win_out = win; - Tau_stop_class_allocation(win->w_base.obj_class->cls_name, 0); + Tau_stop_class_allocation(win->super.s_base.obj_class->cls_name, 0); return OMPI_SUCCESS; } From a4f9d1c0c3ce808ac62484a10285b5520ee607e0 Mon Sep 17 00:00:00 2001 From: Thomas Naughton Date: Tue, 15 Jan 2019 13:55:42 -0500 Subject: [PATCH 3/4] move profile funcs/stubs out to memprof file Guard these using existing OPAL_ENABLE_MEM_PROFILE option that is enabled via '--enable-mem-profile' configury. --- config/opal_configure_options.m4 | 1 + opal/class/opal_object.c | 24 +---------------- opal/class/opal_object.h | 7 +---- opal/util/Makefile.am | 7 +++++ opal/util/memprof.c | 45 ++++++++++++++++++++++++++++++++ opal/util/memprof.h | 39 +++++++++++++++++++++++++++ 6 files changed, 94 insertions(+), 29 deletions(-) create mode 100644 opal/util/memprof.c create mode 100644 opal/util/memprof.h diff --git a/config/opal_configure_options.m4 b/config/opal_configure_options.m4 index 43fcaf3469d..f02a6d0eb4f 100644 --- a/config/opal_configure_options.m4 +++ b/config/opal_configure_options.m4 @@ -120,6 +120,7 @@ else fi AC_DEFINE_UNQUOTED(OPAL_ENABLE_MEM_PROFILE, $WANT_MEM_PROFILE, [Whether we want the memory profiling or not]) +AM_CONDITIONAL([OPAL_ENABLE_MEM_PROFILE], [test "$WANT_MEM_PROFILE" = "1"]) # # Developer picky compiler options diff --git a/opal/class/opal_object.c b/opal/class/opal_object.c index 108b310716a..37c03884ccf 100644 --- a/opal/class/opal_object.c +++ b/opal/class/opal_object.c @@ -32,30 +32,8 @@ #include "opal/sys/atomic.h" #include "opal/class/opal_object.h" #include "opal/constants.h" +#include "opal/util/memprof.h" -void __attribute__((weak)) Tau_track_class_allocation(const char * name, size_t size) { - -} - -void __attribute__((weak)) Tau_track_class_deallocation(const char * name, size_t size) { - -} - -void __attribute__((weak)) Tau_start_class_allocation(const char * name, size_t size, int include_in_parent) { - -} - -void __attribute__((weak)) Tau_stop_class_allocation(const char * name, int record) { - -} - -void __attribute__((weak)) Tau_start_class_deallocation(const char * name, size_t size, int include_in_parent) { - -} - -void __attribute__((weak)) Tau_stop_class_deallocation(const char * name, int record) { - -} /* * Instantiation of class descriptor for the base class. This is diff --git a/opal/class/opal_object.h b/opal/class/opal_object.h index 789f7c2e568..79c8c6b5db6 100644 --- a/opal/class/opal_object.h +++ b/opal/class/opal_object.h @@ -124,6 +124,7 @@ #include #include "opal/threads/thread_usage.h" +#include "opal/util/memprof.h" BEGIN_C_DECLS @@ -132,12 +133,6 @@ BEGIN_C_DECLS #define OPAL_OBJ_MAGIC_ID ((0xdeafbeedULL << 32) + 0xdeafbeedULL) #endif -void __attribute__((weak)) Tau_track_class_allocation(const char * name, size_t size); -void __attribute__((weak)) Tau_track_class_deallocation(const char * name, size_t size); -void __attribute__((weak)) Tau_start_class_allocation(const char * name, size_t size, int include_in_parent); -void __attribute__((weak)) Tau_stop_class_allocation(const char * name, int record); -void __attribute__((weak)) Tau_start_class_deallocation(const char * name, size_t size, int include_in_parent); -void __attribute__((weak)) Tau_stop_class_deallocation(const char * name, int record); /* typedefs ***********************************************************/ diff --git a/opal/util/Makefile.am b/opal/util/Makefile.am index 521acc8bf9f..9e8ec4f402c 100644 --- a/opal/util/Makefile.am +++ b/opal/util/Makefile.am @@ -17,6 +17,8 @@ # Copyright (c) 2016 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2016-2017 IBM Corporation. All rights reserved. +# Copyright (c) 2019 UT-Battelle, LLC. All rights reserved. +# # $COPYRIGHT$ # # Additional copyrights may follow @@ -55,6 +57,7 @@ headers = \ if.h \ keyval_parse.h \ malloc.h \ + memprof.h \ net.h \ numtostr.h \ opal_environ.h \ @@ -119,6 +122,10 @@ if OPAL_COMPILE_TIMING libopalutil_la_SOURCES += timings.c endif +if OPAL_ENABLE_MEM_PROFILE +libopalutil_la_SOURCES += memprof.c +endif + libopalutil_la_LIBADD = \ keyval/libopalutilkeyval.la libopalutil_la_DEPENDENCIES = \ diff --git a/opal/util/memprof.c b/opal/util/memprof.c new file mode 100644 index 00000000000..8afbff0f574 --- /dev/null +++ b/opal/util/memprof.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2019 UT-Battelle, LLC. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" +#include "opal/util/memprof.h" + + +void __attribute__((weak)) Tau_track_class_allocation(const char * name, + size_t size) { + +} + +void __attribute__((weak)) Tau_track_class_deallocation(const char * name, + size_t size) { + +} + +void __attribute__((weak)) Tau_start_class_allocation(const char * name, + size_t size, + int include_in_parent) { + +} + +void __attribute__((weak)) Tau_stop_class_allocation(const char * name, + int record) { + +} + +void __attribute__((weak)) Tau_start_class_deallocation(const char * name, + size_t size, + int include_in_parent) { + +} + +void __attribute__((weak)) Tau_stop_class_deallocation(const char * name, + int record) { + +} diff --git a/opal/util/memprof.h b/opal/util/memprof.h new file mode 100644 index 00000000000..ddc8fafa46e --- /dev/null +++ b/opal/util/memprof.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2018 UT-Battelle, LLC. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef OPAL_MEMPROF_H +#define OPAL_MEMPROF_H + +BEGIN_C_DECLS + +#if OPAL_ENABLE_MEM_PROFILE + + void __attribute__((weak)) Tau_track_class_allocation(const char * name, size_t size); + void __attribute__((weak)) Tau_track_class_deallocation(const char * name, size_t size); + void __attribute__((weak)) Tau_start_class_allocation(const char * name, size_t size, int include_in_parent); + void __attribute__((weak)) Tau_stop_class_allocation(const char * name, int record); + void __attribute__((weak)) Tau_start_class_deallocation(const char * name, size_t size, int include_in_parent); + void __attribute__((weak)) Tau_stop_class_deallocation(const char * name, int record); + +#endif /* OPAL_ENABLE_MEM_PROFILE */ + +END_C_DECLS + +#endif /* OPAL_MEMPROF_H */ From 646c904e60f2278c8a481c1fbba2d2915d5e4fb4 Mon Sep 17 00:00:00 2001 From: Thomas Naughton Date: Tue, 15 Jan 2019 21:30:56 -0500 Subject: [PATCH 4/4] use macros in code and add comments --- ompi/communicator/comm_init.c | 5 ++- ompi/group/group_init.c | 34 ++++++++-------- ompi/mca/pml/ob1/pml_ob1_comm.c | 9 +++-- ompi/mca/pml/yalla/pml_yalla.c | 9 +++-- ompi/win/win.c | 5 ++- opal/class/opal_object.h | 6 +-- opal/util/memprof.c | 2 +- opal/util/memprof.h | 71 ++++++++++++++++++++++++++++----- oshmem/proc/proc.c | 13 +++--- 9 files changed, 106 insertions(+), 48 deletions(-) diff --git a/ompi/communicator/comm_init.c b/ompi/communicator/comm_init.c index 59b559ad450..880e752226a 100644 --- a/ompi/communicator/comm_init.c +++ b/ompi/communicator/comm_init.c @@ -36,6 +36,7 @@ #include "opal/util/bit_ops.h" #include "opal/util/info_subscriber.h" #include "opal/util/string_copy.h" +#include "opal/util/memprof.h" #include "opal/mca/pmix/pmix.h" #include "ompi/constants.h" #include "ompi/mca/pml/pml.h" @@ -242,7 +243,7 @@ ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size ) /* create new communicator element */ new_comm = OBJ_NEW(ompi_communicator_t); - Tau_start_class_allocation(new_comm->super.s_base.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC(new_comm->super.s_base.obj_class->cls_name, 0, 0); new_comm->super.s_info = NULL; new_comm->c_local_group = ompi_group_allocate ( local_size ); if ( 0 < remote_size ) { @@ -259,7 +260,7 @@ ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size ) /* fill in the inscribing hyper-cube dimensions */ new_comm->c_cube_dim = opal_cube_dim(local_size); - Tau_stop_class_allocation(new_comm->super.s_base.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(new_comm->super.s_base.obj_class->cls_name, 0); return new_comm; } diff --git a/ompi/group/group_init.c b/ompi/group/group_init.c index dd10141e5f0..7b4175f9829 100644 --- a/ompi/group/group_init.c +++ b/ompi/group/group_init.c @@ -28,6 +28,8 @@ #include "ompi/constants.h" #include "mpi.h" +#include "opal/util/memprof.h" + /* define class information */ static void ompi_group_construct(ompi_group_t *); static void ompi_group_destruct(ompi_group_t *); @@ -57,11 +59,11 @@ ompi_predefined_group_t *ompi_mpi_group_null_addr = &ompi_mpi_group_null; ompi_group_t *ompi_group_allocate(int group_size) { /* local variables */ - Tau_start_class_allocation("ompi_group_t", 0, 0); + OPAL_MEMPROF_START_ALLOC("ompi_group_t", 0, 0); ompi_proc_t **procs = calloc (group_size, sizeof (ompi_proc_t *)); - Tau_start_class_allocation("ompi_proc_t **", group_size * sizeof(ompi_proc_t *), 0); - Tau_stop_class_allocation("ompi_proc_t **", 1); - Tau_stop_class_allocation("ompi_group_t", 0); + OPAL_MEMPROF_START_ALLOC("ompi_proc_t **", group_size * sizeof(ompi_proc_t *), 0); + OPAL_MEMPROF_STOP_ALLOC("ompi_proc_t **", 1); + OPAL_MEMPROF_STOP_ALLOC("ompi_group_t", 0); ompi_group_t *new_group; if (NULL == procs) { @@ -85,7 +87,7 @@ ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_ /* create new group group element */ new_group = OBJ_NEW(ompi_group_t); - Tau_start_class_allocation(new_group->super.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC(new_group->super.obj_class->cls_name, 0, 0); if (NULL == new_group) { return NULL; @@ -110,7 +112,7 @@ ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_ OMPI_GROUP_SET_DENSE(new_group); ompi_group_increment_proc_count (new_group); - Tau_stop_class_allocation(new_group->super.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(new_group->super.obj_class->cls_name, 0); return new_group; } @@ -131,14 +133,14 @@ ompi_group_t *ompi_group_allocate_sporadic(int group_size) new_group = NULL; goto error_exit; } - Tau_start_class_allocation(new_group->super.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC(new_group->super.obj_class->cls_name, 0, 0); /* allocate array of (grp_sporadic_list )'s */ if (0 < group_size) { new_group->sparse_data.grp_sporadic.grp_sporadic_list = (struct ompi_group_sporadic_list_t *)malloc (sizeof(struct ompi_group_sporadic_list_t ) * group_size); - Tau_start_class_allocation("ompi_group_sporadic_list_t *", sizeof(struct ompi_group_sporadic_list_t ) * group_size, 0); - Tau_stop_class_allocation("ompi_group_sporadic_list_t *", 1); + OPAL_MEMPROF_START_ALLOC("ompi_group_sporadic_list_t *", sizeof(struct ompi_group_sporadic_list_t ) * group_size, 0); + OPAL_MEMPROF_STOP_ALLOC("ompi_group_sporadic_list_t *", 1); /* non-empty group */ if ( NULL == new_group->sparse_data.grp_sporadic.grp_sporadic_list) { @@ -159,7 +161,7 @@ ompi_group_t *ompi_group_allocate_sporadic(int group_size) OMPI_GROUP_SET_SPORADIC(new_group); error_exit: - Tau_stop_class_allocation(new_group->super.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(new_group->super.obj_class->cls_name, 0); return new_group; } @@ -177,7 +179,7 @@ ompi_group_t *ompi_group_allocate_strided(void) new_group = NULL; goto error_exit; } - Tau_start_class_allocation(new_group->super.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC(new_group->super.obj_class->cls_name, 0, 0); /* initialize our rank to MPI_UNDEFINED */ new_group->grp_my_rank = MPI_UNDEFINED; new_group->grp_proc_pointers = NULL; @@ -187,7 +189,7 @@ ompi_group_t *ompi_group_allocate_strided(void) new_group->sparse_data.grp_strided.grp_strided_last_element = -1; error_exit: /* return */ - Tau_stop_class_allocation(new_group->super.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(new_group->super.obj_class->cls_name, 0); return new_group; } ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size) @@ -207,11 +209,11 @@ ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size) goto error_exit; } /* allocate the unsigned char list */ - Tau_start_class_allocation(new_group->super.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC(new_group->super.obj_class->cls_name, 0, 0); new_group->sparse_data.grp_bitmap.grp_bitmap_array = (unsigned char *)malloc (sizeof(unsigned char) * ompi_group_div_ceil(orig_group_size,BSIZE)); - Tau_start_class_allocation("unsigned char *", sizeof(struct ompi_group_sporadic_list_t ) * group_size, 0); - Tau_stop_class_allocation("unsigned char *", 1); + OPAL_MEMPROF_START_ALLOC("unsigned char *", sizeof(struct ompi_group_sporadic_list_t ) * group_size, 0); + OPAL_MEMPROF_STOP_ALLOC("unsigned char *", 1); new_group->sparse_data.grp_bitmap.grp_bitmap_array_len = ompi_group_div_ceil(orig_group_size,BSIZE); @@ -225,7 +227,7 @@ ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size) error_exit: /* return */ - Tau_stop_class_allocation(new_group->super.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(new_group->super.obj_class->cls_name, 0); return new_group; } diff --git a/ompi/mca/pml/ob1/pml_ob1_comm.c b/ompi/mca/pml/ob1/pml_ob1_comm.c index 51d2328eb2a..09e620be730 100644 --- a/ompi/mca/pml/ob1/pml_ob1_comm.c +++ b/ompi/mca/pml/ob1/pml_ob1_comm.c @@ -20,6 +20,7 @@ */ #include "ompi_config.h" +#include "opal/util/memprof.h" #include #include "pml_ob1.h" @@ -108,11 +109,11 @@ OBJ_CLASS_INSTANCE( int mca_pml_ob1_comm_init_size (mca_pml_ob1_comm_t* comm, size_t size) { /* send message sequence-number support - sender side */ - Tau_start_class_allocation(comm->super.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC(comm->super.obj_class->cls_name, 0, 0); comm->procs = (mca_pml_ob1_comm_proc_t **) calloc(size, sizeof (mca_pml_ob1_comm_proc_t *)); - Tau_start_class_allocation("mca_pml_ob1_comm_proc_t", size * sizeof(mca_pml_ob1_comm_proc_t *), 0); - Tau_stop_class_allocation("mca_pml_ob1_comm_proc_t", 1); - Tau_stop_class_allocation(comm->super.obj_class->cls_name, 0); + OPAL_MEMPROF_START_ALLOC("mca_pml_ob1_comm_proc_t", size * sizeof(mca_pml_ob1_comm_proc_t *), 0); + OPAL_MEMPROF_STOP_ALLOC("mca_pml_ob1_comm_proc_t", 1); + OPAL_MEMPROF_STOP_ALLOC(comm->super.obj_class->cls_name, 0); if(NULL == comm->procs) { return OMPI_ERR_OUT_OF_RESOURCE; } diff --git a/ompi/mca/pml/yalla/pml_yalla.c b/ompi/mca/pml/yalla/pml_yalla.c index c0724c0810b..46cb125ffae 100644 --- a/ompi/mca/pml/yalla/pml_yalla.c +++ b/ompi/mca/pml/yalla/pml_yalla.c @@ -19,6 +19,7 @@ #include "opal/runtime/opal.h" #include "opal/memoryhooks/memory.h" +#include "opal/util/memprof.h" #include "opal/mca/memory/base/base.h" #include "opal/mca/pmix/pmix.h" #include "ompi/mca/pml/base/pml_base_bsend.h" @@ -256,11 +257,11 @@ int mca_pml_yalla_add_procs(struct ompi_proc_t **procs, size_t nprocs) return OMPI_ERROR; } - Tau_start_class_allocation(procs[i]->super.super.super.obj_class->cls_name, 0, 0); - Tau_start_class_allocation("mxm_conn_h", sizeof(mxm_conn_h), 0); + OPAL_MEMPROF_START_ALLOC(procs[i]->super.super.super.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC("mxm_conn_h", sizeof(mxm_conn_h), 0); procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML] = conn; - Tau_stop_class_allocation("mxm_conn_h", 1); - Tau_stop_class_allocation(procs[i]->super.super.super.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC("mxm_conn_h", 1); + OPAL_MEMPROF_STOP_ALLOC(procs[i]->super.super.super.obj_class->cls_name, 0); } return OMPI_SUCCESS; diff --git a/ompi/win/win.c b/ompi/win/win.c index 6f36c68ec56..ffd98aded68 100644 --- a/ompi/win/win.c +++ b/ompi/win/win.c @@ -28,6 +28,7 @@ #include "opal/util/info_subscriber.h" #include "opal/util/string_copy.h" +#include "opal/util/memprof.h" #include "mpi.h" #include "ompi/win/win.h" @@ -148,7 +149,7 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int /* create the object */ win = OBJ_NEW(ompi_win_t); - Tau_start_class_allocation(win->super.s_base.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC(win->super.s_base.obj_class->cls_name, 0, 0); if (NULL == win) { return OMPI_ERR_OUT_OF_RESOURCE; } @@ -188,7 +189,7 @@ static int alloc_window(struct ompi_communicator_t *comm, opal_info_t *info, int } *win_out = win; - Tau_stop_class_allocation(win->super.s_base.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(win->super.s_base.obj_class->cls_name, 0); return OMPI_SUCCESS; } diff --git a/opal/class/opal_object.h b/opal/class/opal_object.h index 79c8c6b5db6..0e57b66c204 100644 --- a/opal/class/opal_object.h +++ b/opal/class/opal_object.h @@ -459,7 +459,7 @@ static inline void opal_obj_run_destructors(opal_object_t * object) assert(NULL != object->obj_class); - Tau_track_class_deallocation(object->obj_class->cls_name, object->obj_class->cls_sizeof); + OPAL_MEMPROF_TRACK_DEALLOC(object->obj_class->cls_name, object->obj_class->cls_sizeof); cls_destruct = object->obj_class->cls_destruct_array; while( NULL != *cls_destruct ) { (*cls_destruct)(object); @@ -483,7 +483,7 @@ static inline opal_object_t *opal_obj_new(opal_class_t * cls) opal_object_t *object; assert(cls->cls_sizeof >= sizeof(opal_object_t)); - Tau_start_class_allocation(cls->cls_name, cls->cls_sizeof, 0); + OPAL_MEMPROF_START_ALLOC(cls->cls_name, cls->cls_sizeof, 0); #if OPAL_WANT_MEMCHECKER object = (opal_object_t *) calloc(1, cls->cls_sizeof); @@ -498,7 +498,7 @@ static inline opal_object_t *opal_obj_new(opal_class_t * cls) object->obj_reference_count = 1; opal_obj_run_constructors(object); } - Tau_stop_class_allocation(cls->cls_name, 1); + OPAL_MEMPROF_STOP_ALLOC(cls->cls_name, 1); return object; } diff --git a/opal/util/memprof.c b/opal/util/memprof.c index 8afbff0f574..656e262a8fd 100644 --- a/opal/util/memprof.c +++ b/opal/util/memprof.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2018 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2019 UT-Battelle, LLC. All rights reserved. + * * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/util/memprof.h b/opal/util/memprof.h index ddc8fafa46e..dda91e29a89 100644 --- a/opal/util/memprof.h +++ b/opal/util/memprof.h @@ -1,14 +1,4 @@ /* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. * Copyright (c) 2018 UT-Battelle, LLC. All rights reserved. * * $COPYRIGHT$ @@ -25,6 +15,17 @@ BEGIN_C_DECLS #if OPAL_ENABLE_MEM_PROFILE + /* + * NOTE: Must configure OMPI w/ '--enable-mem-profile', + * otherwise the macros are no-ops. + * Tested with TAU v1.27.1 and v1.28 and PDT 3.25 + * for gathering memory profiling data about OMPI. + * + * Example usage: + * orterun -np 2 tau_exec -T mpi,pdt ring_c + * tau_mem_summarize.py . > ring-np2.csv + */ + void __attribute__((weak)) Tau_track_class_allocation(const char * name, size_t size); void __attribute__((weak)) Tau_track_class_deallocation(const char * name, size_t size); void __attribute__((weak)) Tau_start_class_allocation(const char * name, size_t size, int include_in_parent); @@ -32,6 +33,56 @@ BEGIN_C_DECLS void __attribute__((weak)) Tau_start_class_deallocation(const char * name, size_t size, int include_in_parent); void __attribute__((weak)) Tau_stop_class_deallocation(const char * name, int record); + #define OPAL_MEMPROF_TRACK_ALLOC(name, size) \ + do { \ + Tau_track_class_allocation(name, size); \ + } while(0) + + #define OPAL_MEMPROF_TRACK_DEALLOC(name, size) \ + do { \ + Tau_track_class_deallocation(name, size); \ + } while(0) + + #define OPAL_MEMPROF_START_ALLOC(name, size, include_in_parent) \ + do { \ + Tau_start_class_allocation(name, size, include_in_parent); \ + } while(0) + + #define OPAL_MEMPROF_STOP_ALLOC(name, record) \ + do { \ + Tau_stop_class_allocation(name, record); \ + } while(0) + + #define OPAL_MEMPROF_START_DEALLOC(name, size, include_in_parent) \ + do { \ + Tau_start_class_deallocation(name, size, include_in_parent); \ + } while(0) + + #define OPAL_MEMPROF_STOP_DEALLOC(name, record) \ + do { \ + Tau_stop_class_deallocation(name, record); \ + } while(0) + +#else + + #define OPAL_MEMPROF_TRACK_ALLOC(name, size) \ + do { } while(0) + + #define OPAL_MEMPROF_TRACK_DEALLOC(name, size) \ + do { } while(0) + + #define OPAL_MEMPROF_START_ALLOC(name, size, include_in_parent) \ + do { } while(0) + + #define OPAL_MEMPROF_STOP_ALLOC(name, record) \ + do { } while(0) + + #define OPAL_MEMPROF_START_DEALLOC(name, size, include_in_parent) \ + do { } while(0) + + #define OPAL_MEMPROF_STOP_DEALLOC(name, record) \ + do { } while(0) + #endif /* OPAL_ENABLE_MEM_PROFILE */ END_C_DECLS diff --git a/oshmem/proc/proc.c b/oshmem/proc/proc.c index 6bd6a913de9..56ae0de2bb2 100644 --- a/oshmem/proc/proc.c +++ b/oshmem/proc/proc.c @@ -28,6 +28,7 @@ #include "opal/dss/dss.h" #include "opal/util/arch.h" #include "opal/class/opal_list.h" +#include "opal/util/memprof.h" static opal_mutex_t oshmem_proc_lock; @@ -161,7 +162,7 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz } group = OBJ_NEW(oshmem_group_t); - Tau_start_class_allocation(group->base.obj_class->cls_name, 0, 0); + OPAL_MEMPROF_START_ALLOC(group->base.obj_class->cls_name, 0, 0); if (NULL == group) { return NULL; @@ -174,10 +175,10 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz /* allocate an array */ proc_array = (ompi_proc_t**) malloc(pe_size * sizeof(ompi_proc_t*)); - Tau_start_class_allocation("ompi_proc_t **", pe_size * sizeof(ompi_proc_t*), 0); - Tau_stop_class_allocation("ompi_proc_t **", 1); + OPAL_MEMPROF_START_ALLOC("ompi_proc_t **", pe_size * sizeof(ompi_proc_t*), 0); + OPAL_MEMPROF_STOP_ALLOC("ompi_proc_t **", 1); if (NULL == proc_array) { - Tau_stop_class_allocation(group->base.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(group->base.obj_class->cls_name, 0); OBJ_RELEASE(group); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return NULL ; @@ -229,7 +230,7 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz if (OSHMEM_SUCCESS != mca_scoll_base_select(group)) { opal_output(0, "Error: No collective modules are available: group is not created, returning NULL"); - Tau_stop_class_allocation(group->base.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(group->base.obj_class->cls_name, 0); oshmem_proc_group_destroy_internal(group, 0); OPAL_THREAD_UNLOCK(&oshmem_proc_lock); return NULL; @@ -243,7 +244,7 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_siz } OPAL_THREAD_UNLOCK(&oshmem_proc_lock); - Tau_stop_class_allocation(group->base.obj_class->cls_name, 0); + OPAL_MEMPROF_STOP_ALLOC(group->base.obj_class->cls_name, 0); return group; }