Update PSM2 to include GDR Copy feature

This code surface corresponds to PSM2 release 10.3.35.
In addition to GDR Copy, code changes include-
  - Addition of static version of libpsm2
  - Fix SLES 12.3 libpsm2-2 Provides line
  - Properly release memory mappings before closing context
  - Avoid typecasting unknown addresses to aligned data types
  - Cleanup cuda symbol loading procedure
  - Fix GPU Direct Recv threshold default
  - Fix uniformity of process mapping to HFIs

Signed-off-by: Aravind Gopalakrishnan <[email protected]>

COMMIT

@@ -1 +1 @@
-f8df7f0de7139df384ea8b94dc8567885bf76070
+81a3f06cb764af3238ac326c5f69e24393bae85f

Makefile

@@ -84,16 +84,15 @@ override OUTDIR := $(shell readlink -m $(OUTDIR))
 endif
 endif
 
-LINKER_SCRIPT_FILE := ${OUTDIR}/psm2_linker_script.map
 
 PSM2_VERNO_MAJOR := $(shell sed -n 's/^\#define.*PSM2_VERNO_MAJOR.*0x0\?\([1-9a-f]\?[0-9a-f]\+\).*/\1/p' $(top_srcdir)/psm2.h)
 PSM2_VERNO_MINOR := $(shell sed -n 's/^\#define.*PSM2_VERNO_MINOR.*0x\([0-9]\?[0-9a-f]\+\).*/\1/p' $(top_srcdir)/psm2.h)
 PSM2_LIB_MAJOR   := $(shell printf "%d" ${PSM2_VERNO_MAJOR})
 PSM2_LIB_MINOR   := $(shell printf "%d" `sed -n 's/^\#define.*PSM2_VERNO_MINOR.*\(0x[0-9a-f]\+\).*/\1/p' $(top_srcdir)/psm2.h`)
+LINKER_SCRIPT_FILE = ${OUTDIR}/psm2_linker_script.map
 SOURCES_CHKSUM_FILES = Makefile buildflags.mak $(LINKER_SCRIPT_FILE) \
 		`find . -regex '\(.*\.h\|.*\.c\)' -not -path "./test/*" -not -path "./tools/*" -not -path "_revision.c" | sort`
 SOURCES_CHKSUM_VALUE = $(shell cat ${SOURCES_CHKSUM_FILES} | sha1sum | cut -d' ' -f 1)
-
 OPA_LIB_MAJOR := 4
 OPA_LIB_MINOR := 0
 
@@ -105,8 +104,16 @@ export OPA_LIB_MAJOR
 export OPA_LIB_MINOR
 export CCARCH ?= gcc
 export FCARCH ?= gfortran
+export AR ?= ar
 
 include $(top_srcdir)/buildflags.mak
+# We need to unexport these environs as during mock testing and normal calls,
+# if they are exported then during each submake they will be evaulated again.
+# This is costly and the LINKER_SCRIPT_FILE doesn't exist until after its
+# target rule runs.
+unexport SOURCES_CHKSUM_FILES
+unexport SOURCES_CHKSUM_VALUE
+unexport LINKER_SCRIPT_FILE
 INCLUDES += -I$(top_srcdir)
 
 ifneq (x86_64,$(arch))
@@ -272,18 +279,20 @@ all: outdir symlinks
 	@if [ ! -e $(HISTORY) ] || [ -z "`grep -E '^$(OUTDIR)$$' $(HISTORY)`" ]; then \
 		echo $(OUTDIR) >> $(HISTORY); \
 	fi
+	# Our buildflags.mak exports all variables, all are propogated to submakes.
 	@for subdir in $(SUBDIRS); do \
 		mkdir -p $(OUTDIR)/$$subdir; \
-		$(MAKE) -j $(nthreads) -C $$subdir OUTDIR=$(OUTDIR)/$$subdir $(OPTIONS); \
+		$(MAKE) -j $(nthreads) -C $$subdir OUTDIR=$(OUTDIR)/$$subdir; \
 	done
-	$(MAKE) -j $(nthreads) OUTDIR=$(OUTDIR) $(OPTIONS) $(OUTDIR)/${TARGLIB}.so
+	$(MAKE) -j $(nthreads) $(OUTDIR)/${TARGLIB}.so
+	$(MAKE) -j $(nthreads) $(OUTDIR)/${TARGLIB}.a
 	@mkdir -p $(OUTDIR)/compat
-	$(MAKE) -j $(nthreads) -C compat OUTDIR=$(OUTDIR)/compat $(OPTIONS)
+	$(MAKE) -j $(nthreads) -C compat OUTDIR=$(OUTDIR)/compat
 
 %_clean:
 	make OUTDIR=$* clean
 
-clean: linker_script_file_clean cleanlinks
+clean: cleanlinks
 	rm -rf ${OUTDIR}
 	@if [ -e $(HISTORY) ]; then \
 		grep -v -E "^$(OUTDIR)$$" $(HISTORY) > $(HISTORY)_tmp; \
@@ -294,12 +303,11 @@ clean: linker_script_file_clean cleanlinks
 	fi
 
 mock: OUTDIR := $(MOCK_OUTDIR)
-mock: OPTIONS = PSM2_MOCK_TESTING=1
 mock:
-	$(MAKE) OUTDIR=$(OUTDIR) OPTIONS=$(OPTIONS)
+	$(MAKE) OUTDIR=$(OUTDIR) PSM2_MOCK_TESTING=1
 
 debug: OUTDIR := $(DEBUG_OUTDIR)
-debug: OPTIONS = PSM_DEBUG=1
+debug: OPTIONS := PSM_DEBUG=1
 debug:
 	$(MAKE) OUTDIR=$(OUTDIR) OPTIONS=$(OPTIONS)
 
@@ -338,6 +346,8 @@ install: all
 	(cd ${DESTDIR}${INSTALL_LIB_TARG} ; \
 		ln -sf ${TARGLIB}.so.${MAJOR}.${MINOR} ${TARGLIB}.so.${MAJOR} ; \
 		ln -sf ${TARGLIB}.so.${MAJOR} ${TARGLIB}.so)
+	install -D $(OUTDIR)/${TARGLIB}.a \
+		${DESTDIR}${INSTALL_LIB_TARG}/${TARGLIB}.a
 	install -m 0644 -D psm2.h ${DESTDIR}/usr/include/psm2.h
 	install -m 0644 -D psm2_mq.h ${DESTDIR}/usr/include/psm2_mq.h
 	install -m 0644 -D psm2_am.h ${DESTDIR}/usr/include/psm2_am.h
@@ -479,7 +489,8 @@ ${TARGLIB}-objs := ptl_am/am_reqrep_shmem.o	\
 		   ptl_self/ptl.o		\
 		   opa/*.o			\
 		   psm_diags.o 			\
-		   psmi_wrappers.o
+		   psmi_wrappers.o              \
+		   psm_gdrcpy.o
 
 ${TARGLIB}-objs := $(patsubst %.o, ${OUTDIR}/%.o, ${${TARGLIB}-objs})
 
@@ -501,16 +512,16 @@ $(OUTDIR)/${TARGLIB}.so.${MAJOR}.${MINOR}: ${${TARGLIB}-objs} $(LINKER_SCRIPT_FI
 	date -u -d@$${SOURCE_DATE_EPOCH:-$$(date +%s)} +'char psmi_hfi_build_timestamp[] ="%F %T%:z";' >> ${OUTDIR}/_revision.c
 	echo "char psmi_hfi_sources_checksum[] =\"${SOURCES_CHKSUM_VALUE}\";" >> ${OUTDIR}/_revision.c
 	echo "char psmi_hfi_git_checksum[] =\"`git rev-parse HEAD`\";" >> ${OUTDIR}/_revision.c
-	$(CC) -c $(BASECFLAGS) $(INCLUDES) ${OUTDIR}/_revision.c -o $(OUTDIR)/_revision.o
+	$(CC) -c $(CFLAGS) $(BASECFLAGS) $(INCLUDES) ${OUTDIR}/_revision.c -o $(OUTDIR)/_revision.o
 	$(CC) $(LINKER_SCRIPT) $(LDFLAGS) -o $@ -Wl,-soname=${TARGLIB}.so.${MAJOR} -shared \
 		${${TARGLIB}-objs} $(OUTDIR)/_revision.o -Lopa $(LDLIBS)
 
+$(OUTDIR)/${TARGLIB}.a: $(OUTDIR)/${TARGLIB}.so.${MAJOR}.${MINOR}
+	$(AR) rcs $(OUTDIR)/${TARGLIB}.a ${${TARGLIB}-objs} $(OUTDIR)/_revision.o
+
 ${OUTDIR}/%.o: ${top_srcdir}/%.c
-	$(CC) $(CFLAGS) $(INCLUDES) -MMD -c $< -o $@
+	$(CC) $(CFLAGS) $(BASECFLAGS) $(INCLUDES) -MMD -c $< -o $@
 
 $(LINKER_SCRIPT_FILE): psm2_linker_script_map.in
 	sed "s/_psm2_additional_globals_;/$(PSM2_ADDITIONAL_GLOBALS)/" \
 	     psm2_linker_script_map.in > ${OUTDIR}/psm2_linker_script.map
-
-linker_script_file_clean:
-	rm -f $(LINKER_SCRIPT_FILE)

buildflags.mak

@@ -86,8 +86,9 @@ ASFLAGS += $(BASE_FLAGS)
 
 ifeq ($(PSM2_MOCK_TESTING),1)
 BASECFLAGS += -DPSM2_MOCK_TESTING=1
-# we skip the linker script for testing version, we want all symbols to be
-# reachable from outside the library
+unexport LINKER_SCRIPT
+# We skip the linker script for mock testing version, we want all symbols
+# to be reachable from outside the library
 else
 LINKER_SCRIPT := -Wl,--version-script $(LINKER_SCRIPT_FILE)
 endif
@@ -178,33 +179,36 @@ endif
 
 BASECFLAGS += -fpic -fPIC -D_GNU_SOURCE
 
-ifeq (${CCARCH},gcc)
-  BASECFLAGS += -funwind-tables
-endif
-
 ifneq (,${PSM_VALGRIND})
-  CFLAGS += -DPSM_VALGRIND
+  BASECFLAGS += -DPSM_VALGRIND
 else
-  CFLAGS += -DNVALGRIND
+  BASECFLAGS += -DNVALGRIND
 endif
 
 ASFLAGS += -g3 -fpic
 
 BASECFLAGS += ${OPA_CFLAGS}
 
 ifeq (${CCARCH},icc)
-    BASECFLAGS += -O3 -g3 -fpic -fPIC -D_GNU_SOURCE -DPACK_STRUCT_STL=packed,
-    CFLAGS += $(BASECFLAGS)
+    BASECFLAGS += -fpic -fPIC -D_GNU_SOURCE -DPACK_STRUCT_STL=packed,
     LDFLAGS += -static-intel
 else
 	ifeq (${CCARCH},gcc)
-	    CFLAGS += $(BASECFLAGS) -Wno-strict-aliasing -Wformat-security
+	    BASECFLAGS += -funwind-tables -Wno-strict-aliasing -Wformat-security
 	else
-	    ifeq (${CCARCH},gcc4)
-		CFLAGS += $(BASECFLAGS)
-	    else
+	    ifneq (${CCARCH},gcc4)
 		$(error Unknown compiler arch "${CCARCH}")
 	    endif # gcc4
 	endif # gcc
 endif # icc
 
+# We run export here to ensure all the above setup is in the environment
+# for sub makes. However, we exclude this during clean and distclean
+# to avoid resolution of some variables that don't need to be resolved
+# and avoid unnecessary missing file warnings during cleanup.
+ifneq ($(MAKECMDGOALS), clean)
+ifneq ($(MAKECMDGOALS), distclean)
+export
+endif
+endif
+

compat/Makefile

@@ -76,7 +76,7 @@ install: all
 	install -D $(OUTDIR)/${COMPATLIB}.so.${MAJOR} ${DESTDIR}${COMPAT_LIB_TARG}/${COMPATLIB}.so.${MAJOR}
 
 $(OUTDIR)/%.o: $(compat_build_dir)/%.c
-	$(CC) $(CFLAGS) $(INCLUDES) -MMD -c $< -o $@
+	$(CC) $(CFLAGS) $(BASECFLAGS) $(INCLUDES) -MMD -c $< -o $@
 
 $(OUTDIR)/${COMPATLIB}.so.${MAJOR}: ${${COMPATLIB}-objs}
 	$(CC) $(BASECFLAGS) $(LINKER_SCRIPT) $(LDFLAGS) -Wl,-soname=${COMPATLIB}.so.${MAJOR} -shared \

compat/buildflags.mak

@@ -88,15 +88,12 @@ ASFLAGS += -g3 -fpic
 
 ifeq (${CCARCH},icc)
     BASECFLAGS += -O3 -g3
-    CFLAGS += $(BASECFLAGS)
     LDFLAGS += -static-intel
 else
 	ifeq (${CCARCH},gcc)
-	    CFLAGS += $(BASECFLAGS) -Wno-strict-aliasing
+	    BASECFLAGS += -Wno-strict-aliasing
 	else
-		ifeq (${CCARCH},gcc4)
-			CFLAGS += $(BASECFLAGS)
-		else
+		ifneq (${CCARCH},gcc4)
 			$(error Unknown compiler arch "${CCARCH}")
 		endif
 	endif

include/opa_service.h

@@ -75,6 +75,11 @@
 
 /* base name of path (without unit #) for qib driver */
 #define HFI_DEVICE_PATH "/dev/hfi1"
+
+#ifdef PSM_CUDA
+#define GDR_DEVICE_PATH "/dev/hfi1_gdr"
+#endif
+
 #define HFI_CLASS_PATH "/sys/class/infiniband/hfi1"
 
 /* Commands used to communicate with driver. */

libpsm2.spec.in

@@ -73,7 +73,8 @@ Obsoletes: hfi1-psm < 1.0.0
 %package -n @RPM_NAME@@RPM_NAME_BASEEXT@
 %endif
 Summary: Intel PSM2 Libraries
-Provides: @RPM_NAME@
+Provides: @RPM_NAME@ = %{version}-%{release}
+Provides: @RPM_NAME@%{_isa} = %{version}-%{release}
 %if 0%{?suse_version}
 BuildRequires: libnuma-devel
 Requires: libnuma1
@@ -155,6 +156,7 @@ make %{?_smp_mflags}
 
 %files -n @RPM_NAME@-devel
 %{_libdir}/@[email protected]
+%{_libdir}/@[email protected]
 %{_includedir}/psm2.h
 %{_includedir}/psm2_mq.h
 %{_includedir}/psm2_am.h

libuuid/Makefile

@@ -55,8 +55,7 @@ OUTDIR = .
 
 this_srcdir := $(shell readlink -m .)
 top_srcdir := $(this_srcdir)/..
-include $(top_srcdir)/buildflags.mak
-CFLAGS += -DPSM_UUID=1 -Wno-unused-function
+BASECFLAGS += -DPSM_UUID=1 -Wno-unused-function
 INCLUDES += -I$(top_srcdir)
 
 ${TARGLIB}-objs := psm_uuid.o parse.o pack.o unpack.o unparse.o
@@ -70,10 +69,10 @@ IGNORE_DEP_TARGETS = clean
 all .DEFAULT: ${${TARGLIB}-objs}
 
 $(OUTDIR)/%.d: $(this_srcdir)/%.c
-	$(CC) $(CFLAGS) $(INCLUDES) $< -MM -MF $@ -MQ $(@:.d=.o)
+	$(CC) $(CFLAGS) $(BASECFLAGS) $(INCLUDES) $< -MM -MF $@ -MQ $(@:.d=.o)
 
 $(OUTDIR)/%.o: $(this_srcdir)/%.c | ${DEPS}
-	$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@
+	$(CC) $(CFLAGS) $(BASECFLAGS) $(INCLUDES) -c $< -o $@
 
 clean:
 	@if [ -d $(OUTDIR) ]; then \

makesrpm.sh

@@ -113,7 +113,7 @@ while [ "$1" != "" ]; do
                         if [ -z "$1" ]; then
                             usage
                         fi
-                        $RPM_NAME_BASEEXT="$1"
+                        RPM_NAME_BASEEXT="$1"
                         export RPM_NAME_BASEEXT="$1"
                         ;;
         -r | -rpmname)  shift

opa/Makefile

@@ -59,10 +59,8 @@ MINOR := $(OPA_LIB_MINOR)
 
 this_srcdir := $(shell readlink -m .)
 top_srcdir := $(this_srcdir)/..
-include $(top_srcdir)/buildflags.mak
-BASECFLAGS += -D_GNU_SOURCE
-INCLUDES += -I$(top_srcdir) -I$(top_srcdir)/ptl_ips
 
+INCLUDES += -I$(top_srcdir) -I$(top_srcdir)/ptl_ips
 ifeq (${arch},x86_64)
 	PLATFORM_OBJ=opa_dwordcpy-x86_64-fast.o
 else
@@ -86,13 +84,13 @@ install: all
 	@echo "Nothing to do for install."
 
 $(OUTDIR)/%.d:  $(this_srcdir)/%.c
-	$(CC) $(CFLAGS) $(INCLUDES) $< -MM -MF $@ -MQ $(@:.d=.o)
+	$(CC) $(CFLAGS) $(BASECFLAGS) $(INCLUDES) $< -MM -MF $@ -MQ $(@:.d=.o)
 
 $(OUTDIR)/%.d:  $(this_srcdir)/%.S
-	$(CC) $(CFLAGS) $(INCLUDES) $< -MM -MF $@ -MQ $(@:.d=.o)
+	$(CC) $(CFLAGS) $(BASECFLAGS) $(INCLUDES) $< -MM -MF $@ -MQ $(@:.d=.o)
 
 $(OUTDIR)/%.o: $(this_srcdir)/%.c | ${DEPS}
-	$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@
+	$(CC) $(CFLAGS) $(BASECFLAGS) $(INCLUDES) -c $< -o $@
 
 $(OUTDIR)/%.o: $(this_srcdir)/%.S | ${DEPS}
 	$(CC) $(ASFLAGS) -c $< -o $@

opa/opa_time.c

@@ -69,6 +69,16 @@
 
 #include "opa_user.h"
 
+#ifdef min
+#undef min
+#endif
+#define min(a, b) ((a) < (b) ? (a) : (b))
+
+#ifdef max
+#undef max
+#endif
+#define max(a, b) ((a) > (b) ? (a) : (b))
+
 /* init the cycle counter to picosecs/cycle conversion automatically */
 /* at program startup, if it's using timing functions. */
 static void init_picos_per_cycle(void) __attribute__ ((constructor));
@@ -224,6 +234,7 @@ static uint32_t hfi_timebase_from_cpuinfo(uint32_t old_pico_per_cycle)
 {
 	/* we only validate once */
 	uint32_t new_pico_per_cycle = old_pico_per_cycle;
+	uint32_t max_bet_new_old_pico, min_bet_new_old_pico;
 
 	char hostname[80];
 	gethostname(hostname, 80);
@@ -262,8 +273,10 @@ static uint32_t hfi_timebase_from_cpuinfo(uint32_t old_pico_per_cycle)
 	}
 #endif
 
+	max_bet_new_old_pico = max(new_pico_per_cycle, old_pico_per_cycle);
+	min_bet_new_old_pico = min(new_pico_per_cycle, old_pico_per_cycle);
 	/* If there's no change (within a small range), just return the old one */
-	if (abs(new_pico_per_cycle - old_pico_per_cycle) < 5)
+	if ((max_bet_new_old_pico - min_bet_new_old_pico) < 5)
 		return old_pico_per_cycle;
 
 	if (hfi_timebase_isvalid(new_pico_per_cycle)) {