Merge branch 'amd-develop' into amd-master

Change-Id: I5b104cbe9aa97f4152fcd7d5d86b606f69492f70
ROCm · Sep 30, 2016 · 4381557 · 4381557
2 parents 8ac4a51 + 4f61127
commit 4381557
Show file tree

Hide file tree

Showing 100 changed files with 2,997 additions and 1,530 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -145,6 +145,9 @@ add_to_config(_buildInfo COMPILE_HIP_ATP_MARKER)
 #############################
 # Build steps
 #############################
+# Rebuild cmake cache updates .buildInfo and .version
+add_custom_target(update_build_and_version_info ALL COMMAND make rebuild_cache)
+
 # Build clang hipify if enabled
 if(BUILD_CLANG_HIPIFY)
     add_subdirectory(clang-hipify)
@@ -153,18 +156,23 @@ endif()
 # Build hip_hcc if platform is hcc
 if(HIP_PLATFORM STREQUAL "hcc")
     include_directories(${PROJECT_SOURCE_DIR}/include)
+    set(HIP_HCC_BUILD_FLAGS)
     if(COMPILE_HIP_ATP_MARKER)
         include_directories(/opt/rocm/profiler/CXLActivityLogger/include)
-        set(CMAKE_CXX_FLAGS " -DCOMPILE_HIP_ATP_MARKER=1 ${CMAKE_CXX_FLAGS}")
-        set(CMAKE_C_FLAGS   " -DCOMPILE_HIP_ATP_MARKER=1 ${CMAKE_C_FLAGS}")
+        set(HIP_HCC_BUILD_FLAGS "${HIP_HCC_BUILD_FLAGS} -DCOMPILE_HIP_ATP_MARKER=1")
     endif()
 
+    # Add HIP_VERSION to CMAKE_<LANG>_FLAGS
+    set(HIP_HCC_BUILD_FLAGS "${HIP_HCC_BUILD_FLAGS} -DHIP_VERSION_MAJOR=${HIP_VERSION_MAJOR} -DHIP_VERSION_MINOR=${HIP_VERSION_MINOR} -DHIP_VERSION_PATCH=${HIP_VERSION_PATCH}")
+
+    # Add remaining flags
+    set(HIP_HCC_BUILD_FLAGS "${HIP_HCC_BUILD_FLAGS} -fPIC -hc -I${HCC_HOME}/include -I${HSA_PATH}/include -I/opt/rocm/libhsakmt/include/libhsakmt -stdlib=libc++")
+
+    # Set compiler and compiler flags
     set(CMAKE_CXX_COMPILER "${HCC_HOME}/bin/hcc")
     set(CMAKE_C_COMPILER   "${HCC_HOME}/bin/hcc")
-
-    # Set HIP_HCC so we know this is HIP compile, some files are shared with HCC (staging_buffer).
-    set(CMAKE_CXX_FLAGS " -fPIC -hc -I${HCC_HOME}/include -I${HSA_PATH}/include -I/opt/rocm/libhsakmt/include/libhsakmt -stdlib=libc++ -DHIP_HCC ${CMAKE_CXX_FLAGS}")
-    set(CMAKE_C_FLAGS   " -fPIC -hc -I${HCC_HOME}/include -I${HSA_PATH}/include -I/opt/rocm/libhsakmt/include/libhsakmt -stdlib=libc++ -DHIP_HCC ${CMAKE_C_FLAGS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${HIP_HCC_BUILD_FLAGS}")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${HIP_HCC_BUILD_FLAGS}")
 
     set(SOURCE_FILES src/device_util.cpp
                      src/hip_hcc.cpp
@@ -177,7 +185,6 @@ if(HIP_PLATFORM STREQUAL "hcc")
                      src/hip_peer.cpp
                      src/hip_stream.cpp
                      src/hip_fp16.cpp
-                     src/unpinned_copy_engine.cpp
                      src/hip_module.cpp)
 
     if(${HIP_LIB_TYPE} EQUAL 0)
@@ -188,6 +195,13 @@ if(HIP_PLATFORM STREQUAL "hcc")
         add_library(hip_hcc SHARED ${SOURCE_FILES})
     endif()
 
+    # Generate hcc_version.txt
+    add_custom_target(query_hcc_version COMMAND ${HCC_HOME}/bin/hcc --version > ${PROJECT_BINARY_DIR}/hcc_version.tmp)
+    add_custom_target(check_hcc_version COMMAND ${CMAKE_COMMAND} -E copy_if_different ${PROJECT_BINARY_DIR}/hcc_version.tmp ${PROJECT_BINARY_DIR}/hcc_version.txt DEPENDS query_hcc_version)
+    set_source_files_properties(${PROJECT_BINARY_DIR}/hcc_version.txt PROPERTIES GENERATED TRUE)
+    set_source_files_properties(${SOURCE_FILES} PROPERTIES OBJECT_DEPENDS ${PROJECT_BINARY_DIR}/hcc_version.txt)
+    add_dependencies(hip_hcc check_hcc_version update_build_and_version_info)
+
     # Generate .buildInfo
     file(WRITE "${PROJECT_BINARY_DIR}/.buildInfo" ${_buildInfo})
 endif()
@@ -300,3 +314,25 @@ if(POLICY CMP0037)
     cmake_policy(POP)
 endif()
 
+#############################
+# Testing steps
+#############################
+# Target: test
+set(BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/hip_tests)
+configure_file(tests/hip_tests.txt ${BUILD_DIR}/CMakeLists.txt @ONLY)
+if(POLICY CMP0037)
+    cmake_policy(PUSH)
+    cmake_policy(SET CMP0037 OLD)
+endif()
+add_custom_target(install_for_test COMMAND "${CMAKE_COMMAND}" --build . --target install
+                                   WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+execute_process(COMMAND getconf _NPROCESSORS_ONLN OUTPUT_VARIABLE DASH_JAY OUTPUT_STRIP_TRAILING_WHITESPACE)
+add_custom_target(test COMMAND ${CMAKE_COMMAND} .
+                       COMMAND make -j ${DASH_JAY}
+                       COMMAND make test
+                       WORKING_DIRECTORY ${BUILD_DIR}
+                       DEPENDS install_for_test)
+if(POLICY CMP0037)
+    cmake_policy(POP)
+endif()
+
diff --git a/RELEASE.md b/RELEASE.md
@@ -7,6 +7,7 @@ Upcoming:
   number of commands (>1K) without synchronizing.  
 - Register keyword now silently ignored on HCC (previously would emit warning).
 - Doc updates: Add some more frequently asked questions to FAQ, fix TOC in some files, review.
+- Cookbook.
 
 ===================================================================================================
 

diff --git a/bin/hipcc b/bin/hipcc
@@ -60,6 +60,7 @@ parse_config_file("$HIP_PATH/lib/.buildInfo", \%hipConfig);
 #HIP_PLATFORM controls whether to use NVCC or HCC for compilation:
 $HIP_PLATFORM= `$HIP_PATH/bin/hipconfig --platform` // "hcc";
 $HIP_VERSION= `$HIP_PATH/bin/hipconfig --version`;
+($HIP_VERSION_MAJOR, $HIP_VERSION_MINOR, $HIP_VERSION_PATCH) = split(/\./, $HIP_VERSION);
 
 if ($verbose & 0x2) {
     print ("HIP_PATH=$HIP_PATH\n");
@@ -151,14 +152,14 @@ if ($HIP_PLATFORM eq "hcc") {
     $HIPCC="$CUDA_PATH/bin/nvcc";
     $HIPCXXFLAGS .= " -I$CUDA_PATH/include";
 
-    $HIPLDFLAGS = "-lcuda -lcudart";
+    $HIPLDFLAGS = "-lcuda -lcudart -L$CUDA_PATH/lib64";
 } else {
     printf ("error: unknown HIP_PLATFORM = '$HIP_PLATFORM'");
     exit (-1);
 }
 
 # Add paths to common HIP includes:
-$HIPCXXFLAGS .= " -I$HIP_PATH/include" ;
+$HIPCXXFLAGS .= " -I$HIP_PATH/include -DHIP_VERSION_MAJOR=$HIP_VERSION_MAJOR -DHIP_VERSION_MINOR=$HIP_VERSION_MINOR -DHIP_VERSION_PATCH=$HIP_VERSION_PATCH" ;
 
 my $compileOnly = 0;
 my $needCXXFLAGS = 0;  # need to add CXX flags to compile step
@@ -224,12 +225,12 @@ foreach $arg (@ARGV)
         $needLDFLAGS = 1;
     }
 
-    if($arg eq '-stdlib=libc++' and $setStdLib eq 0)
+    if(($arg eq '-stdlib=libc++') and ($setStdLib eq 0))
     {
         $HIPCXXFLAGS .= " -stdlib=libc++";
         $setStdLib = 1;
     }
-    if($arg eq '-stdlib=libstdc++' and $setStdLib eq 0)
+    if(($arg eq '-stdlib=libstdc++') and ($setStdLib eq 0))
     {
         $HIPCXXFLAGS .= " -stdlib=libstdc++";
         $setStdLib = 1;
@@ -296,7 +297,7 @@ if ($needHipHcc) {
     $HIP_LIB_TYPE = $hipConfig{'HIP_LIB_TYPE'} // 0;
 
     if ($HIP_LIB_TYPE eq 0) {
-        $HIPLDFLAGS .= " $HIP_PATH/lib/device_util.cpp.o $HIP_PATH/lib/hip_device.cpp.o $HIP_PATH/lib/hip_error.cpp.o $HIP_PATH/lib/hip_event.cpp.o $HIP_PATH/lib/hip_hcc.cpp.o $HIP_PATH/lib/hip_memory.cpp.o $HIP_PATH/lib/hip_peer.cpp.o $HIP_PATH/lib/hip_stream.cpp.o $HIP_PATH/lib/unpinned_copy_engine.cpp.o $HIP_PATH/lib/hip_ldg.cpp.o $HIP_PATH/lib/hip_fp16.cpp.o $HIP_PATH/lib/hip_context.cpp.o $HIP_PATH/lib/hip_module.cpp.o";
+        $HIPLDFLAGS .= " $HIP_PATH/lib/device_util.cpp.o $HIP_PATH/lib/hip_device.cpp.o $HIP_PATH/lib/hip_error.cpp.o $HIP_PATH/lib/hip_event.cpp.o $HIP_PATH/lib/hip_hcc.cpp.o $HIP_PATH/lib/hip_memory.cpp.o $HIP_PATH/lib/hip_peer.cpp.o $HIP_PATH/lib/hip_stream.cpp.o $HIP_PATH/lib/hip_ldg.cpp.o $HIP_PATH/lib/hip_fp16.cpp.o $HIP_PATH/lib/hip_context.cpp.o $HIP_PATH/lib/hip_module.cpp.o";
     } elsif ($HIP_LIB_TYPE eq 1) {
         $HIPLDFLAGS .= " -L$HIP_PATH/lib -lhip_hcc" ;
     } else {
@@ -332,7 +333,7 @@ if ($printHipVersion) {
 }
 if ($runCmd) {
     if ($HIP_PLATFORM eq "hcc" and exists($hipConfig{'HCC_VERSION'}) and $HCC_VERSION ne $hipConfig{'HCC_VERSION'}) {
-        print ("HIP was built using $hipConfig{'HCC_VERSION'}, but you are using $HCC_VERSION. Please rebuild HIP.\n") && die ();
+        print ("HIP ($HIP_PATH) was built using hcc $hipConfig{'HCC_VERSION'}, but you are using hcc $HCC_VERSION. Please rebuild HIP including cmake.\n") && die ();
     }
     system ("$CMD") and die ();
 }
diff --git a/bin/hipify b/bin/hipify
@@ -288,7 +288,7 @@ while (@ARGV) {
         $ft{'mem'} += s/\bcudaMallocHost\b/hipHostMalloc/g;  # note conversion to standard hipHost* naming convention
         $ft{'mem'} += s/\bcudaFree\b/hipFree/g;
         $ft{'mem'} += s/\bcudaFreeHost\b/hipHostFree/g;    # note conversion to standard hipHost* naming convention
-		$ft{'mem'} += s/\bcudaHostAlloc\b/hipHostAlloc/g;
+		$ft{'mem'} += s/\bcudaHostAlloc\b/hipHostMalloc/g;
 		$ft{'mem'} += s/\bcudaHostGetDevicePointer\b/hipHostGetDevicePointer/g;
 		$ft{'mem'} += s/\bcudaHostAllocDefault\b/hipHostMallocDefault/g;
 		$ft{'mem'} += s/\bcudaHostAllocPortable\b/hipHostMallocPortable/g;

diff --git a/cmake/FindHIP.cmake b/cmake/FindHIP.cmake
@@ -193,9 +193,15 @@ macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_op
     foreach(arg ${ARGN})
         if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
             set(_hipcc_found_options TRUE)
+            set(_hcc_found_options FALSE)
+            set(_nvcc_found_options FALSE)
         elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
+            set(_hipcc_found_options FALSE)
             set(_hcc_found_options TRUE)
+            set(_nvcc_found_options FALSE)
         elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
+            set(_hipcc_found_options FALSE)
+            set(_hcc_found_options FALSE)
             set(_nvcc_found_options TRUE)
         elseif(
                 "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
@@ -235,9 +241,9 @@ endmacro()
 function(HIP_COMPUTE_BUILD_PATH path build_path)
     # Convert to cmake style paths
     file(TO_CMAKE_PATH "${path}" bpath)
-    if (IS_ABSOLUTE "${bpath}")
+    if(IS_ABSOLUTE "${bpath}")
         string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
-        if (_binary_dir_pos EQUAL 0)
+        if(_binary_dir_pos EQUAL 0)
             file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
         else()
             file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
@@ -316,7 +322,7 @@ endmacro()
 ###############################################################################
 # MACRO: Prepare cmake commands for the target
 ###############################################################################
-macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files)
+macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
     set(_hip_flags "")
     set(_hip_build_configuration "${CMAKE_BUILD_TYPE}")
     if(HIP_HOST_COMPILATION_CPP)
@@ -378,6 +384,7 @@ macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files)
 
     # Reset the output variable
     set(_hip_generated_files "")
+    set(_hip_source_files "")
 
     # Iterate over all arguments and create custom commands for all source files
     foreach(file ${ARGN})
@@ -391,77 +398,79 @@ macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files)
             set(host_flag TRUE)
         endif()
 
-        if (NOT host_flag)
-        # Determine output directory
-        HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
-        set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
-
-        get_filename_component(basename ${file} NAME)
-        set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
-        set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
-
-        # Set file names
-        set(generated_file "${generated_file_path}/${generated_file_basename}")
-        set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
-        set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
-        set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
-
-        # Set properties for object files
-        set_source_files_properties("${generated_file}"
-            PROPERTIES
-            EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
-            )
+        if(NOT host_flag)
+            # Determine output directory
+            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
+            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
+
+            get_filename_component(basename ${file} NAME)
+            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
+            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
+
+            # Set file names
+            set(generated_file "${generated_file_path}/${generated_file_basename}")
+            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
+            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
+            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
+
+            # Set properties for object files
+            set_source_files_properties("${generated_file}"
+                PROPERTIES
+                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
+                )
 
-        # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
-        get_filename_component(file_path "${file}" PATH)
-        if(IS_ABSOLUTE "${file_path}")
-            set(source_file "${file}")
-        else()
-            set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
-        endif()
+            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
+            get_filename_component(file_path "${file}" PATH)
+            if(IS_ABSOLUTE "${file_path}")
+                set(source_file "${file}")
+            else()
+                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
+            endif()
 
-        # Bring in the dependencies
-        HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
+            # Bring in the dependencies
+            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
 
-        # Configure the build script
-        configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
-        file(GENERATE
-            OUTPUT "${custom_target_script}"
-            INPUT "${custom_target_script_pregen}"
-            )
-        set(main_dep DEPENDS ${source_file})
-        set(verbose_output "$(VERBOSE)")
-
-        # Create up the comment string
-        file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
-        set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
-
-        # Build the generated file and dependency file
-        add_custom_command(
-            OUTPUT ${generated_file}
-            # These output files depend on the source_file and the contents of cmake_dependency_file
-            ${main_dep}
-            DEPENDS ${HIP_HIPCC_DEPEND}
-            DEPENDS ${custom_target_script}
-            # Make sure the output directory exists before trying to write to it.
-            COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
-            COMMAND ${CMAKE_COMMAND} ARGS
-            -D verbose:BOOL=${verbose_output}
-            -D build_configuration:STRING=${_hip_build_configuration}
-            -D "generated_file:STRING=${generated_file}"
-            -P "${custom_target_script}"
-            WORKING_DIRECTORY "${hip_compile_output_dir}"
-            COMMENT "${hip_build_comment_string}"
-            )
+            # Configure the build script
+            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
+            file(GENERATE
+                OUTPUT "${custom_target_script}"
+                INPUT "${custom_target_script_pregen}"
+                )
+            set(main_dep DEPENDS ${source_file})
+            set(verbose_output "$(VERBOSE)")
+
+            # Create up the comment string
+            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
+            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
+
+            # Build the generated file and dependency file
+            add_custom_command(
+                OUTPUT ${generated_file}
+                # These output files depend on the source_file and the contents of cmake_dependency_file
+                ${main_dep}
+                DEPENDS ${HIP_HIPCC_DEPEND}
+                DEPENDS ${custom_target_script}
+                # Make sure the output directory exists before trying to write to it.
+                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
+                COMMAND ${CMAKE_COMMAND} ARGS
+                -D verbose:BOOL=${verbose_output}
+                -D build_configuration:STRING=${_hip_build_configuration}
+                -D "generated_file:STRING=${generated_file}"
+                -P "${custom_target_script}"
+                WORKING_DIRECTORY "${hip_compile_output_dir}"
+                COMMENT "${hip_build_comment_string}"
+                )
 
-        # Make sure the build system knows the file is generated
-        set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
-        list(APPEND _hip_generated_files ${generated_file})
+            # Make sure the build system knows the file is generated
+            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
+            list(APPEND _hip_generated_files ${generated_file})
+            list(APPEND _hip_source_files ${file})
         endif()
     endforeach()
 
     # Set the return parameter
     set(${_generated_files} ${_hip_generated_files})
+    set(${_source_files} ${_hip_source_files})
 endmacro()
 
 ###############################################################################
@@ -470,7 +479,8 @@ endmacro()
 macro(HIP_ADD_EXECUTABLE hip_target)
     # Separate the sources from the options
     HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
+    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
+    list(REMOVE_ITEM _sources ${_source_files})
     set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_EXECUTABLE} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET>")
     add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
     set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
@@ -482,7 +492,8 @@ endmacro()
 macro(HIP_ADD_LIBRARY hip_target)
     # Separate the sources from the options
     HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
+    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options})
+    list(REMOVE_ITEM _sources ${_source_files})
     add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
     set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
 endmacro()