From 94865c302a51b0cd3586c2bfbf0f4f6ed19f1e62 Mon Sep 17 00:00:00 2001 From: rachguo Date: Sat, 9 Mar 2024 00:11:58 -0800 Subject: [PATCH] update sanity checks for the scripts and a fix --- cmake/handle_duplicate_object_files.py | 42 ++++++++++++++++++++++++-- cmake/onnxruntime.cmake | 17 ++++++++--- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/cmake/handle_duplicate_object_files.py b/cmake/handle_duplicate_object_files.py index a61ec274ae99c..1d918504b9b19 100644 --- a/cmake/handle_duplicate_object_files.py +++ b/cmake/handle_duplicate_object_files.py @@ -3,8 +3,9 @@ # Licensed under the MIT License. import os -import sys +import re import shutil +import sys # Note: This script is mainly used for handling extracting duplicate named .o files under different subdirectories for @@ -12,15 +13,18 @@ def main(): source_dir = sys.argv[1] dest_dir = sys.argv[2] + files_from_static_lib = sys.argv[3] + files_from_source_dir = [] for subdir, dirs, files in os.walk(source_dir): for file_name in files: if file_name.endswith(".o"): + files_from_source_dir.append(file_name.strip()) dest_name_without_extension, _ = os.path.splitext(file_name) counter = 0 dest_file = f"{dest_name_without_extension}.o" while os.path.exists(os.path.join(dest_dir, dest_file)): - print("Duplicates" + os.path.join(dest_dir, dest_file)) + print("Duplicate named files: " + os.path.join(dest_dir, dest_file)) counter += 1 dest_file = f"{dest_name_without_extension}_{counter}.o" @@ -28,6 +32,40 @@ def main(): source_file = os.path.join(source_dir, subdir, file_name) shutil.copy(source_file, destination_path) + # Sanity check to ensure the number of .o object from the original cmake source directory matches with the number + # of .o files extracted from each onnxruntime library + file_lists_from_static_lib = [] + with open(files_from_static_lib, "r") as file: + filenames = file.readlines() + for filename in filenames: + file_lists_from_static_lib.append(filename.strip()) + + sorted_list1 = sorted(file_lists_from_static_lib) + sorted_list2 = sorted(files_from_source_dir) + + if len(sorted_list1) != len(sorted_list2): + print( + "Caught a mismatch in the number of .o object files from the original cmake source directory: ", + len(sorted_list1), + "the number of .o files extracted from the static onnxruntime lib: ", + len(sorted_list2), + "for: ", + os.path.basename(source_dir), + ) + + if sorted_list1 == sorted_list2: + print( + "Sanity check passed: object files from original source directory matches with files extracted " + + "from static library for: ", + os.path.basename(source_dir), + ) + else: + print( + "Error: Mismatch between object files from original source directory " + + "and the .o files extracted from static library for: ", + os.path.basename(source_dir), + ) + if __name__ == "__main__": main() diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index d0b964f47bcfc..6fac840be22df 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -305,8 +305,8 @@ if(onnxruntime_BUILD_APPLE_FRAMEWORK) # to enforce symbol visibility. doing it this way limits the symbols included from the .a files to symbols used # by the ORT .o files. - # If it's an onnxruntime library, extract .o files to a separate directory for each library to avoid any clashes - # with filenames (e.g. utils.o) + # If it's an onnxruntime library, extract .o files from the original cmake build path to a separate directory for + # each library to avoid any clashes with filenames (e.g. utils.o) foreach(_LIB ${onnxruntime_INTERNAL_LIBRARIES} ) GET_TARGET_PROPERTY(_LIB_TYPE ${_LIB} TYPE) if(_LIB_TYPE STREQUAL "STATIC_LIBRARY") @@ -320,9 +320,16 @@ if(onnxruntime_BUILD_APPLE_FRAMEWORK) # Simply use 'ar ARGS -x' for extracting the .o files would possibly cause duplicate naming files being overwritten. set(CUR_TARGET_CMAKE_SOURCE_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_LIB}.dir) add_custom_command(TARGET onnxruntime POST_BUILD - COMMAND mkdir -p ${CUR_STATIC_LIB_OBJ_DIR} - COMMAND ${CMAKE_COMMAND} -E env python3 ${CMAKE_CURRENT_SOURCE_DIR}/handle_duplicate_object_files.py ${CUR_TARGET_CMAKE_SOURCE_LIB_DIR} ${CUR_STATIC_LIB_OBJ_DIR} - WORKING_DIRECTORY ${CUR_STATIC_LIB_OBJ_DIR}) + COMMAND ar -t $ | grep "\.o$" > object_file_list.txt + WORKING_DIRECTORY ${CUR_STATIC_LIB_OBJ_DIR}) + add_custom_command(TARGET onnxruntime POST_BUILD + COMMAND mkdir -p ${CUR_STATIC_LIB_OBJ_DIR} + COMMAND ${CMAKE_COMMAND} -E env python3 ${CMAKE_CURRENT_SOURCE_DIR}/handle_duplicate_object_files.py ${CUR_TARGET_CMAKE_SOURCE_LIB_DIR} ${CUR_STATIC_LIB_OBJ_DIR} ${CUR_STATIC_LIB_OBJ_DIR}/object_file_list.txt + WORKING_DIRECTORY ${CUR_STATIC_LIB_OBJ_DIR}) + else() + add_custom_command(TARGET onnxruntime POST_BUILD + COMMAND ar ARGS -x $ + WORKING_DIRECTORY ${CUR_STATIC_LIB_OBJ_DIR}) endif() endif() endforeach()