From d965666766060119fdbc5578bfd3d3742e3faf15 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 17 Jan 2024 15:40:18 -0800 Subject: [PATCH 01/27] update --- onnxruntime/test/providers/cpu/model_tests.cc | 127 +++++++++--------- 1 file changed, 63 insertions(+), 64 deletions(-) diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 859e082716760..745c677c4eed4 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -3,6 +3,12 @@ #include #include +#include +#include +#include +#include +#include +#include #include #include "core/session/onnxruntime_c_api.h" @@ -15,9 +21,6 @@ #include #include "default_providers.h" #include "test/onnx/TestCase.h" -#include -#include -#include #ifdef USE_DNNL #include "core/providers/dnnl/dnnl_provider_factory.h" @@ -361,46 +364,46 @@ TEST_P(ModelTest, Run) { } using ORT_STRING_VIEW = std::basic_string_view; -static ORT_STRING_VIEW opset7 = ORT_TSTR("opset7"); -static ORT_STRING_VIEW opset8 = ORT_TSTR("opset8"); -static ORT_STRING_VIEW opset9 = ORT_TSTR("opset9"); -static ORT_STRING_VIEW opset10 = ORT_TSTR("opset10"); -static ORT_STRING_VIEW opset11 = ORT_TSTR("opset11"); -static ORT_STRING_VIEW opset12 = ORT_TSTR("opset12"); -static ORT_STRING_VIEW opset13 = ORT_TSTR("opset13"); -static ORT_STRING_VIEW opset14 = ORT_TSTR("opset14"); -static ORT_STRING_VIEW opset15 = ORT_TSTR("opset15"); -static ORT_STRING_VIEW opset16 = ORT_TSTR("opset16"); -static ORT_STRING_VIEW opset17 = ORT_TSTR("opset17"); -static ORT_STRING_VIEW opset18 = ORT_TSTR("opset18"); +static constexpr ORT_STRING_VIEW opset7 = ORT_TSTR("opset7"); +static constexpr ORT_STRING_VIEW opset8 = ORT_TSTR("opset8"); +static constexpr ORT_STRING_VIEW opset9 = ORT_TSTR("opset9"); +static constexpr ORT_STRING_VIEW opset10 = ORT_TSTR("opset10"); +static constexpr ORT_STRING_VIEW opset11 = ORT_TSTR("opset11"); +static constexpr ORT_STRING_VIEW opset12 = ORT_TSTR("opset12"); +static constexpr ORT_STRING_VIEW opset13 = ORT_TSTR("opset13"); +static constexpr ORT_STRING_VIEW opset14 = ORT_TSTR("opset14"); +static constexpr ORT_STRING_VIEW opset15 = ORT_TSTR("opset15"); +static constexpr ORT_STRING_VIEW opset16 = ORT_TSTR("opset16"); +static constexpr ORT_STRING_VIEW opset17 = ORT_TSTR("opset17"); +static constexpr ORT_STRING_VIEW opset18 = ORT_TSTR("opset18"); // TODO: enable opset19 tests -// static ORT_STRING_VIEW opset19 = ORT_TSTR("opset19"); +// static constexpr ORT_STRING_VIEW opset19 = ORT_TSTR("opset19"); -static ORT_STRING_VIEW provider_name_cpu = ORT_TSTR("cpu"); -static ORT_STRING_VIEW provider_name_tensorrt = ORT_TSTR("tensorrt"); +static constexpr ORT_STRING_VIEW provider_name_cpu = ORT_TSTR("cpu"); +static constexpr ORT_STRING_VIEW provider_name_tensorrt = ORT_TSTR("tensorrt"); #ifdef USE_MIGRAPHX -static ORT_STRING_VIEW provider_name_migraphx = ORT_TSTR("migraphx"); +static constexpr ORT_STRING_VIEW provider_name_migraphx = ORT_TSTR("migraphx"); #endif -static ORT_STRING_VIEW provider_name_openvino = ORT_TSTR("openvino"); -static ORT_STRING_VIEW provider_name_cuda = ORT_TSTR("cuda"); +static constexpr ORT_STRING_VIEW provider_name_openvino = ORT_TSTR("openvino"); +static constexpr ORT_STRING_VIEW provider_name_cuda = ORT_TSTR("cuda"); #ifdef USE_ROCM -static ORT_STRING_VIEW provider_name_rocm = ORT_TSTR("rocm"); +static constexpr ORT_STRING_VIEW provider_name_rocm = ORT_TSTR("rocm"); #endif -static ORT_STRING_VIEW provider_name_dnnl = ORT_TSTR("dnnl"); +static constexpr ORT_STRING_VIEW provider_name_dnnl = ORT_TSTR("dnnl"); // For any non-Android system, NNAPI will only be used for ort model converter #if defined(USE_NNAPI) && defined(__ANDROID__) -static ORT_STRING_VIEW provider_name_nnapi = ORT_TSTR("nnapi"); +static constexpr ORT_STRING_VIEW provider_name_nnapi = ORT_TSTR("nnapi"); #endif #ifdef USE_RKNPU -static ORT_STRING_VIEW provider_name_rknpu = ORT_TSTR("rknpu"); +static constexpr ORT_STRING_VIEW provider_name_rknpu = ORT_TSTR("rknpu"); #endif #ifdef USE_ACL -static ORT_STRING_VIEW provider_name_acl = ORT_TSTR("acl"); +static constexpr ORT_STRING_VIEW provider_name_acl = ORT_TSTR("acl"); #endif #ifdef USE_ARMNN -static ORT_STRING_VIEW provider_name_armnn = ORT_TSTR("armnn"); +static constexpr ORT_STRING_VIEW provider_name_armnn = ORT_TSTR("armnn"); #endif -static ORT_STRING_VIEW provider_name_dml = ORT_TSTR("dml"); +static constexpr ORT_STRING_VIEW provider_name_dml = ORT_TSTR("dml"); ::std::vector<::std::basic_string> GetParameterStrings() { // Map key is provider name(CPU, CUDA, etc). Value is the ONNX node tests' opsets to run. @@ -598,7 +601,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { ORT_TSTR("SSD"), // needs to run symbolic shape inference shape first ORT_TSTR("size") // INVALID_ARGUMENT: Cannot find binding of given name: x }; - std::vector> paths; + std::vector paths; for (std::pair> kvp : provider_names) { // Setup ONNX node tests. The test data is preloaded on our CI build machines. @@ -627,7 +630,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { } #endif - ORT_STRING_VIEW provider_name = kvp.first; + const ORT_STRING_VIEW provider_name = kvp.first; std::unordered_set> all_disabled_tests(std::begin(immutable_broken_tests), std::end(immutable_broken_tests)); if (provider_name == provider_name_cuda) { @@ -682,45 +685,41 @@ ::std::vector<::std::basic_string> GetParameterStrings() { all_disabled_tests.insert(ORT_TSTR("fp16_tiny_yolov2")); while (!paths.empty()) { - std::basic_string node_data_root_path = paths.back(); + std::filesystem::path node_data_root_path = paths.back(); paths.pop_back(); - std::basic_string my_dir_name = GetLastComponent(node_data_root_path); - ORT_TRY { - LoopDir(node_data_root_path, [&](const ORTCHAR_T* filename, OrtFileType f_type) -> bool { - if (filename[0] == ORT_TSTR('.')) - return true; - if (f_type == OrtFileType::TYPE_DIR) { - std::basic_string p = ConcatPathComponent(node_data_root_path, filename); - paths.push_back(p); - return true; - } - std::basic_string filename_str = filename; - if (!HasExtensionOf(filename_str, ORT_TSTR("onnx"))) - return true; - std::basic_string test_case_name = my_dir_name; - if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) - test_case_name = test_case_name.substr(5); - if (all_disabled_tests.find(test_case_name) != all_disabled_tests.end()) - return true; + if (!std::filesystem::exists(node_data_root_path) || !std::filesystem::is_directory(node_data_root_path)) { + continue; + } + for (auto const& dir_entry : std::filesystem::directory_iterator(node_data_root_path)) { + if (dir_entry.is_directory()) { + paths.push_back(dir_entry.path()); + continue; + } + const std::filesystem::path& path = dir_entry.path(); + if (!path.filename().has_extension()) { + continue; + } + if (path.filename().extension().compare(ORT_TSTR(".onnx")) != 0) continue; + std::basic_string test_case_name = path.parent_path().filename().string(); + if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) + test_case_name = test_case_name.substr(5); + if (all_disabled_tests.find(test_case_name) != all_disabled_tests.end()) + continue; #ifdef DISABLE_ML_OPS - auto starts_with = [](const std::basic_string& find_in, - const std::basic_string& find_what) { - return find_in.compare(0, find_what.size(), find_what) == 0; - }; - if (starts_with(test_case_name, ORT_TSTR("XGBoost_")) || starts_with(test_case_name, ORT_TSTR("coreml_")) || - starts_with(test_case_name, ORT_TSTR("scikit_")) || starts_with(test_case_name, ORT_TSTR("libsvm_"))) { - return true; - } + auto starts_with = [](const std::basic_string& find_in, + const std::basic_string& find_what) { + return find_in.compare(0, find_what.size(), find_what) == 0; + }; + if (starts_with(test_case_name, ORT_TSTR("XGBoost_")) || starts_with(test_case_name, ORT_TSTR("coreml_")) || + starts_with(test_case_name, ORT_TSTR("scikit_")) || starts_with(test_case_name, ORT_TSTR("libsvm_"))) { + continue; + } #endif - std::basic_ostringstream oss; - oss << provider_name << ORT_TSTR("_") << ConcatPathComponent(node_data_root_path, filename_str); - v.emplace_back(oss.str()); - return true; - }); + std::basic_ostringstream oss; + oss << provider_name << ORT_TSTR("_") << path.string(); + v.emplace_back(oss.str()); } - ORT_CATCH(const std::exception&) { - } // ignore non-exist dir } } return v; From 9d552308841eb788cdf19f87619b86389831403d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 17 Jan 2024 16:43:53 -0800 Subject: [PATCH 02/27] update --- onnxruntime/test/onnx/TestCase.cc | 132 +++++++++--------- onnxruntime/test/onnx/TestCase.h | 18 +-- onnxruntime/test/onnx/onnx_model_info.cc | 6 +- onnxruntime/test/onnx/onnx_model_info.h | 10 +- onnxruntime/test/onnx/testcase_request.cc | 2 +- onnxruntime/test/perftest/TFModelInfo.cc | 58 -------- onnxruntime/test/perftest/TFModelInfo.h | 32 ----- .../test/perftest/performance_runner.cc | 5 - onnxruntime/test/providers/cpu/model_tests.cc | 1 - 9 files changed, 84 insertions(+), 180 deletions(-) delete mode 100644 onnxruntime/test/perftest/TFModelInfo.cc delete mode 100644 onnxruntime/test/perftest/TFModelInfo.h diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 57c2061883736..ae2aa96253d13 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -267,12 +267,12 @@ void LoopDataFile(int test_data_pb_fd, bool is_input, const TestModelInfo& model } // namespace #if !defined(ORT_MINIMAL_BUILD) -std::unique_ptr TestModelInfo::LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url) { +std::unique_ptr TestModelInfo::LoadOnnxModel(const std::filesystem::path& model_url) { return std::make_unique(model_url); } #endif -std::unique_ptr TestModelInfo::LoadOrtModel(_In_ const PATH_CHAR_TYPE* model_url) { +std::unique_ptr TestModelInfo::LoadOrtModel(const std::filesystem::path& model_url) { return std::make_unique(model_url, true); } @@ -290,7 +290,7 @@ class OnnxTestCase : public ITestCase { mutable std::vector debuginfo_strings_; mutable onnxruntime::OrtMutex m_; - std::vector> test_data_dirs_; + std::vector test_data_dirs_; std::string GetDatasetDebugInfoString(size_t dataset_id) const override { std::lock_guard l(m_); @@ -343,7 +343,7 @@ class OnnxTestCase : public ITestCase { size_t GetDataCount() const override { return test_data_dirs_.size(); } const std::string& GetNodeName() const override { return model_info_->GetNodeName(); } - const PATH_CHAR_TYPE* GetModelUrl() const override { return model_info_->GetModelUrl(); } + const std::filesystem::path& GetModelUrl() const override { return model_info_->GetModelUrl(); } const std::string& GetTestCaseName() const override { return test_case_name_; } std::string GetTestCaseVersion() const override { return model_info_->GetNominalOpsetVersion(); } @@ -396,7 +396,14 @@ static std::string trim_str(const std::string& in) { return s; } -static bool read_config_file(const std::basic_string& path, std::map& fc) { +/** + * @brief Read a text file that each line is a key value pair separated by ':' + * @param path File path + * @param fc output key value pairs + * @return True, success. False, the file doesn't exist or could be read. + */ +static bool ReadConfigFile(const std::filesystem::path& path, std::map& fc) { + if (!std::filesystem::exists(path)) return false; std::ifstream infile(path); if (!infile.good()) { return false; @@ -474,10 +481,10 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, ORT_THROW("index out of bound"); } - PATH_STRING_TYPE test_data_pb = ConcatPathComponent( - test_data_dirs_[id], (is_input ? ORT_TSTR("inputs.pb") : ORT_TSTR("outputs.pb"))); + std::filesystem::path test_data_pb = + test_data_dirs_[id] / (is_input ? ORT_TSTR("inputs.pb") : ORT_TSTR("outputs.pb")); int test_data_pb_fd; - auto st = Env::Default().FileOpenRd(test_data_pb, test_data_pb_fd); + auto st = Env::Default().FileOpenRd(test_data_pb.string(), test_data_pb_fd); if (st.IsOK()) { // has an all-in-one input file std::ostringstream oss; { @@ -505,20 +512,24 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::vector test_data_pb_files; const PATH_STRING_TYPE& dir_path = test_data_dirs_[id]; - LoopDir(dir_path, - [&test_data_pb_files, &dir_path, is_input](const PATH_CHAR_TYPE* filename, OrtFileType f_type) -> bool { - if (filename[0] == '.') return true; - if (f_type != OrtFileType::TYPE_REG) return true; - std::basic_string filename_str = filename; - if (!HasExtensionOf(filename_str, ORT_TSTR("pb"))) return true; - const std::basic_string file_prefix = - is_input ? ORT_TSTR("input_") : ORT_TSTR("output_"); - if (!filename_str.compare(0, file_prefix.length(), file_prefix)) { - std::basic_string p = ConcatPathComponent(dir_path, filename_str); - test_data_pb_files.push_back(p); - } - return true; - }); + std::filesystem::path dir_fs_path(dir_path); + if (!std::filesystem::exists(dir_fs_path)) return; + + for (auto const& dir_entry : std::filesystem::directory_iterator(dir_fs_path)) { + if (!dir_entry.is_regular_file()) continue; + const std::filesystem::path& path = dir_entry.path(); + if (!path.filename().has_extension()) { + continue; + } + if (path.filename().extension().compare(ORT_TSTR(".pb")) != 0) continue; + const std::basic_string file_prefix = + is_input ? ORT_TSTR("input_") : ORT_TSTR("output_"); + auto filename_str = path.filename().string(); + if (filename_str.compare(0, file_prefix.length(), file_prefix) == 0) { + std::basic_string p = ConcatPathComponent(dir_path, filename_str); + test_data_pb_files.push_back(p); + } + } SortFileNames(test_data_pb_files); @@ -691,11 +702,13 @@ void OnnxTestCase::ConvertTestData(const ONNX_NAMESPACE::OptionalProto& test_dat OnnxTestCase::OnnxTestCase(const std::string& test_case_name, _In_ std::unique_ptr model, double default_per_sample_tolerance, double default_relative_per_sample_tolerance) : test_case_name_(test_case_name), model_info_(std::move(model)) { - std::basic_string test_case_dir = model_info_->GetDir(); - + std::filesystem::path test_case_dir = model_info_->GetDir(); + if (!std::filesystem::exists(test_case_dir)) { + ORT_THROW("test case dir doesn't exist"); + } // parse config - std::basic_string config_path = - ConcatPathComponent(test_case_dir, ORT_TSTR("config.txt")); + std::filesystem::path config_path = + test_case_dir / ORT_TSTR("config.txt"); /* Note: protobuf-lite doesn't support reading protobuf files as text-format. Config.txt is exactly that. That's the reason I've to parse the file in a different way to read the configs. Currently this affects 2 tests - fp16_tiny_yolov2 and fp16_inception_v1. It's not clear why we've to use protobuf @@ -705,7 +718,7 @@ OnnxTestCase::OnnxTestCase(const std::string& test_case_name, _In_ std::unique_p per_sample_tolerance_ = default_per_sample_tolerance; relative_per_sample_tolerance_ = default_relative_per_sample_tolerance; post_processing_ = false; - if (read_config_file(config_path, fc)) { + if (ReadConfigFile(config_path, fc)) { if (fc.count("per_sample_tolerance") > 0) { per_sample_tolerance_ = stod(fc["per_sample_tolerance"]); } @@ -716,16 +729,11 @@ OnnxTestCase::OnnxTestCase(const std::string& test_case_name, _In_ std::unique_p post_processing_ = fc["post_processing"] == "true"; } } - - LoopDir(test_case_dir, [&test_case_dir, this](const PATH_CHAR_TYPE* filename, OrtFileType f_type) -> bool { - if (filename[0] == '.') return true; - if (f_type == OrtFileType::TYPE_DIR) { - std::basic_string p = ConcatPathComponent(test_case_dir, filename); - test_data_dirs_.push_back(p); - debuginfo_strings_.push_back(ToUTF8String(p)); - } - return true; - }); + for (auto const& dir_entry : std::filesystem::directory_iterator(test_case_dir)) { + if (!dir_entry.is_directory()) continue; + test_data_dirs_.push_back(dir_entry.path()); + debuginfo_strings_.push_back(ToUTF8String(dir_entry.path().string())); + } } void LoadTests(const std::vector>& input_paths, @@ -737,20 +745,19 @@ void LoadTests(const std::vector>& input_paths const std::function)>& process_function) { std::vector> paths(input_paths); while (!paths.empty()) { - std::basic_string node_data_root_path = paths.back(); + std::filesystem::path node_data_root_path = paths.back(); paths.pop_back(); - std::basic_string my_dir_name = GetLastComponent(node_data_root_path); - LoopDir(node_data_root_path, [&](const PATH_CHAR_TYPE* filename, OrtFileType f_type) -> bool { - if (filename[0] == '.') return true; - if (f_type == OrtFileType::TYPE_DIR) { - std::basic_string p = ConcatPathComponent(node_data_root_path, filename); - paths.push_back(p); - return true; + if (!std::filesystem::exists(node_data_root_path)) continue; + std::filesystem::path my_dir_name = node_data_root_path.filename(); + for (auto const& dir_entry : std::filesystem::directory_iterator(node_data_root_path)) { + if (dir_entry.is_directory()) { + paths.push_back(dir_entry.path()); + continue; } - - std::basic_string filename_str = filename; - bool is_onnx_format = HasExtensionOf(filename_str, ORT_TSTR("onnx")); - bool is_ort_format = HasExtensionOf(filename_str, ORT_TSTR("ort")); + if (!dir_entry.is_regular_file()) continue; + std::filesystem::path filename_str = dir_entry.path().filename(); + bool is_onnx_format = filename_str.has_extension() && (filename_str.extension().compare(ORT_TSTR(".onnx")) == 0); + bool is_ort_format = filename_str.has_extension() && (filename_str.extension().compare(ORT_TSTR(".ort")) == 0); bool is_valid_model = false; #if !defined(ORT_MINIMAL_BUILD) @@ -759,42 +766,40 @@ void LoadTests(const std::vector>& input_paths is_valid_model = is_valid_model || is_ort_format; if (!is_valid_model) - return true; + continue; - std::basic_string test_case_name = my_dir_name; + std::basic_string test_case_name = my_dir_name.native(); if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) test_case_name = test_case_name.substr(5); if (!whitelisted_test_cases.empty() && std::find(whitelisted_test_cases.begin(), whitelisted_test_cases.end(), test_case_name) == whitelisted_test_cases.end()) { - return true; + continue; } - if (disabled_tests.find(test_case_name) != disabled_tests.end()) return true; - - std::basic_string p = ConcatPathComponent(node_data_root_path, filename_str); + if (disabled_tests.find(test_case_name) != disabled_tests.end()) continue; std::unique_ptr model_info; if (is_onnx_format) { #if !defined(ORT_MINIMAL_BUILD) - model_info = TestModelInfo::LoadOnnxModel(p.c_str()); + model_info = TestModelInfo::LoadOnnxModel(dir_entry.path()); #else ORT_THROW("onnx model is not supported in this build"); #endif } else if (is_ort_format) { - model_info = TestModelInfo::LoadOrtModel(p.c_str()); + model_info = TestModelInfo::LoadOrtModel(dir_entry.path()); } else { ORT_NOT_IMPLEMENTED(ToUTF8String(filename_str), " is not supported"); } auto test_case_dir = model_info->GetDir(); - auto test_case_name_in_log = test_case_name + ORT_TSTR(" in ") + test_case_dir; + auto test_case_name_in_log = test_case_name + ORT_TSTR(" in ") + test_case_dir.native(); #if !defined(ORT_MINIMAL_BUILD) && !defined(USE_QNN) // to skip some models like *-int8 or *-qdq if ((reinterpret_cast(model_info.get()))->HasDomain(ONNX_NAMESPACE::AI_ONNX_TRAINING_DOMAIN) || (reinterpret_cast(model_info.get()))->HasDomain(ONNX_NAMESPACE::AI_ONNX_PREVIEW_TRAINING_DOMAIN)) { fprintf(stderr, "Skip test case:: %s %s\n", ToUTF8String(test_case_name_in_log).c_str(), " as it has training domain"); - return true; + continue; } #endif @@ -809,7 +814,7 @@ void LoadTests(const std::vector>& input_paths }); if (!has_test_data) { fprintf(stderr, "Skip test case:: %s %s\n", ToUTF8String(test_case_name_in_log).c_str(), " due to no test data"); - return true; + continue; } if (broken_tests) { @@ -820,7 +825,7 @@ void LoadTests(const std::vector>& input_paths (opset_version == TestModelInfo::unknown_version || iter->broken_opset_versions_.empty() || iter->broken_opset_versions_.find(opset_version) != iter->broken_opset_versions_.end())) { fprintf(stderr, "Skip test case:: %s %s\n", ToUTF8String(test_case_name_in_log).c_str(), " due to broken_tests"); - return true; + continue; } } @@ -829,7 +834,7 @@ void LoadTests(const std::vector>& input_paths std::string keyword = *iter2; if (ToUTF8String(test_case_name).find(keyword) != std::string::npos) { fprintf(stderr, "Skip test case:: %s %s\n", ToUTF8String(test_case_name_in_log).c_str(), " as it is in broken test keywords"); - return true; + continue; } } } @@ -841,8 +846,7 @@ void LoadTests(const std::vector>& input_paths tolerances.relative(tolerance_key)); fprintf(stdout, "Load Test Case: %s\n", ToUTF8String(test_case_name_in_log).c_str()); process_function(std::move(l)); - return true; - }); + } } } diff --git a/onnxruntime/test/onnx/TestCase.h b/onnxruntime/test/onnx/TestCase.h index 96b0b5f6f7c08..0cb92056d378e 100644 --- a/onnxruntime/test/onnx/TestCase.h +++ b/onnxruntime/test/onnx/TestCase.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ class ITestCase { virtual void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::unordered_map& name_data_map, bool is_input) const = 0; - virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0; + virtual const std::filesystem::path& GetModelUrl() const = 0; virtual const std::string& GetNodeName() const = 0; virtual const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const = 0; virtual const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const = 0; @@ -50,14 +51,9 @@ class ITestCase { class TestModelInfo { public: - virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0; - virtual std::basic_string GetDir() const { - std::basic_string test_case_dir; - auto st = onnxruntime::GetDirNameFromFilePath(GetModelUrl(), test_case_dir); - if (!st.IsOK()) { - ORT_THROW("GetDirNameFromFilePath failed"); - } - return test_case_dir; + virtual const std::filesystem::path& GetModelUrl() const = 0; + virtual std::filesystem::path GetDir() const { + return GetModelUrl().parent_path(); } virtual const std::string& GetNodeName() const = 0; virtual const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const = 0; @@ -70,10 +66,10 @@ class TestModelInfo { virtual ~TestModelInfo() = default; #if !defined(ORT_MINIMAL_BUILD) - static std::unique_ptr LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url); + static std::unique_ptr LoadOnnxModel(const std::filesystem::path& model_url); #endif - static std::unique_ptr LoadOrtModel(_In_ const PATH_CHAR_TYPE* model_url); + static std::unique_ptr LoadOrtModel(const std::filesystem::path& model_url); static const std::string unknown_version; }; diff --git a/onnxruntime/test/onnx/onnx_model_info.cc b/onnxruntime/test/onnx/onnx_model_info.cc index d6afa99382e61..cc90d898acca7 100644 --- a/onnxruntime/test/onnx/onnx_model_info.cc +++ b/onnxruntime/test/onnx/onnx_model_info.cc @@ -14,7 +14,7 @@ using namespace onnxruntime; -OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url, bool is_ort_model) +OnnxModelInfo::OnnxModelInfo(const std::filesystem::path& model_url, bool is_ort_model) : model_url_(model_url) { if (is_ort_model) { InitOrtModelInfo(model_url); @@ -38,7 +38,7 @@ static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField< } } -void OnnxModelInfo::InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { // parse model +void OnnxModelInfo::InitOnnxModelInfo(const std::filesystem::path& model_url) { // parse model int model_fd; auto st = Env::Default().FileOpenRd(model_url, model_fd); if (!st.IsOK()) { @@ -91,7 +91,7 @@ void OnnxModelInfo::InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { / #endif // #if !defined(ORT_MINIMAL_BUILD) -void OnnxModelInfo::InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { +void OnnxModelInfo::InitOrtModelInfo(const std::filesystem::path& model_url) { std::vector bytes; size_t num_bytes = 0; const auto model_location = ToWideString(model_url); diff --git a/onnxruntime/test/onnx/onnx_model_info.h b/onnxruntime/test/onnx/onnx_model_info.h index a0aa27df64a94..48e297376aff5 100644 --- a/onnxruntime/test/onnx/onnx_model_info.h +++ b/onnxruntime/test/onnx/onnx_model_info.h @@ -13,16 +13,16 @@ class OnnxModelInfo : public TestModelInfo { std::vector input_value_info_; std::vector output_value_info_; std::unordered_map domain_to_version_; - const std::basic_string model_url_; + const std::filesystem::path model_url_; #if !defined(ORT_MINIMAL_BUILD) - void InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url); + void InitOnnxModelInfo(const std::filesystem::path& model_url); #endif - void InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url); + void InitOrtModelInfo(const std::filesystem::path& model_url); public: - OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url, bool is_ort_model = false); + OnnxModelInfo(const std::filesystem::path& path, bool is_ort_model = false); bool HasDomain(const std::string& name) const { return domain_to_version_.find(name) != domain_to_version_.end(); } @@ -32,7 +32,7 @@ class OnnxModelInfo : public TestModelInfo { return iter == domain_to_version_.end() ? -1 : iter->second; } - const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); } + const std::filesystem::path& GetModelUrl() const override { return model_url_; } std::string GetNominalOpsetVersion() const override { return onnx_nominal_opset_vesion_; } const std::string& GetNodeName() const override { return node_name_; } diff --git a/onnxruntime/test/onnx/testcase_request.cc b/onnxruntime/test/onnx/testcase_request.cc index 9ca8273ac907b..9d653571ca2ec 100644 --- a/onnxruntime/test/onnx/testcase_request.cc +++ b/onnxruntime/test/onnx/testcase_request.cc @@ -36,7 +36,7 @@ bool TestCaseRequestContext::SetupSession() { ORT_TRY { const auto* test_case_name = test_case_.GetTestCaseName().c_str(); session_opts_.SetLogId(test_case_name); - Ort::Session session{env_, test_case_.GetModelUrl(), session_opts_}; + Ort::Session session{env_, test_case_.GetModelUrl().native().c_str(), session_opts_}; session_ = std::move(session); LOGF_DEFAULT(INFO, "Testing %s\n", test_case_name); return true; diff --git a/onnxruntime/test/perftest/TFModelInfo.cc b/onnxruntime/test/perftest/TFModelInfo.cc deleted file mode 100644 index 82f5359545b4d..0000000000000 --- a/onnxruntime/test/perftest/TFModelInfo.cc +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "TFModelInfo.h" - -#include - -#include - -std::unique_ptr TFModelInfo::Create(_In_ const PATH_CHAR_TYPE* model_url) { - std::unique_ptr model_info = std::make_unique(); - - model_info->model_url_ = model_url; - std::basic_string meta_file_path = model_url; - meta_file_path.append(ORT_TSTR(".meta")); - const onnxruntime::Env& env = onnxruntime::Env::Default(); - size_t len; - auto status = env.GetFileLength(meta_file_path.c_str(), len); - if (!status.IsOK()) { - ORT_THROW(status.ErrorMessage()); - } - std::string file_content; - file_content.resize(len); - auto buffer_span = gsl::make_span(&file_content[0], file_content.size()); - status = onnxruntime::Env::Default().ReadFileIntoBuffer(meta_file_path.c_str(), 0, len, buffer_span); - if (!status.IsOK()) { - ORT_THROW(status.ErrorMessage()); - } - // this string is not null terminated - std::istringstream is{file_content}; - - std::string line; - while (std::getline(is, line)) { - size_t line_len = 0; - if (!line.empty() && line.back() == '\n') { - line_len = line.length() - 1; - if (line_len > 0 && line[line_len - 1] == '\r') { - --line_len; - } - line.resize(line_len); - } - if (line.empty()) continue; - if (line.compare(0, 6, "input=") == 0) { - model_info->input_names_.push_back(line.substr(6)); - } else if (line.compare(0, 7, "output=") == 0) { - model_info->output_names_.push_back(line.substr(7)); - } else { - ORT_THROW("unknown line:", line.size()); - } - } - - return model_info; -} - -int TFModelInfo::GetInputCount() const { return static_cast(input_names_.size()); } -int TFModelInfo::GetOutputCount() const { return static_cast(output_names_.size()); } -const std::string& TFModelInfo::GetInputName(size_t i) const { return input_names_[i]; } -const std::string& TFModelInfo::GetOutputName(size_t i) const { return output_names_[i]; } diff --git a/onnxruntime/test/perftest/TFModelInfo.h b/onnxruntime/test/perftest/TFModelInfo.h deleted file mode 100644 index 2ca60010e300b..0000000000000 --- a/onnxruntime/test/perftest/TFModelInfo.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "TestCase.h" -#include -#include - -class TFModelInfo : public TestModelInfo { - public: - const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); } - - const std::string& GetNodeName() const override { return node_name_; } - const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t) const override { return nullptr; } - const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t) const override { return nullptr; } - - int GetInputCount() const override; - int GetOutputCount() const override; - const std::string& GetInputName(size_t i) const override; - const std::string& GetOutputName(size_t i) const override; - ~TFModelInfo() override = default; - - static std::unique_ptr Create(_In_ const PATH_CHAR_TYPE* model_url); - TFModelInfo() = default; - - private: - std::basic_string model_url_; - std::vector input_names_; - std::vector output_names_; - std::string node_name_; -}; diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index 9f2cbcf6a21f1..995ae8bbc5e79 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -10,7 +10,6 @@ #include #include "TestCase.h" -#include "TFModelInfo.h" #include "utils.h" #include "ort_test_session.h" #ifdef HAVE_TENSORFLOW @@ -270,10 +269,6 @@ static std::unique_ptr CreateModelInfo(const PerformanceTestConfi ORT_NOT_IMPLEMENTED(ToUTF8String(file_path), " is not supported"); } - if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("tf")) == 0) { - return TFModelInfo::Create(performance_test_config_.model_info.model_file_path.c_str()); - } - ORT_NOT_IMPLEMENTED(ToUTF8String(performance_test_config_.backend), " is not supported"); } diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 745c677c4eed4..79b8b90c96abf 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -48,7 +48,6 @@ #include "test/compare_ortvalue.h" #include "test/onnx/heap_buffer.h" #include "test/onnx/onnx_model_info.h" -#include "test/onnx/callback.h" #include "test/onnx/testcase_request.h" extern std::unique_ptr ort_env; From 0c7f1b0d583190b6e08aa53639ce22acad221f6b Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 17 Jan 2024 17:12:36 -0800 Subject: [PATCH 03/27] Fix build --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 1034a82cb2854..a94ad8b8fe855 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1472,7 +1472,7 @@ def generate_build_tree( ): if is_windows(): cflags = ["/guard:cf", "/DWIN32", "/D_WINDOWS"] - if args.parallel: + if number_of_parallel_jobs(args) > 1: cflags += ["/MP"] if not args.use_gdk: # Target Windows 10 From cc89bbce065e03ecde2854e5634e2b24ab666c9e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 17 Jan 2024 17:12:43 -0800 Subject: [PATCH 04/27] stash --- onnxruntime/test/providers/cpu/model_tests.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 79b8b90c96abf..c212cfba25bac 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -699,7 +699,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { continue; } if (path.filename().extension().compare(ORT_TSTR(".onnx")) != 0) continue; - std::basic_string test_case_name = path.parent_path().filename().string(); + std::basic_string test_case_name = path.parent_path().filename().native(); if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) test_case_name = test_case_name.substr(5); if (all_disabled_tests.find(test_case_name) != all_disabled_tests.end()) @@ -716,7 +716,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { } #endif std::basic_ostringstream oss; - oss << provider_name << ORT_TSTR("_") << path.string(); + oss << provider_name << ORT_TSTR("_") << path.native(); v.emplace_back(oss.str()); } } From 6cb7e0b22e5c503d91e0c4834ee19950a8c8b356 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 18 Jan 2024 13:50:26 -0800 Subject: [PATCH 05/27] update --- onnxruntime/test/onnx/TestCase.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index ae2aa96253d13..987e33e92cf88 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -511,8 +511,8 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::vector test_data_pb_files; - const PATH_STRING_TYPE& dir_path = test_data_dirs_[id]; - std::filesystem::path dir_fs_path(dir_path); + + std::filesystem::path dir_fs_path = test_data_dirs_[id]; if (!std::filesystem::exists(dir_fs_path)) return; for (auto const& dir_entry : std::filesystem::directory_iterator(dir_fs_path)) { @@ -524,10 +524,9 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, if (path.filename().extension().compare(ORT_TSTR(".pb")) != 0) continue; const std::basic_string file_prefix = is_input ? ORT_TSTR("input_") : ORT_TSTR("output_"); - auto filename_str = path.filename().string(); + auto filename_str = path.filename().native(); if (filename_str.compare(0, file_prefix.length(), file_prefix) == 0) { - std::basic_string p = ConcatPathComponent(dir_path, filename_str); - test_data_pb_files.push_back(p); + test_data_pb_files.push_back(path.native()); } } From bea81a63aa6613acc86eb566185a71fcd7052ed8 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 18 Jan 2024 15:04:38 -0800 Subject: [PATCH 06/27] update --- onnxruntime/test/unittest_main/test_main.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/onnxruntime/test/unittest_main/test_main.cc b/onnxruntime/test/unittest_main/test_main.cc index 97169df36fdd7..4a2e9f283b593 100644 --- a/onnxruntime/test/unittest_main/test_main.cc +++ b/onnxruntime/test/unittest_main/test_main.cc @@ -57,11 +57,12 @@ auto const placeholder = std::unique_ptr(nvinfer1::createInf int TEST_MAIN(int argc, char** argv) { int status = 0; - + std::cout<<"Init Google Test"<(kLogLevelEnvironmentVariableName); @@ -72,7 +73,7 @@ int TEST_MAIN(int argc, char** argv) { std::cout << "Setting log level to " << *log_level << "\n"; ort_env->UpdateEnvWithCustomLogLevel(static_cast(*log_level)); } - + std::cout<<"Start running tests"< Date: Fri, 19 Jan 2024 09:45:38 -0800 Subject: [PATCH 07/27] update --- cmake/CMakeLists.txt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 7d7304630c00e..7a45341717d65 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1245,17 +1245,15 @@ if (onnxruntime_USE_TVM) $) set(onnxruntime_tvm_libs onnxruntime_providers_tvm) - - # needs to link with stdc++fs in Linux - if (UNIX) - if (NOT APPLE) - set(FS_STDLIB stdc++fs) - endif() - endif() - list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm ${FS_STDLIB}) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm) list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES tvm) endif() +# needs to link with stdc++fs in Linux +if (UNIX AND "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9) + set(FS_STDLIB stdc++fs) +endif() +list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${FS_STDLIB}) # onnxruntime-extensions if (onnxruntime_USE_EXTENSIONS) From bf29599c2fb81c2ae80e7114e785453302c974d3 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 19 Jan 2024 11:09:36 -0800 Subject: [PATCH 08/27] push --- cmake/onnxruntime_unittests.cmake | 124 +++++++++--------- onnxruntime/test/providers/cpu/model_tests.cc | 9 +- 2 files changed, 72 insertions(+), 61 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index fa395802d95ff..7591e4b7105bf 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -743,34 +743,36 @@ target_include_directories(onnxruntime_test_utils PUBLIC "${TEST_SRC_DIR}/util/i set_target_properties(onnxruntime_test_utils PROPERTIES FOLDER "ONNXRuntimeTest") source_group(TREE ${TEST_SRC_DIR} FILES ${onnxruntime_test_utils_src}) -set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx) -file(GLOB onnx_test_runner_common_srcs CONFIGURE_DEPENDS - ${onnx_test_runner_src_dir}/*.h - ${onnx_test_runner_src_dir}/*.cc) +if(NOT IOS) + set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx) + file(GLOB onnx_test_runner_common_srcs CONFIGURE_DEPENDS + ${onnx_test_runner_src_dir}/*.h + ${onnx_test_runner_src_dir}/*.cc) -list(REMOVE_ITEM onnx_test_runner_common_srcs ${onnx_test_runner_src_dir}/main.cc) + list(REMOVE_ITEM onnx_test_runner_common_srcs ${onnx_test_runner_src_dir}/main.cc) -onnxruntime_add_static_library(onnx_test_runner_common ${onnx_test_runner_common_srcs}) -if(MSVC) - target_compile_options(onnx_test_runner_common PRIVATE "$<$:SHELL:--compiler-options /utf-8>" - "$<$>:/utf-8>") -else() - target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11) - target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) - onnxruntime_add_include_to_target(onnx_test_runner_common nsync::nsync_cpp) -endif() -if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) - #TODO: fix the warnings, they are dangerous - target_compile_options(onnx_test_runner_common PRIVATE "/wd4244") -endif() -onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework - onnxruntime_test_utils onnx onnx_proto re2::re2 flatbuffers::flatbuffers Boost::mp11 safeint_interface) + onnxruntime_add_static_library(onnx_test_runner_common ${onnx_test_runner_common_srcs}) + if(MSVC) + target_compile_options(onnx_test_runner_common PRIVATE "$<$:SHELL:--compiler-options /utf-8>" + "$<$>:/utf-8>") + else() + target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11) + target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) + onnxruntime_add_include_to_target(onnx_test_runner_common nsync::nsync_cpp) + endif() + if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + #TODO: fix the warnings, they are dangerous + target_compile_options(onnx_test_runner_common PRIVATE "/wd4244") + endif() + onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework + onnxruntime_test_utils onnx onnx_proto re2::re2 flatbuffers::flatbuffers Boost::mp11 safeint_interface) -add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) -target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) + add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) + target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} + ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) -set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest") + set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest") +endif() set(all_tests ${onnxruntime_test_common_src} ${onnxruntime_test_ir_src} ${onnxruntime_test_optimizer_src} ${onnxruntime_test_framework_src} ${onnxruntime_test_providers_src} ${onnxruntime_test_quantiztion_src}) @@ -1052,45 +1054,47 @@ if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) list(APPEND onnx_test_libs onnxruntime_language_interop onnxruntime_pyop) endif() -onnxruntime_add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) -if(MSVC) - target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" - "$<$>:/utf-8>") -endif() -if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") - set_target_properties(onnx_test_runner PROPERTIES - XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" - ) -endif() -if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) - set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") - else() - set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1") - endif() -endif() +if (NOT IOS) + onnxruntime_add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) + if(MSVC) + target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" + "$<$>:/utf-8>") + endif() + if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + set_target_properties(onnx_test_runner PROPERTIES + XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" + ) + endif() + if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") + else() + set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1") + endif() + endif() -target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs} nlohmann_json::nlohmann_json) -target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) -if (onnxruntime_USE_ROCM) - target_include_directories(onnx_test_runner PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) -endif() -if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) - target_link_libraries(onnx_test_runner PRIVATE Python::Python) -endif() -set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") + target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs} nlohmann_json::nlohmann_json) + target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) + if (onnxruntime_USE_ROCM) + target_include_directories(onnx_test_runner PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) + endif() + if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) + target_link_libraries(onnx_test_runner PRIVATE Python::Python) + endif() + set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") -if (onnxruntime_USE_TVM) - if (WIN32) - target_link_options(onnx_test_runner PRIVATE "/STACK:4000000") - endif() -endif() + if (onnxruntime_USE_TVM) + if (WIN32) + target_link_options(onnx_test_runner PRIVATE "/STACK:4000000") + endif() + endif() -install(TARGETS onnx_test_runner - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - BUNDLE DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + install(TARGETS onnx_test_runner + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + BUNDLE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) if(onnxruntime_BUILD_BENCHMARKS) diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index c212cfba25bac..b27e6291d6ee3 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -698,7 +698,14 @@ ::std::vector<::std::basic_string> GetParameterStrings() { if (!path.filename().has_extension()) { continue; } - if (path.filename().extension().compare(ORT_TSTR(".onnx")) != 0) continue; + if (path.filename().native().empty() || path.filename().native().compare(0, 1, ORT_TSTR(".")) == 0) { + // Ignore hidden files. + continue; + } + if (path.filename().extension().compare(ORT_TSTR(".onnx")) != 0) { + // Ignore the files that are not ONNX models + continue; + } std::basic_string test_case_name = path.parent_path().filename().native(); if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) test_case_name = test_case_name.substr(5); From c1574c726f15eb42c67645805a1bbdb26894085d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 19 Jan 2024 12:44:44 -0800 Subject: [PATCH 09/27] update --- cmake/onnxruntime_unittests.cmake | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 7591e4b7105bf..1c15ad6b03d54 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -772,6 +772,7 @@ if(NOT IOS) ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest") + set(onnx_test_runner_common_lib onnx_test_runner_common) endif() set(all_tests ${onnxruntime_test_common_src} ${onnxruntime_test_ir_src} ${onnxruntime_test_optimizer_src} @@ -828,6 +829,12 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") endif() endif() +if (IOS OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + list(REMOVE_ITEM all_tests + "${TEST_SRC_DIR}/providers/cpu/providers/cpu/model_tests.cc" + ) +endif() + set(test_all_args) if (onnxruntime_USE_TENSORRT) # TRT EP CI takes much longer time when updating to TRT 8.2 @@ -845,7 +852,7 @@ AddTest( TARGET onnxruntime_test_all SOURCES ${all_tests} ${onnxruntime_unittest_main_src} LIBS - onnx_test_runner_common ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} + ${onnx_test_runner_common_lib} ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} onnx_test_data_proto DEPENDS ${all_dependencies} TEST_ARGS ${test_all_args} From 2bd4e4e415213640721700aa968ab21ae30362e9 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 24 Jan 2024 09:15:25 -0800 Subject: [PATCH 10/27] update --- winml/test/model/model_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winml/test/model/model_tests.cpp b/winml/test/model/model_tests.cpp index f40f08ad2696d..27d74d7d6b034 100644 --- a/winml/test/model/model_tests.cpp +++ b/winml/test/model/model_tests.cpp @@ -118,7 +118,7 @@ TEST_P(ModelTest, Run) { LearningModelDevice device = nullptr; LearningModelSession session = nullptr; LearningModelBinding binding = nullptr; - WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(m_testCase->GetModelUrl())); + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(m_testCase->GetModelUrl().native())); WINML_EXPECT_NO_THROW(device = LearningModelDevice(m_deviceKind)); WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device)); for (size_t i = 0; i < m_testCase->GetDataCount(); i++) { From fecd8b085687ea337a713ee61025def3bf7f0f16 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 24 Jan 2024 09:16:02 -0800 Subject: [PATCH 11/27] revert --- onnxruntime/test/unittest_main/test_main.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/onnxruntime/test/unittest_main/test_main.cc b/onnxruntime/test/unittest_main/test_main.cc index 4a2e9f283b593..97169df36fdd7 100644 --- a/onnxruntime/test/unittest_main/test_main.cc +++ b/onnxruntime/test/unittest_main/test_main.cc @@ -57,12 +57,11 @@ auto const placeholder = std::unique_ptr(nvinfer1::createInf int TEST_MAIN(int argc, char** argv) { int status = 0; - std::cout<<"Init Google Test"<(kLogLevelEnvironmentVariableName); @@ -73,7 +72,7 @@ int TEST_MAIN(int argc, char** argv) { std::cout << "Setting log level to " << *log_level << "\n"; ort_env->UpdateEnvWithCustomLogLevel(static_cast(*log_level)); } - std::cout<<"Start running tests"< Date: Wed, 24 Jan 2024 09:18:15 -0800 Subject: [PATCH 12/27] revert --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 8f044d3242b32..6e5cd7b57e403 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1472,7 +1472,7 @@ def generate_build_tree( ): if is_windows(): cflags = ["/guard:cf", "/DWIN32", "/D_WINDOWS"] - if number_of_parallel_jobs(args) > 1: + if args.parallel: cflags += ["/MP"] if not args.use_gdk: # Target Windows 10 From 82cb246ad185f7f655bc3b7c6974e2b47ab19972 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 24 Jan 2024 09:21:30 -0800 Subject: [PATCH 13/27] update --- cmake/onnxruntime_unittests.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 069f210294bd3..2e55adeea776c 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -829,7 +829,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") endif() endif() -if (IOS OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten") +if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR CMAKE_SYSTEM_NAME STREQUAL "iOS") + message("Disable model tests in onnxruntime_test_all") list(REMOVE_ITEM all_tests "${TEST_SRC_DIR}/providers/cpu/providers/cpu/model_tests.cc" ) From 2fc154393a36c40259d1b197fd0264e2f1e9190e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 24 Jan 2024 11:23:11 -0800 Subject: [PATCH 14/27] format code --- onnxruntime/test/onnx/TestCase.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 987e33e92cf88..781671e610866 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -511,7 +511,6 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::vector test_data_pb_files; - std::filesystem::path dir_fs_path = test_data_dirs_[id]; if (!std::filesystem::exists(dir_fs_path)) return; From fef0732b27942782c2b0a0e7d575c9a88ac692a1 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 26 Feb 2024 12:05:19 -0800 Subject: [PATCH 15/27] Enable CPUINFO --- cmake/external/onnxruntime_external_deps.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 22d12b128dc1f..85307a022fc8a 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -260,9 +260,6 @@ if (onnxruntime_ENABLE_CPUINFO) if (${onnxruntime_target_platform} MATCHES "^(ARM.*|arm.*)$" ) message(WARNING "Cpuinfo not included for compilation problems with Windows ARM.") set(CPUINFO_SUPPORTED FALSE) - elseif (WIN32 AND NOT CMAKE_CXX_STANDARD_LIBRARIES MATCHES kernel32.lib) - message(WARNING "Cpuinfo not included non-Desktop builds") - set(CPUINFO_SUPPORTED FALSE) endif() elseif (NOT ${onnxruntime_target_platform} MATCHES "^(i[3-6]86|AMD64|x86(_64)?|armv[5-8].*|aarch64|arm64)$") message(WARNING From 05947cb3a4589087a142d845796c1a16912f467e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 29 Feb 2024 16:46:56 -0800 Subject: [PATCH 16/27] update --- onnxruntime/test/onnx/TestCase.cc | 4 ++++ onnxruntime/test/onnx/onnx_model_info.cc | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 781671e610866..09c8ae213bad2 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -754,6 +754,10 @@ void LoadTests(const std::vector>& input_paths } if (!dir_entry.is_regular_file()) continue; std::filesystem::path filename_str = dir_entry.path().filename(); + if (filename_str.empty() || filename_str.native()[0] == ORT_TSTR('.')) { + // Ignore hidden files. + continue; + } bool is_onnx_format = filename_str.has_extension() && (filename_str.extension().compare(ORT_TSTR(".onnx")) == 0); bool is_ort_format = filename_str.has_extension() && (filename_str.extension().compare(ORT_TSTR(".ort")) == 0); bool is_valid_model = false; diff --git a/onnxruntime/test/onnx/onnx_model_info.cc b/onnxruntime/test/onnx/onnx_model_info.cc index cc90d898acca7..f23012aee9fd2 100644 --- a/onnxruntime/test/onnx/onnx_model_info.cc +++ b/onnxruntime/test/onnx/onnx_model_info.cc @@ -50,7 +50,9 @@ void OnnxModelInfo::InitOnnxModelInfo(const std::filesystem::path& model_url) { const bool parse_result = model_pb.ParseFromZeroCopyStream(&input) && input.GetErrno() == 0; if (!parse_result) { (void)Env::Default().FileClose(model_fd); - ORT_THROW("Failed to load model because protobuf parsing failed."); + std::ostringstream oss; + oss << "Failed to load model from " << model_url << " because protobuf parsing failed."; + ORT_THROW(oss.str()); } (void)Env::Default().FileClose(model_fd); { From 6b3ba1227ee52dde128b734d68a50c68ea42f6c2 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 29 Feb 2024 20:50:14 -0800 Subject: [PATCH 17/27] update --- cmake/onnxruntime_unittests.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 9ecaa1178a26a..58ac8412438b1 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -825,7 +825,7 @@ endif() if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR CMAKE_SYSTEM_NAME STREQUAL "iOS") message("Disable model tests in onnxruntime_test_all") list(REMOVE_ITEM all_tests - "${TEST_SRC_DIR}/providers/cpu/providers/cpu/model_tests.cc" + "${TEST_SRC_DIR}/providers/cpu/model_tests.cc" ) endif() From 68d759d7534f0c78031f80b1e5f15fecf2d7d49d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 4 Mar 2024 09:31:57 -0800 Subject: [PATCH 18/27] update --- cmake/CMakeLists.txt | 1 - cmake/onnxruntime_unittests.cmake | 13 +------ .../test/perftest/performance_runner.cc | 35 ++++--------------- .../test/perftest/test_configuration.h | 1 - 4 files changed, 8 insertions(+), 42 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 09e79e0cd85e9..c40fe544c5b27 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -118,7 +118,6 @@ option(onnxruntime_GCOV_COVERAGE "Compile with options necessary to run code cov option(onnxruntime_DONT_VECTORIZE "Do not vectorize operations in Eigen" OFF) option(onnxruntime_USE_FULL_PROTOBUF "Link to libprotobuf instead of libprotobuf-lite when this option is ON" OFF) -option(tensorflow_C_PACKAGE_PATH "Path to tensorflow C package installation dir") option(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS "Enable operator implemented in language other than cpp" OFF) option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS "Dump debug information about node inputs and outputs when executing the model." OFF) cmake_dependent_option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS_ENABLE_DUMP_TO_SQLDB "Build dump debug information about node inputs and outputs with support for sql database." OFF "onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS" OFF) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 58ac8412438b1..215014425de59 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1175,7 +1175,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) endif() -if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) +if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP AND NOT ${CMAKE_SYSTEM_NAME} STREQUAL "iOS") #perf test runner set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) set(onnxruntime_perf_test_src_patterns @@ -1212,11 +1212,6 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) set(SYS_PATH_LIB shlwapi) endif() endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") - set_target_properties(onnxruntime_perf_test PROPERTIES - XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" - ) - endif() if (onnxruntime_BUILD_SHARED_LIB) #It will dynamically link to onnxruntime. So please don't add onxruntime_graph/onxruntime_framework/... here. @@ -1239,12 +1234,6 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) if(WIN32) target_link_libraries(onnxruntime_perf_test PRIVATE debug dbghelp advapi32) endif() - if(tensorflow_C_PACKAGE_PATH) - target_include_directories(onnxruntime_perf_test PRIVATE ${tensorflow_C_PACKAGE_PATH}/include) - target_link_directories(onnxruntime_perf_test PRIVATE ${tensorflow_C_PACKAGE_PATH}/lib) - target_link_libraries(onnxruntime_perf_test PRIVATE tensorflow) - target_compile_definitions(onnxruntime_perf_test PRIVATE HAVE_TENSORFLOW) - endif() else() target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) endif() diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index 78e5647346c1d..3e715642b7914 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -12,9 +12,6 @@ #include "TestCase.h" #include "utils.h" #include "ort_test_session.h" -#ifdef HAVE_TENSORFLOW -#include "tf_test_session.h" -#endif using onnxruntime::Status; // TODO: Temporary, while we bring up the threadpool impl... @@ -259,43 +256,25 @@ Status PerformanceRunner::ForkJoinRepeat() { } static std::unique_ptr CreateModelInfo(const PerformanceTestConfig& performance_test_config_) { - if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("ort")) == 0) { const auto& file_path = performance_test_config_.model_info.model_file_path; #if !defined(ORT_MINIMAL_BUILD) - if (HasExtensionOf(file_path, ORT_TSTR("onnx"))) { - return TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str()); - } + if (HasExtensionOf(file_path, ORT_TSTR("onnx"))) { + return TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str()); + } #endif - if (HasExtensionOf(file_path, ORT_TSTR("ort"))) { - return TestModelInfo::LoadOrtModel(performance_test_config_.model_info.model_file_path.c_str()); - } - - ORT_NOT_IMPLEMENTED(ToUTF8String(file_path), " is not supported"); + if (HasExtensionOf(file_path, ORT_TSTR("ort"))) { + return TestModelInfo::LoadOrtModel(performance_test_config_.model_info.model_file_path.c_str()); } - ORT_NOT_IMPLEMENTED(ToUTF8String(performance_test_config_.backend), " is not supported"); -} - -static std::unique_ptr CreateSession(Ort::Env& env, std::random_device& rd, - const PerformanceTestConfig& performance_test_config_, - const TestModelInfo& test_model_info) { - if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("ort")) == 0) { - return std::make_unique(env, rd, performance_test_config_, test_model_info); - } -#ifdef HAVE_TENSORFLOW - if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("tf")) == 0) { - return new TensorflowTestSession(rd, performance_test_config_, test_model_info); - } -#endif - ORT_NOT_IMPLEMENTED(ToUTF8String(performance_test_config_.backend), " is not supported"); + ORT_NOT_IMPLEMENTED(ToUTF8String(file_path), " is not supported"); } PerformanceRunner::PerformanceRunner(Ort::Env& env, const PerformanceTestConfig& test_config, std::random_device& rd) : performance_test_config_(test_config), test_model_info_(CreateModelInfo(test_config)) { session_create_start_ = std::chrono::high_resolution_clock::now(); - session_ = CreateSession(env, rd, test_config, *test_model_info_); + session_ = std::make_unique(env, rd, performance_test_config_, test_model_info); session_create_end_ = std::chrono::high_resolution_clock::now(); } diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index 74c8eb472cb3e..70a6b12690d5d 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -70,7 +70,6 @@ struct PerformanceTestConfig { ModelInfo model_info; MachineConfig machine_config; RunConfig run_config; - std::basic_string backend = ORT_TSTR("ort"); }; } // namespace perftest From 74b3e3345e5b36ee39d36ecdbc75a39ee314d5de Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 4 Mar 2024 09:35:13 -0800 Subject: [PATCH 19/27] update --- onnxruntime/test/perftest/performance_runner.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index 3e715642b7914..c2ff56292aaf5 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -256,7 +256,7 @@ Status PerformanceRunner::ForkJoinRepeat() { } static std::unique_ptr CreateModelInfo(const PerformanceTestConfig& performance_test_config_) { - const auto& file_path = performance_test_config_.model_info.model_file_path; + const auto& file_path = performance_test_config_.model_info.model_file_path; #if !defined(ORT_MINIMAL_BUILD) if (HasExtensionOf(file_path, ORT_TSTR("onnx"))) { return TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str()); From b0a9bcfe26c899e5a36f7888cfe62412c9bd0e84 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 4 Mar 2024 11:46:38 -0800 Subject: [PATCH 20/27] update --- onnxruntime/test/perftest/performance_runner.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index 3e715642b7914..50dfee6de6c7e 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -274,7 +274,7 @@ PerformanceRunner::PerformanceRunner(Ort::Env& env, const PerformanceTestConfig& : performance_test_config_(test_config), test_model_info_(CreateModelInfo(test_config)) { session_create_start_ = std::chrono::high_resolution_clock::now(); - session_ = std::make_unique(env, rd, performance_test_config_, test_model_info); + session_ = std::make_unique(env, rd, performance_test_config_, test_model_info_); session_create_end_ = std::chrono::high_resolution_clock::now(); } From f6540094ec5ef9381af70703b83577e121ce7aa1 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 4 Mar 2024 12:43:48 -0800 Subject: [PATCH 21/27] update --- onnxruntime/test/perftest/command_args_parser.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 729ad34368453..16c90c39f300f 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -199,7 +199,7 @@ static bool ParseSessionConfigs(const std::string& configs_string, /*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) { int ch; - while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:C:AMPIDZvhsqzn"))) != -1) { + while ((ch = getopt(argc, argv, ORT_TSTR("m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:C:AMPIDZvhsqzn"))) != -1) { switch (ch) { case 'f': { std::basic_string dim_name; @@ -228,9 +228,6 @@ static bool ParseSessionConfigs(const std::string& configs_string, return false; } break; - case 'b': - test_config.backend = optarg; - break; case 'p': test_config.run_config.profile_file = optarg; break; From 210cd1bbd6a77a5908816b15d5a87a992ff743ed Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 4 Mar 2024 13:17:13 -0800 Subject: [PATCH 22/27] update --- onnxruntime/test/perftest/performance_runner.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index b4386fa3e231a..08d77008dc25c 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -274,7 +274,7 @@ PerformanceRunner::PerformanceRunner(Ort::Env& env, const PerformanceTestConfig& : performance_test_config_(test_config), test_model_info_(CreateModelInfo(test_config)) { session_create_start_ = std::chrono::high_resolution_clock::now(); - session_ = std::make_unique(env, rd, performance_test_config_, test_model_info_); + session_ = std::make_unique(env, rd, performance_test_config_, *test_model_info_); session_create_end_ = std::chrono::high_resolution_clock::now(); } From cc23da1c42ba4f8c14aae4053f705e2a3d9521a3 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 4 Mar 2024 14:44:13 -0800 Subject: [PATCH 23/27] update --- cmake/onnxruntime_unittests.cmake | 115 +++++++++++++++--------------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 215014425de59..752e23818933c 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1175,80 +1175,81 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) endif() -if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP AND NOT ${CMAKE_SYSTEM_NAME} STREQUAL "iOS") - #perf test runner - set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) - set(onnxruntime_perf_test_src_patterns - "${onnxruntime_perf_test_src_dir}/*.cc" - "${onnxruntime_perf_test_src_dir}/*.h") +if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) + if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + #perf test runner + set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) + set(onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/*.cc" + "${onnxruntime_perf_test_src_dir}/*.h") - if(WIN32) - list(APPEND onnxruntime_perf_test_src_patterns - "${onnxruntime_perf_test_src_dir}/windows/*.cc" - "${onnxruntime_perf_test_src_dir}/windows/*.h" ) - else () - list(APPEND onnxruntime_perf_test_src_patterns - "${onnxruntime_perf_test_src_dir}/posix/*.cc" - "${onnxruntime_perf_test_src_dir}/posix/*.h" ) - endif() + if(WIN32) + list(APPEND onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/windows/*.cc" + "${onnxruntime_perf_test_src_dir}/windows/*.h" ) + else () + list(APPEND onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/posix/*.cc" + "${onnxruntime_perf_test_src_dir}/posix/*.h" ) + endif() - file(GLOB onnxruntime_perf_test_src CONFIGURE_DEPENDS - ${onnxruntime_perf_test_src_patterns} - ) - onnxruntime_add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src} ${ONNXRUNTIME_ROOT}/core/platform/path_lib.cc) - if(MSVC) - target_compile_options(onnxruntime_perf_test PRIVATE "$<$:SHELL:--compiler-options /utf-8>" + file(GLOB onnxruntime_perf_test_src CONFIGURE_DEPENDS + ${onnxruntime_perf_test_src_patterns} + ) + onnxruntime_add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src} ${ONNXRUNTIME_ROOT}/core/platform/path_lib.cc) + if(MSVC) + target_compile_options(onnxruntime_perf_test PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") - endif() - target_include_directories(onnxruntime_perf_test PRIVATE ${onnx_test_runner_src_dir} ${ONNXRUNTIME_ROOT} + endif() + target_include_directories(onnxruntime_perf_test PRIVATE ${onnx_test_runner_src_dir} ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR}) - if (onnxruntime_USE_ROCM) - target_include_directories(onnxruntime_perf_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) - endif() - if (WIN32) - target_compile_options(onnxruntime_perf_test PRIVATE ${disabled_warnings}) - if (NOT DEFINED SYS_PATH_LIB) - set(SYS_PATH_LIB shlwapi) + if (onnxruntime_USE_ROCM) + target_include_directories(onnxruntime_perf_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) + endif() + if (WIN32) + target_compile_options(onnxruntime_perf_test PRIVATE ${disabled_warnings}) + if (NOT DEFINED SYS_PATH_LIB) + set(SYS_PATH_LIB shlwapi) + endif() endif() - endif() - if (onnxruntime_BUILD_SHARED_LIB) - #It will dynamically link to onnxruntime. So please don't add onxruntime_graph/onxruntime_framework/... here. - #onnxruntime_common is kind of ok because it is thin, tiny and totally stateless. - set(onnxruntime_perf_test_libs + if (onnxruntime_BUILD_SHARED_LIB) + #It will dynamically link to onnxruntime. So please don't add onxruntime_graph/onxruntime_framework/... here. + #onnxruntime_common is kind of ok because it is thin, tiny and totally stateless. + set(onnxruntime_perf_test_libs onnx_test_runner_common onnxruntime_test_utils onnxruntime_common onnxruntime onnxruntime_flatbuffers onnx_test_data_proto ${onnxruntime_EXTERNAL_LIBRARIES} ${GETOPT_LIB_WIDE} ${SYS_PATH_LIB} ${CMAKE_DL_LIBS}) - if(NOT WIN32) - list(APPEND onnxruntime_perf_test_libs nsync::nsync_cpp) - if(onnxruntime_USE_SNPE) - list(APPEND onnxruntime_perf_test_libs onnxruntime_providers_snpe) + if(NOT WIN32) + list(APPEND onnxruntime_perf_test_libs nsync::nsync_cpp) + if(onnxruntime_USE_SNPE) + list(APPEND onnxruntime_perf_test_libs onnxruntime_providers_snpe) + endif() endif() + if (CMAKE_SYSTEM_NAME STREQUAL "Android") + list(APPEND onnxruntime_perf_test_libs ${android_shared_libs}) + endif() + target_link_libraries(onnxruntime_perf_test PRIVATE ${onnxruntime_perf_test_libs} Threads::Threads) + if(WIN32) + target_link_libraries(onnxruntime_perf_test PRIVATE debug dbghelp advapi32) + endif() + else() + target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) endif() - if (CMAKE_SYSTEM_NAME STREQUAL "Android") - list(APPEND onnxruntime_perf_test_libs ${android_shared_libs}) - endif() - target_link_libraries(onnxruntime_perf_test PRIVATE ${onnxruntime_perf_test_libs} Threads::Threads) - if(WIN32) - target_link_libraries(onnxruntime_perf_test PRIVATE debug dbghelp advapi32) - endif() - else() - target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) - endif() - set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest") + set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest") - if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS AND NOT onnxruntime_BUILD_SHARED_LIB) - target_link_libraries(onnxruntime_perf_test PRIVATE onnxruntime_language_interop onnxruntime_pyop) - endif() + if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS AND NOT onnxruntime_BUILD_SHARED_LIB) + target_link_libraries(onnxruntime_perf_test PRIVATE onnxruntime_language_interop onnxruntime_pyop) + endif() - if (onnxruntime_USE_TVM) - if (WIN32) - target_link_options(onnxruntime_perf_test PRIVATE "/STACK:4000000") + if (onnxruntime_USE_TVM) + if (WIN32) + target_link_options(onnxruntime_perf_test PRIVATE "/STACK:4000000") + endif() endif() endif() - # shared lib if (onnxruntime_BUILD_SHARED_LIB) onnxruntime_add_static_library(onnxruntime_mocked_allocator ${TEST_SRC_DIR}/util/test_allocator.cc) From 01c00c592031b905d80be1ee7d994b5e55108ea9 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 5 Mar 2024 16:24:54 -0800 Subject: [PATCH 24/27] Update onnxruntime/test/providers/cpu/model_tests.cc Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- onnxruntime/test/providers/cpu/model_tests.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 51daabcf649a0..7461717377144 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -712,10 +712,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { continue; } const std::filesystem::path& path = dir_entry.path(); - if (!path.filename().has_extension()) { - continue; - } - if (path.filename().native().empty() || path.filename().native().compare(0, 1, ORT_TSTR(".")) == 0) { + if (!path.has_filename() || path.filename().native().compare(0, 1, ORT_TSTR(".")) == 0) { // Ignore hidden files. continue; } From e0c9d868627ca625cdb0abfbee7708848c6e1f91 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 5 Mar 2024 16:25:06 -0800 Subject: [PATCH 25/27] update --- cmake/onnxruntime_unittests.cmake | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 752e23818933c..724f72954e7fe 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -822,7 +822,9 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") "${TEST_SRC_DIR}/providers/cpu/tensor/grid_sample_test.cc") endif() -if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR CMAKE_SYSTEM_NAME STREQUAL "iOS") +if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR IOS) + # Because we do not run these model tests in our web or iOS CI build pipelines, and some test code uses C++17 + # filesystem functions that are not available in the iOS version we target. message("Disable model tests in onnxruntime_test_all") list(REMOVE_ITEM all_tests "${TEST_SRC_DIR}/providers/cpu/model_tests.cc" @@ -884,7 +886,7 @@ endif() # the default logger tests conflict with the need to have an overall default logger # so skip in this type of target_compile_definitions(onnxruntime_test_all PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS) -if (CMAKE_SYSTEM_NAME STREQUAL "iOS") +if (IOS) target_compile_definitions(onnxruntime_test_all_xc PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS) endif() if(onnxruntime_RUN_MODELTEST_IN_DEBUG_MODE) @@ -1304,7 +1306,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) target_compile_definitions(onnxruntime_shared_lib_test PRIVATE USE_DUMMY_EXA_DEMANGLE=1) endif() - if (CMAKE_SYSTEM_NAME STREQUAL "iOS") + if (IOS) add_custom_command( TARGET onnxruntime_shared_lib_test POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory @@ -1592,7 +1594,7 @@ if (NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") DEPENDS ${all_dependencies} ) - if (CMAKE_SYSTEM_NAME STREQUAL "iOS") + if (IOS) add_custom_command( TARGET onnxruntime_customopregistration_test POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory From e589fb26ca741550ace51d67d719b5bf9d9b5913 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 5 Mar 2024 19:53:17 -0800 Subject: [PATCH 26/27] update --- cmake/onnxruntime_unittests.cmake | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 1d4c3e0662773..7763b1fe91a28 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") +if (IOS) find_package(XCTest REQUIRED) endif() @@ -18,7 +18,7 @@ function(AddTest) cmake_parse_arguments(_UT "DYN" "TARGET" "LIBS;SOURCES;DEPENDS;TEST_ARGS" ${ARGN}) list(REMOVE_DUPLICATES _UT_SOURCES) - if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + if (IOS) onnxruntime_add_executable(${_UT_TARGET} ${TEST_SRC_DIR}/xctest/orttestmain.m) else() onnxruntime_add_executable(${_UT_TARGET} ${_UT_SOURCES}) @@ -129,7 +129,7 @@ function(AddTest) endif() endif(onnxruntime_GENERATE_TEST_REPORTS) - if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + if (IOS) # target_sources(${_UT_TARGET} PRIVATE ${TEST_SRC_DIR}/xctest/orttestmain.m) set_target_properties(${_UT_TARGET} PROPERTIES FOLDER "ONNXRuntimeTest" MACOSX_BUNDLE_BUNDLE_NAME ${_UT_TARGET} @@ -1064,7 +1064,7 @@ if (NOT IOS) target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + if(IOS) set_target_properties(onnx_test_runner PROPERTIES XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" ) @@ -1179,7 +1179,7 @@ endif() if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) - if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + if(NOT IOS) #perf test runner set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) set(onnxruntime_perf_test_src_patterns @@ -1394,7 +1394,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) target_compile_options(onnxruntime_mlas_test PRIVATE "$<$:SHELL:--compiler-options /wd26426>" "$<$>:/wd26426>") endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + if(IOS) set_target_properties(onnxruntime_mlas_test PROPERTIES XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" ) From dcb8d10bbfe1ca117c5fac544a23151747b5e5d9 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 8 Mar 2024 11:45:22 -0800 Subject: [PATCH 27/27] update --- cmake/onnxruntime_unittests.cmake | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 7763b1fe91a28..cce39ae0f5d47 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1064,11 +1064,6 @@ if (NOT IOS) target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") endif() - if(IOS) - set_target_properties(onnx_test_runner PROPERTIES - XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" - ) - endif() if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1")