Model accuracy test tool

cmake/onnxruntime_unittests.cmake

@@ -1171,6 +1171,39 @@ endif()
 
 
 if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
+  # Accuracy test runner
+  set(onnxruntime_acc_test_src_dir ${TEST_SRC_DIR}/acc_test)
+  set(onnxruntime_acc_test_src_patterns
+  "${onnxruntime_acc_test_src_dir}/*.cc"
+  "${onnxruntime_acc_test_src_dir}/*.h")
+
+  file(GLOB onnxruntime_acc_test_src CONFIGURE_DEPENDS
+    ${onnxruntime_acc_test_src_patterns}
+    )
+  onnxruntime_add_executable(onnxruntime_acc_test ${onnxruntime_acc_test_src})
+  target_include_directories(onnxruntime_acc_test PRIVATE ${REPO_ROOT}/include/onnxruntime/core/session)
+  if (WIN32)
+    target_compile_options(onnxruntime_acc_test PRIVATE ${disabled_warnings})
+  endif()
+  if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS")
+    set_target_properties(onnxruntime_acc_test PROPERTIES
+      XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
+    )
+  endif()
+
+  if (onnxruntime_BUILD_SHARED_LIB)
+    set(onnxruntime_acc_test_libs onnxruntime)
+    target_link_libraries(onnxruntime_acc_test PRIVATE ${onnxruntime_acc_test_libs})
+  endif()
+
+  set_target_properties(onnxruntime_acc_test PROPERTIES FOLDER "ONNXRuntimeTest")
+
+  if (onnxruntime_USE_TVM)
+    if (WIN32)
+      target_link_options(onnxruntime_acc_test PRIVATE "/STACK:4000000")
+    endif()
+  endif()
+
   #perf test runner
   set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest)
   set(onnxruntime_perf_test_src_patterns

onnxruntime/test/acc_test/acc_task.cc

@@ -0,0 +1,86 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+#include "acc_task.h"
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <variant>
+#include <vector>
+
+static std::vector<Ort::Value> RunInference(Ort::Session& session, const ModelIOInfo& model_io_info,
+                                            Span<const char> input_buffer) {
+  // Setup input
+  const std::vector<IOInfo>& input_infos = model_io_info.inputs;
+  const size_t num_inputs = input_infos.size();
+  std::vector<Ort::Value> ort_inputs;
+  std::vector<const char*> ort_input_names;
+
+  ort_inputs.reserve(num_inputs);
+  ort_input_names.reserve(num_inputs);
+
+  for (size_t input_offset = 0, i = 0; i < num_inputs; input_offset += input_infos[i].total_data_size, i++) {
+    assert(input_offset < input_buffer.size());
+    const IOInfo& input_info = input_infos[i];
+    Span<const char> input_data(&input_buffer[input_offset], input_info.total_data_size);
+    auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+
+    ort_inputs.emplace_back(Ort::Value::CreateTensor(memory_info, (void*)input_data.data(), input_data.size(),
+                                                     input_info.shape.data(), input_info.shape.size(),
+                                                     input_info.data_type));
+    ort_input_names.push_back(input_info.name.c_str());
+  }
+
+  const size_t num_outputs = model_io_info.outputs.size();
+  std::vector<const char*> ort_output_names;
+  ort_output_names.reserve(num_outputs);
+
+  for (size_t i = 0; i < num_outputs; i++) {
+    ort_output_names.push_back(model_io_info.outputs[i].name.c_str());
+  }
+
+  return session.Run(Ort::RunOptions{nullptr}, ort_input_names.data(), ort_inputs.data(),
+                     ort_inputs.size(), ort_output_names.data(), ort_output_names.size());
+}
+
+void Task::Run() {
+  std::vector<Ort::Value> ort_output_vals = RunInference(session_, model_io_info_, input_buffer_);
+
+  AccuracyCheck* accuracy_check_data = std::get_if<AccuracyCheck>(&variant_);
+  if (accuracy_check_data) {
+    const std::vector<IOInfo>& output_infos = model_io_info_.get().outputs;
+    const size_t num_outputs = output_infos.size();
+    Span<const char> expected_output_buffer = accuracy_check_data->expected_output_buffer;
+
+    for (size_t output_offset = 0, i = 0; i < num_outputs; output_offset += output_infos[i].total_data_size, i++) {
+      assert(output_offset < expected_output_buffer.size());
+      const IOInfo& output_info = output_infos[i];
+      Span<const char> raw_expected_output(&expected_output_buffer[output_offset], output_info.total_data_size);
+
+      accuracy_check_data->output_acc_metric[i] = ComputeAccuracyMetric(ort_output_vals[i].GetConst(),
+                                                                        raw_expected_output,
+                                                                        output_info);
+    }
+    return;
+  }
+
+  Inference* inference_data = std::get_if<Inference>(&variant_);
+  if (inference_data) {
+    Span<char>& output_buffer = inference_data->output_buffer;
+
+    // Unfortunately, we have to copy output values (Ort::Value is not copyable, so it is limited when stored in a std::vector)
+    const std::vector<IOInfo>& output_infos = model_io_info_.get().outputs;
+    const size_t num_outputs = output_infos.size();
+
+    for (size_t output_offset = 0, i = 0; i < num_outputs; output_offset += output_infos[i].total_data_size, i++) {
+      assert(output_offset < output_buffer.size());
+      std::memcpy(&output_buffer[output_offset],
+                  ort_output_vals[i].GetTensorRawData(),
+                  output_infos[i].total_data_size);
+    }
+    return;
+  }
+
+  // Should not reach this line unless we add a new (unhandled) std::variant type.
+  std::cerr << "[ERROR]: Unhandled std::variant type for Task::variant_ member." << std::endl;
+  std::abort();
+}

onnxruntime/test/acc_test/acc_task.h

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+#pragma once
+#include <onnxruntime_cxx_api.h>
+#include <variant>
+#include <functional>
+#include "basic_utils.h"
+#include "model_io_utils.h"
+
+class Task {
+ private:
+  struct Inference {
+    Span<char> output_buffer;
+  };
+
+  struct AccuracyCheck {
+    Span<const char> expected_output_buffer;
+    Span<AccMetrics> output_acc_metric;
+  };
+
+ public:
+  Task() = default;
+  Task(Task&& other) = default;
+  Task(const Task& other) = default;
+  Task(Ort::Session& session, const ModelIOInfo& model_io_info,
+       Span<const char> input_buffer, Span<char> output_buffer)
+      : session_(session), model_io_info_(model_io_info), input_buffer_(input_buffer), variant_(Inference{output_buffer}) {}
+  Task(Ort::Session& session, const ModelIOInfo& model_io_info,
+       Span<const char> input_buffer, Span<const char> expected_output_buffer, Span<AccMetrics> output_acc_metric)
+      : session_(session), model_io_info_(model_io_info), input_buffer_(input_buffer), variant_(AccuracyCheck{expected_output_buffer, output_acc_metric}) {}
+
+  static Task CreateInferenceTask(Ort::Session& session, const ModelIOInfo& model_io_info,
+                                  Span<const char> input_buffer, Span<char> output_buffer) {
+    return Task(session, model_io_info, input_buffer, output_buffer);
+  }
+
+  static Task CreateAccuracyCheckTask(Ort::Session& session, const ModelIOInfo& model_io_info,
+                                      Span<const char> input_buffer, Span<const char> expected_output_buffer,
+                                      Span<AccMetrics> output_acc_metric) {
+    return Task(session, model_io_info, input_buffer, expected_output_buffer, output_acc_metric);
+  }
+
+  void Run();
+
+ private:
+  std::reference_wrapper<Ort::Session> session_;
+  std::reference_wrapper<const ModelIOInfo> model_io_info_;
+  Span<const char> input_buffer_;
+  std::variant<Inference, AccuracyCheck> variant_;
+};

onnxruntime/test/acc_test/basic_utils.cc

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+#include "basic_utils.h"
+#include <fstream>
+#include <string>
+
+bool FillBytesFromBinaryFile(Span<char> array, const std::string& binary_filepath) {
+  std::ifstream input_ifs(binary_filepath, std::ifstream::binary);
+
+  if (!input_ifs.is_open()) {
+    return false;
+  }
+
+  size_t file_byte_size = 0;
+  input_ifs.seekg(0, input_ifs.end);
+  file_byte_size = input_ifs.tellg();
+  input_ifs.seekg(0, input_ifs.beg);
+
+  if (file_byte_size != array.size()) {
+    return false;
+  }
+
+  input_ifs.read(array.data(), file_byte_size);
+  return static_cast<bool>(input_ifs);
+}
+
+int64_t GetFileIndexSuffix(const std::string& filename_wo_ext, const char* prefix) {
+  int64_t index = -1;
+  const char* str = filename_wo_ext.c_str();
+
+  // Move past the prefix.
+  while (*str && *prefix && *str == *prefix) {
+    str++;
+    prefix++;
+  }
+
+  if (*prefix) {
+    return -1;  // File doesn't start with the prefix.
+  }
+
+  // Parse the input index from file name.
+  index = 0;
+  while (*str) {
+    int64_t c = *str;
+    if (!(c >= '0' && c <= '9')) {
+      return -1;  // Not a number.
+    }
+
+    index *= 10;
+    index += (c - '0');
+    str++;
+  }
+
+  return index;
+}
+

onnxruntime/test/acc_test/basic_utils.h

@@ -0,0 +1,116 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+#pragma once
+#include <array>
+#include <cassert>
+#include <cmath>
+#include <condition_variable>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+// Make a bootleg std::span for C++ versions older than 20
+template <typename T>
+class Span {
+ public:
+  Span() = default;
+  Span(T* data, size_t size) : data_(data), size_(size) {}
+  Span(std::vector<T>& vec) : data_(vec.data()), size_(vec.size()) {}
+  Span(const std::vector<std::remove_const_t<T>>& vec) : data_(vec.data()), size_(vec.size()) {}
+
+  template <size_t N>
+  Span(std::array<T, N> arr) : data_(arr.data()), size_(N) {}
+
+  Span(const Span& other) = default;
+  Span(Span&& other) = default;
+
+  Span& operator=(const Span& other) = default;
+  Span& operator=(Span&& other) = default;
+
+  T& operator[](size_t index) const {
+    return data_[index];
+  }
+
+  T* data() const { return data_; }
+  size_t size() const { return size_; }
+  bool empty() const { return size_ == 0; }
+
+ private:
+  T* data_{nullptr};
+  size_t size_{0};
+};
+
+template <typename T>
+static Span<T> ReinterpretBytesAsSpan(Span<std::conditional_t<std::is_const_v<T>, const char, char>> bytes_span) {
+  return Span<T>(reinterpret_cast<T*>(bytes_span.data()), bytes_span.size() / sizeof(T));
+}
+
+template <typename T>
+constexpr int64_t GetShapeSize(Span<T> shape) {
+  int64_t size = 1;
+
+  for (size_t i = 0; i < shape.size(); i++) {
+    size *= shape[i];
+  }
+
+  return size;
+}
+
+int64_t GetFileIndexSuffix(const std::string& filename_wo_ext, const char* prefix);
+bool FillBytesFromBinaryFile(Span<char> array, const std::string& binary_filepath);
+
+constexpr double EPSILON_DBL = 2e-16;
+
+struct AccMetrics {
+  double rmse = 0.0;
+  double snr = 0.0;
+  double min_val = 0.0;
+  double max_val = 0.0;
+  double min_expected_val = 0.0;
+  double max_expected_val = 0.0;
+  friend bool operator==(const AccMetrics& l, const AccMetrics& r) {
+    if (l.rmse != r.rmse) return false;
+    if (l.min_val != r.min_val) return false;
+    if (l.max_val != r.max_val) return false;
+    if (l.min_expected_val != r.min_expected_val) return false;
+    if (l.max_expected_val != r.max_expected_val) return false;
+    if (l.snr != r.snr) return false;
+
+    return true;
+  }
+  friend bool operator!=(const AccMetrics& l, const AccMetrics& r) {
+    return !(l == r);
+  }
+};
+
+template <typename T>
+void GetAccuracy(Span<const T> expected_output, Span<const T> actual_output, AccMetrics& metrics) {
+  // Compute RMSE. This is not a great way to measure accuracy, but ....
+  assert(expected_output.size() == actual_output.size());
+  const size_t num_outputs = expected_output.size();
+
+  metrics.rmse = 0.0;
+  metrics.min_val = static_cast<double>(actual_output[0]);
+  metrics.max_val = static_cast<double>(actual_output[0]);
+  metrics.min_expected_val = static_cast<double>(expected_output[0]);
+  metrics.max_expected_val = static_cast<double>(expected_output[0]);
+  double tensor_norm = 0.0;
+  double diff_norm = 0.0;
+  for (size_t i = 0; i < num_outputs; i++) {
+    double diff = static_cast<double>(actual_output[i]) - static_cast<double>(expected_output[i]);
+    diff_norm += diff * diff;
+    tensor_norm += static_cast<double>(expected_output[i]) * static_cast<double>(expected_output[i]);
+
+    metrics.rmse += diff * diff;
+    metrics.min_val = std::min(metrics.min_val, static_cast<double>(actual_output[i]));
+    metrics.max_val = std::max(metrics.max_val, static_cast<double>(actual_output[i]));
+    metrics.min_expected_val = std::min(metrics.min_expected_val, static_cast<double>(expected_output[i]));
+    metrics.max_expected_val = std::max(metrics.max_expected_val, static_cast<double>(expected_output[i]));
+  }
+
+  metrics.rmse = std::sqrt(metrics.rmse / static_cast<double>(num_outputs));
+
+  tensor_norm = std::max(std::sqrt(tensor_norm), EPSILON_DBL);
+  diff_norm = std::max(std::sqrt(diff_norm), EPSILON_DBL);
+  metrics.snr = 20.0 * std::log10(tensor_norm / diff_norm);
+}