merge with master

.github/workflows/lint.yml

@@ -97,6 +97,7 @@ jobs:
             --exclude=java/src/main/native/*.c
             --exclude=onnxruntime/core/mlas/inc/*
             --exclude=onnxruntime/core/mlas/lib/*
+            --exclude=onnxruntime/contrib_ops/cuda/bert/flash_attention/*
           filter: "-runtime/references"
 
   lint-js:

.github/workflows/mac.yml

@@ -54,11 +54,10 @@ jobs:
           --test \
           --build_shared_lib \
           --build_objc \
+          --use_coreml \
           --use_xnnpack \
           --use_binskim_compliant_compile_flags
 
-        # TODO add --use_coreml once unit test failures are addressed
-
   Objective-C-StaticAnalysis:
     runs-on: macos-14
 

.github/workflows/sca.yml

@@ -16,6 +16,8 @@ env:
 
 jobs:
   Onnxruntime-SCA-training-CUDA:
+    permissions:
+      security-events: write
     runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
     steps:
       - uses: actions/checkout@v4
@@ -57,6 +59,8 @@ jobs:
 
   # No python
   Onnxruntime-SCA-win32-WINML-x64:
+    permissions:
+      security-events: write
     runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
     steps:
       - uses: actions/checkout@v4
@@ -95,6 +99,8 @@ jobs:
 
   # No java, No python
   Onnxruntime-SCA-win32-WINML-x86:
+    permissions:
+      security-events: write
     runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
     steps:
       - uses: actions/checkout@v4

.lintrunner.toml

@@ -136,6 +136,7 @@ exclude_patterns = [
     'onnxruntime/core/mickey/cutlass_ext/**', # CUTLASS based libs recommends NO automatic code formatting
     'onnxruntime/core/mickey/gemm/**', # CUTLASS based libs recommends NO automatic code formatting
     'winml/lib/Api.Image/shaders/**',  # Contains data chunks
+    'onnxruntime/contrib_ops/cuda/bert/flash_attention/flash_fwd_launch_template.h', # Bool Switches hang Clang
 ]
 command = [
     'python',

cmake/CMakeLists.txt

@@ -101,8 +101,9 @@ option(onnxruntime_BUILD_OBJC "Build Objective-C library" OFF)
 option(onnxruntime_USE_PREINSTALLED_EIGEN "Use pre-installed EIGEN. Need to provide eigen_SOURCE_PATH if turn this on." OFF)
 option(onnxruntime_BUILD_BENCHMARKS "Build ONNXRuntime micro-benchmarks" OFF)
 option(onnxruntime_USE_LLVM "Build TVM with LLVM" OFF)
+option(onnxruntime_USE_VSINPU "Build with VSINPU support" OFF)
 
-cmake_dependent_option(onnxruntime_USE_FLASH_ATTENTION "Build flash attention kernel for scaled dot product attention" ON "NOT WIN32; onnxruntime_USE_CUDA" OFF)
+cmake_dependent_option(onnxruntime_USE_FLASH_ATTENTION "Build flash attention kernel for scaled dot product attention" ON "onnxruntime_USE_CUDA" OFF)
 option(onnxruntime_USE_MEMORY_EFFICIENT_ATTENTION "Build memory efficient attention kernel for scaled dot product attention" ON)
 
 option(onnxruntime_BUILD_FOR_NATIVE_MACHINE "Enable this option for turning on optimization specific to this machine" OFF)
@@ -123,7 +124,6 @@ option(onnxruntime_GCOV_COVERAGE "Compile with options necessary to run code cov
 option(onnxruntime_DONT_VECTORIZE "Do not vectorize operations in Eigen" OFF)
 
 option(onnxruntime_USE_FULL_PROTOBUF "Link to libprotobuf instead of libprotobuf-lite when this option is ON" OFF)
-option(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS "Enable operator implemented in language other than cpp" OFF)
 option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS "Dump debug information about node inputs and outputs when executing the model." OFF)
 cmake_dependent_option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS_ENABLE_DUMP_TO_SQLDB "Build dump debug information about node inputs and outputs with support for sql database." OFF "onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS" OFF)
 option(onnxruntime_USE_DML "Build with DirectML support" OFF)
@@ -374,6 +374,10 @@ if (onnxruntime_USE_ROCM)
     message(WARNING "ck_tile can only be enabled on ROCm >= 6.0 due to compatibility and compilation speed, disable automatically")
     set(onnxruntime_USE_COMPOSABLE_KERNEL_CK_TILE OFF)
   endif()
+  if (onnxruntime_USE_COMPOSABLE_KERNEL_CK_TILE AND CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message(WARNING "ck_tile hits compiler error in Debug build, disable automatically")
+    set(onnxruntime_USE_COMPOSABLE_KERNEL_CK_TILE OFF)
+  endif()
 endif()
 
 
@@ -439,13 +443,6 @@ if (onnxruntime_ENABLE_MEMORY_PROFILE)
 endif()
 
 set(ONNX_ML 1)
-if (NOT onnxruntime_ENABLE_PYTHON)
-  set(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS OFF)
-endif()
-
-if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-  add_compile_definitions(ENABLE_LANGUAGE_INTEROP_OPS)
-endif()
 
 if (NOT (UNIX AND onnxruntime_ENABLE_PYTHON AND onnxruntime_ENABLE_TRAINING AND (NOT onnxruntime_BUILD_SHARED_LIB)))
   if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
@@ -578,11 +575,15 @@ endif()
 #Need python to generate def file
 if (onnxruntime_BUILD_SHARED_LIB OR onnxruntime_ENABLE_PYTHON)
   if (onnxruntime_ENABLE_PYTHON)
-    if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS OR onnxruntime_REQUIRE_PYTHON_EMBED_LIB)
+    if (onnxruntime_REQUIRE_PYTHON_EMBED_LIB)
       find_package(Python 3.8 COMPONENTS Interpreter Development NumPy)
     else()
       find_package(Python 3.8 COMPONENTS Interpreter Development.Module NumPy)
     endif()
+    message("Numpy version: ${Python_NumPy_VERSION}")
+    if(Python_NumPy_VERSION VERSION_LESS "2.0.0")
+      message(WARNING "The build binary will not be compatible with NumPy 2.0 because the NumPy installed on this machine is too low.")
+    endif()
   else()
     find_package(Python 3.8 COMPONENTS Interpreter)
   endif()
@@ -742,6 +743,9 @@ if (onnxruntime_USE_CUDA)
     message( STATUS "Turn off flash attention since CUDA compiler version < 11.6")
     set(onnxruntime_USE_FLASH_ATTENTION OFF)
     set(onnxruntime_USE_MEMORY_EFFICIENT_ATTENTION OFF)
+  elseif(WIN32 AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 12)
+    message( STATUS "Flash-Attention unsupported in Windows with CUDA compiler version < 12.0")
+    set(onnxruntime_USE_FLASH_ATTENTION OFF)
   endif()
   if (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.4)
     message( FATAL_ERROR "Failed build due to CUDA compiler version < 11.4")
@@ -794,6 +798,11 @@ if (onnxruntime_USE_RKNPU)
     list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_RKNPU=1)
     list(APPEND ONNXRUNTIME_PROVIDER_NAMES rknpu)
 endif()
+if (onnxruntime_USE_VSINPU)
+    list(APPEND ORT_PROVIDER_FLAGS -DUSE_VSINPU=1)
+    list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_VSINPU=1)
+    list(APPEND ONNXRUNTIME_PROVIDER_NAMES vsinpu)
+endif()
 if (onnxruntime_USE_NNAPI_BUILTIN)
     list(APPEND ORT_PROVIDER_FLAGS -DUSE_NNAPI=1)
     list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_NNAPI_BUILTIN=1)
@@ -1338,6 +1347,10 @@ if (onnxruntime_USE_OPENVINO)
 
   add_definitions(-DUSE_OPENVINO=1)
 
+  if(onnxruntime_NPU_NO_FALLBACK)
+    add_definitions(-DOPENVINO_DISABLE_NPU_FALLBACK=1)
+  endif()
+
   if (onnxruntime_USE_OPENVINO_GPU)
     add_definitions(-DOPENVINO_CONFIG_GPU=1)
   endif()
@@ -1407,14 +1420,6 @@ string(APPEND ORT_BUILD_INFO "build type=${CMAKE_BUILD_TYPE}")
 string(APPEND ORT_BUILD_INFO ", cmake cxx flags: ${CMAKE_CXX_FLAGS}")
 configure_file(onnxruntime_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_config.h)
 get_property(onnxruntime_GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
-if (onnxruntime_GENERATOR_IS_MULTI_CONFIG)
-  configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/Debug/requirements.txt)
-  configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/Release/requirements.txt)
-  configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/RelWithDebInfo/requirements.txt)
-  configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/MinSizeRel/requirements.txt)
-else()
-  configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/requirements.txt)
-endif()
 
 if (onnxruntime_USE_CUDA)
   set(CMAKE_CUDA_RUNTIME_LIBRARY Shared)

cmake/adjust_global_compile_flags.cmake

@@ -321,7 +321,7 @@ else()
     string(APPEND CMAKE_CXX_FLAGS " -mavx512f -mavx512cd -mavx512bw -mavx512dq -mavx512vl")
     string(APPEND CMAKE_C_FLAGS " -mavx512f -mavx512cd -mavx512bw -mavx512dq -mavx512vl")
   endif()
-  if (CMAKE_SYSTEM_NAME STREQUAL "Android" AND Onnxruntime_GCOV_COVERAGE)
+  if (CMAKE_SYSTEM_NAME STREQUAL "Android" AND onnxruntime_GCOV_COVERAGE)
     string(APPEND CMAKE_CXX_FLAGS " -g -O0 --coverage ")
     string(APPEND CMAKE_C_FLAGS   " -g -O0 --coverage ")
   endif()

cmake/onnxruntime.cmake

@@ -189,6 +189,7 @@ set(onnxruntime_INTERNAL_LIBRARIES
   ${PROVIDERS_SNPE}
   ${PROVIDERS_TVM}
   ${PROVIDERS_RKNPU}
+  ${PROVIDERS_VSINPU}
   ${PROVIDERS_XNNPACK}
   ${PROVIDERS_WEBNN}
   ${PROVIDERS_AZURE}
@@ -205,13 +206,6 @@ set(onnxruntime_INTERNAL_LIBRARIES
   onnxruntime_flatbuffers
 )
 
-if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-  list(APPEND onnxruntime_INTERNAL_LIBRARIES
-    onnxruntime_language_interop
-    onnxruntime_pyop
-  )
-endif()
-
 if (onnxruntime_USE_EXTENSIONS)
   list(APPEND onnxruntime_INTERNAL_LIBRARIES
     onnxruntime_extensions

cmake/onnxruntime_providers.cmake

@@ -80,6 +80,9 @@ endif()
 if(onnxruntime_USE_RKNPU)
   set(PROVIDERS_RKNPU onnxruntime_providers_rknpu)
 endif()
+if(onnxruntime_USE_VSINPU)
+  set(PROVIDERS_VSINPU onnxruntime_providers_vsinpu)
+endif()
 if(onnxruntime_USE_DML)
   set(PROVIDERS_DML onnxruntime_providers_dml)
 endif()
@@ -188,6 +191,35 @@ if (onnxruntime_USE_TVM)
   include(onnxruntime_providers_tvm.cmake)
 endif()
 
+if (onnxruntime_USE_VSINPU)
+  add_definitions(-DUSE_VSINPU=1)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
+  file(GLOB_RECURSE onnxruntime_providers_vsinpu_srcs
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/builders/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/builders/*.cc"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/vsinpu/*.cc"
+    "${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/shared/utils/utils.cc"
+  )
+  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vsinpu_srcs})
+  add_library(onnxruntime_providers_vsinpu ${onnxruntime_providers_vsinpu_srcs})
+  onnxruntime_add_include_to_target(onnxruntime_providers_vsinpu
+    onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf-lite flatbuffers Boost::mp11
+    safeint_interface nsync::nsync_cpp)
+  add_dependencies(onnxruntime_providers_vsinpu ${onnxruntime_EXTERNAL_DEPENDENCIES})
+  set_target_properties(onnxruntime_providers_vsinpu PROPERTIES FOLDER "ONNXRuntime" LINKER_LANGUAGE CXX)
+  target_include_directories(onnxruntime_providers_vsinpu PRIVATE ${ONNXRUNTIME_ROOT} $ENV{TIM_VX_INSTALL}/include)
+
+  find_library(TIMVX_LIBRARY NAMES tim-vx PATHS $ENV{TIM_VX_INSTALL}/lib NO_DEFAULT_PATH)
+  if(TIMVX_LIBRARY)
+    target_link_libraries(onnxruntime_providers_vsinpu PRIVATE ${TIMVX_LIBRARY})
+  else()
+    message(FATAL_ERROR "Cannot find TIM-VX library!")
+  endif()
+
+endif()
+
 if (onnxruntime_USE_XNNPACK)
   include(onnxruntime_providers_xnnpack.cmake)
 endif()

cmake/onnxruntime_providers_tensorrt.cmake

@@ -13,7 +13,6 @@
   set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
   set(PROTOBUF_LIBRARY ${PROTOBUF_LIB})
   if (WIN32)
-    add_definitions(-D_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING=1)
     set(OLD_CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS})
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /wd4324 /wd4701 /wd4804 /wd4702 /wd4458 /wd4703")
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")

cmake/onnxruntime_python.cmake

@@ -193,10 +193,6 @@ target_link_libraries(onnxruntime_pybind11_state PRIVATE
     ${pybind11_lib}
 )
 
-if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-  target_link_libraries(onnxruntime_pybind11_state PRIVATE onnxruntime_language_interop onnxruntime_pyop)
-endif()
-
 set(onnxruntime_pybind11_state_dependencies
     ${onnxruntime_EXTERNAL_DEPENDENCIES}
     ${pybind11_dep}
@@ -566,6 +562,9 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E copy
       ${ONNXRUNTIME_ROOT}/__init__.py
       $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/
+  COMMAND ${CMAKE_COMMAND} -E copy
+      ${REPO_ROOT}/requirements.txt
+      $<TARGET_FILE_DIR:${build_output_target}>
   COMMAND ${CMAKE_COMMAND} -E copy
       ${REPO_ROOT}/ThirdPartyNotices.txt
       $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/
@@ -1027,6 +1026,3 @@ if (onnxruntime_USE_QNN)
 endif()
 
 endif()
-if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-  include(onnxruntime_language_interop_ops.cmake)
-endif()

cmake/onnxruntime_training.cmake

@@ -141,10 +141,6 @@ if (onnxruntime_BUILD_UNIT_TESTS)
       Boost::mp11 safeint_interface
   )
 
-  if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-      list(APPEND ONNXRUNTIME_LIBS onnxruntime_language_interop onnxruntime_pyop)
-  endif()
-
   if(UNIX AND NOT APPLE)
     if (HAS_NO_MAYBE_UNINITIALIZED)
       target_compile_options(onnxruntime_training_mnist PUBLIC "-Wno-maybe-uninitialized")

cmake/onnxruntime_unittests.cmake

@@ -546,6 +546,10 @@ if(onnxruntime_USE_NNAPI_BUILTIN)
   list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_nnapi)
 endif()
 
+if(onnxruntime_USE_VSINPU)
+  list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_vsinpu)
+endif()
+
 if(onnxruntime_USE_JSEP)
   list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_js)
 endif()
@@ -583,16 +587,13 @@ if(onnxruntime_USE_ARMNN)
   list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_armnn)
 endif()
 
-if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-  set(ONNXRUNTIME_INTEROP_TEST_LIBS PRIVATE onnxruntime_language_interop onnxruntime_pyop)
-endif()
-
 set(ONNXRUNTIME_TEST_LIBS
     onnxruntime_session
     ${ONNXRUNTIME_INTEROP_TEST_LIBS}
     ${onnxruntime_libs}
     # CUDA, ROCM, TENSORRT, MIGRAPHX, DNNL, and OpenVINO are dynamically loaded at runtime
     ${PROVIDERS_NNAPI}
+    ${PROVIDERS_VSINPU}
     ${PROVIDERS_JS}
     ${PROVIDERS_QNN}
     ${PROVIDERS_SNPE}
@@ -916,10 +917,6 @@ endif()
 if (onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS)
   target_compile_definitions(onnxruntime_test_all PRIVATE DEBUG_NODE_INPUTS_OUTPUTS)
 endif()
-
-if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-  target_link_libraries(onnxruntime_test_all PRIVATE onnxruntime_language_interop onnxruntime_pyop)
-endif()
 if (onnxruntime_USE_ROCM)
   if (onnxruntime_USE_COMPOSABLE_KERNEL)
     target_compile_definitions(onnxruntime_test_all PRIVATE USE_COMPOSABLE_KERNEL)
@@ -1057,10 +1054,6 @@ set(onnx_test_libs
   onnx_test_data_proto
   ${onnxruntime_EXTERNAL_LIBRARIES})
 
-if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-  list(APPEND onnx_test_libs onnxruntime_language_interop onnxruntime_pyop)
-endif()
-
 if (NOT IOS)
     onnxruntime_add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc)
     if(MSVC)
@@ -1241,10 +1234,6 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
     endif()
     set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest")
 
-    if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS AND NOT onnxruntime_BUILD_SHARED_LIB)
-      target_link_libraries(onnxruntime_perf_test PRIVATE onnxruntime_language_interop onnxruntime_pyop)
-    endif()
-
     if (onnxruntime_USE_TVM)
       if (WIN32)
         target_link_options(onnxruntime_perf_test PRIVATE "/STACK:4000000")
@@ -1474,10 +1463,6 @@ endif()
       onnxruntime_flatbuffers
     )
 
-    if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
-      list(APPEND ONNXRUNTIME_TEST_LIBS onnxruntime_language_interop onnxruntime_pyop)
-    endif()
-
     target_link_libraries(onnxruntime_test_trainer PRIVATE
       ${ONNXRUNTIME_TEST_LIBS}
       ${onnxruntime_EXTERNAL_LIBRARIES}

cmake/patches/protobuf/protobuf_cmake.patch

@@ -12,11 +12,12 @@ index 04cb3303a..c023001de 100644
      /wd4305 # 'identifier' : truncation from 'type1' to 'type2'
      /wd4307 # 'operator' : integral constant overflow
      /wd4309 # 'conversion' : truncation of constant value
-@@ -259,7 +257,6 @@ if (MSVC)
+@@ -259,7 +257,7 @@ if (MSVC)
      /wd4355 # 'this' : used in base member initializer list
      /wd4506 # no definition for inline function 'function'
      /wd4800 # 'type' : forcing value to bool 'true' or 'false' (performance warning)
 -    /wd4996 # The compiler encountered a deprecated declaration.
++    ${onnxruntime_PROTOBUF_EXTRA_WARNING_DISABLEMENT}
    )
    # Allow big object
    add_definitions(/bigobj)