Skip to content

Commit

Permalink
Merge branch 'main' into Cjian/win_c++20
Browse files Browse the repository at this point in the history
  • Loading branch information
jchen351 committed Oct 4, 2023
2 parents 200fa0e + 1bc1157 commit 3eb33e9
Show file tree
Hide file tree
Showing 243 changed files with 13,349 additions and 3,080 deletions.
49 changes: 0 additions & 49 deletions cgmanifests/generate_cgmanifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,55 +90,6 @@ def add_github_dep(name, parsed_url):
git_deps[dep] = name


with open(
os.path.join(REPO_DIR, "tools", "ci_build", "github", "linux", "docker", "Dockerfile.manylinux2_28_cuda11"),
) as f:
for line in f:
if not line.strip():
package_name = None
package_filename = None
package_url = None
if package_filename is None:
m = re.match(r"RUN\s+export\s+(.+?)_ROOT=(\S+).*", line)
if m is not None:
package_name = m.group(1)
package_filename = m.group(2)
else:
m = re.match(r"RUN\s+export\s+(.+?)_VERSION=(\S+).*", line)
if m is not None:
package_name = m.group(1)
package_filename = m.group(2)
elif package_url is None:
m = re.match(r"(.+?)_DOWNLOAD_URL=(\S+)", line)
if m is not None:
package_url = m.group(2)
if package_name == "LIBXCRYPT":
package_url = m.group(2) + "/v" + package_filename + ".tar.gz"
elif package_name == "CMAKE":
package_url = m.group(2) + "/v" + package_filename + "/cmake-" + package_filename + ".tar.gz"
else:
package_url = m.group(2) + "/" + package_filename + ".tar.gz"
parsed_url = urlparse(package_url)
if parsed_url.hostname == "github.com":
add_github_dep("manylinux dependency " + package_name, parsed_url)
else:
registration = {
"Component": {
"Type": "other",
"other": {
"Name": package_name.lower(),
"Version": package_filename.split("-")[-1],
"DownloadUrl": package_url,
},
"comments": "manylinux dependency",
}
}
registrations.append(registration)
package_name = None
package_filename = None
package_url = None


def normalize_path_separators(path):
return path.replace(os.path.sep, "/")

Expand Down
116 changes: 10 additions & 106 deletions cgmanifests/generated/cgmanifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,112 +2,6 @@
"$schema": "https://json.schemastore.org/component-detection-manifest.json",
"Version": 1,
"Registrations": [
{
"Component": {
"Type": "other",
"other": {
"Name": "autoconf",
"Version": "2.71",
"DownloadUrl": "http://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "automake",
"Version": "1.16.5",
"DownloadUrl": "http://ftp.gnu.org/gnu/automake/automake-1.16.5.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "libtool",
"Version": "2.4.7",
"DownloadUrl": "http://ftp.gnu.org/gnu/libtool/libtool-2.4.7.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "git",
"Version": "2.36.2",
"DownloadUrl": "https://www.kernel.org/pub/software/scm/git/git-2.36.2.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "sqlite_autoconf",
"Version": "3390200",
"DownloadUrl": "https://www.sqlite.org/2022/sqlite-autoconf-3390200.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "openssl",
"Version": "1.1.1q",
"DownloadUrl": "https://www.openssl.org/source/openssl-1.1.1q.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "50cf2b6dd4fdf04309445f2eec8de7051d953abf",
"repositoryUrl": "https://github.com/besser82/libxcrypt.git"
},
"comments": "manylinux dependency LIBXCRYPT"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "a896e3d066448b3530dbcaa48869fafefd738f57",
"repositoryUrl": "https://github.com/emscripten-core/emsdk.git"
},
"comments": "git submodule at cmake/external/emsdk"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "7a2ed51a6b682a83e345ff49fc4cfd7ca47550db",
"repositoryUrl": "https://github.com/google/libprotobuf-mutator.git"
},
"comments": "git submodule at cmake/external/libprotobuf-mutator"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "e2525550194ce3d8a2c4a3af451c9d9b3ae6650e",
"repositoryUrl": "https://github.com/onnx/onnx.git"
},
"comments": "git submodule at cmake/external/onnx"
}
},
{
"component": {
"type": "git",
Expand Down Expand Up @@ -268,6 +162,16 @@
"comments": "mp11"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "fdefbe85ed9c362b95b9b401cd19db068a76141f",
"repositoryUrl": "https://github.com/onnx/onnx.git"
},
"comments": "onnx"
}
},
{
"component": {
"type": "git",
Expand Down
14 changes: 8 additions & 6 deletions cmake/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ option(onnxruntime_ENABLE_PYTHON "Enable python buildings" OFF)
# Enable it may cause LNK1169 error
option(onnxruntime_ENABLE_MEMLEAK_CHECKER "Experimental: Enable memory leak checker in Windows debug build" OFF)
option(onnxruntime_USE_CUDA "Build with CUDA support" OFF)
# Enable ONNX Runtime CUDA EP's internal unit tests that directly access the EP's internal functions instead of through
# OpKernels. When the option is ON, we will have two copies of GTest library in the same process. It is not a typical
# use. If you hit any problem with that, please do not report it to GTest. Turn OFF the following build option instead.
cmake_dependent_option(onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS "Build with CUDA unit tests" OFF "onnxruntime_USE_CUDA;onnxruntime_BUILD_UNIT_TESTS;LINUX" OFF)

option(onnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO "When building with CUDA support, generate device code line number information." OFF)
option(onnxruntime_USE_OPENVINO "Build with OpenVINO support" OFF)
option(onnxruntime_USE_COREML "Build with CoreML support" OFF)
Expand Down Expand Up @@ -146,10 +151,11 @@ option(onnxruntime_DISABLE_SPARSE_TENSORS "Disable sparse tensors data types" OF
option(onnxruntime_DISABLE_OPTIONAL_TYPE "Disable optional type" OFF)
option(onnxruntime_DISABLE_FLOAT8_TYPES "Disable float 8 types" OFF)
option(onnxruntime_MINIMAL_BUILD "Exclude as much as possible from the build. Support ORT format models. No support for ONNX format models." OFF)
cmake_dependent_option(onnxruntime_DISABLE_RTTI "Disable RTTI" ON "NOT onnxruntime_ENABLE_PYTHON" OFF)
cmake_dependent_option(onnxruntime_DISABLE_RTTI "Disable RTTI" ON "NOT onnxruntime_ENABLE_PYTHON;NOT onnxruntime_USE_CUDA" OFF)
# For now onnxruntime_DISABLE_EXCEPTIONS will only work with onnxruntime_MINIMAL_BUILD, more changes (ONNX, non-CPU EP, ...) are required to run this standalone
cmake_dependent_option(onnxruntime_DISABLE_EXCEPTIONS "Disable exception handling. Requires onnxruntime_MINIMAL_BUILD currently." ON "onnxruntime_MINIMAL_BUILD;NOT onnxruntime_ENABLE_PYTHON" OFF)
option(onnxruntime_DISABLE_ABSEIL "Do not link to Abseil. Redefine Inlined containers to STD containers." OFF)
# Even when onnxruntime_DISABLE_ABSEIL is ON, ONNX Runtime still needs to link to abseil.
option(onnxruntime_DISABLE_ABSEIL "Do not use Abseil data structures in ONNX Runtime source code. Redefine Inlined containers to STD containers." OFF)

option(onnxruntime_EXTENDED_MINIMAL_BUILD "onnxruntime_MINIMAL_BUILD with support for execution providers that compile kernels." OFF)
option(onnxruntime_MINIMAL_BUILD_CUSTOM_OPS "Add custom operator kernels support to a minimal build." OFF)
Expand Down Expand Up @@ -269,10 +275,6 @@ if (onnxruntime_ENABLE_TRAINING_APIS)
endif()
endif()

if (onnxruntime_USE_CUDA)
set(onnxruntime_DISABLE_RTTI OFF)
endif()

if (onnxruntime_USE_ROCM)
if (WIN32)
message(FATAL_ERROR "ROCM does not support build in Windows!")
Expand Down
4 changes: 2 additions & 2 deletions cmake/deps.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ microsoft_gsl;https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip;cf36
microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.zip;e4a542a323c070376f7c2d1973d0f7ddbc1d2fa5
mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.1.1.zip;d5ee7d34223d0567892db5179849939c8769dc41
mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.82.0.zip;9bc9e01dffb64d9e0773b2e44d2f22c51aace063
onnx;https://github.com/onnx/onnx/archive/e2525550194ce3d8a2c4a3af451c9d9b3ae6650e.zip;782f23d788185887f520a90535513e244218e928
onnx;https://github.com/onnx/onnx/archive/14303de049144035dfd94ace5f7a3b44773b1aad.zip;250eab9690392b248d75b56e605fb49eca373442
#use the commit of supporting all the plugins and TRT 8.6-GA (https://github.com/onnx/onnx-tensorrt/commit/0462dc31ae78f48744b6141ae376df1f96d3f459)
onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/0462dc31ae78f48744b6141ae376df1f96d3f459.zip;5ff086361956cceb81ed17453a1fd8db2aa4328d
protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v21.12.zip;7cf2733949036c7d52fda017badcab093fe73bfa
Expand All @@ -44,4 +44,4 @@ tensorboard;https://github.com/tensorflow/tensorboard/archive/373eb09e4c5d2b3cc2
cutlass;https://github.com/NVIDIA/cutlass/archive/refs/tags/v3.0.0.zip;0f95b3c1fc1bd1175c4a90b2c9e39074d1bccefd
utf8_range;https://github.com/protocolbuffers/utf8_range/archive/72c943dea2b9240cd09efde15191e144bc7c7d38.zip;9925739c9debc0efa2adcb194d371a35b6a03156
extensions;https://github.com/microsoft/onnxruntime-extensions/archive/94142d8391c9791ec71c38336436319a2d4ac7a0.zip;4365ac5140338b4cb75a39944a4be276e3829b3c
composable_kernel;https://github.com/ROCmSoftwarePlatform/composable_kernel/archive/d52ec01652b7d620386251db92455968d8d90bdc.zip;6b5ce8edf3625f8817086c194fbf94b664e1b0e0
composable_kernel;https://github.com/ROCmSoftwarePlatform/composable_kernel/archive/d52ec01652b7d620386251db92455968d8d90bdc.zip;6b5ce8edf3625f8817086c194fbf94b664e1b0e0
56 changes: 56 additions & 0 deletions cmake/deps_update_and_upload.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# in case deps.txt is updated, run this file to update and upload the dependencies so that CI can use them.
# Before running the script, increase the version number found at:
# https://aiinfra.visualstudio.com/Lotus/_artifacts/feed/Lotus/UPack/onnxruntime_build_dependencies/versions
# Run without --do-upload once to verify downloading. Use --do-upload when you are ready to publish.
# python cmake/deps_update_and_upload.py --root-path C:/temp/onnxruntime_deps --version 1.0.82 --do-upload
# update version number in tools\ci_build\github\azure-pipelines\templates\download-deps.yml
import re
import subprocess
import os
import argparse
import tempfile

parser = argparse.ArgumentParser(description="Update dependencies and publish to Azure Artifacts")
parser.add_argument(
"--root-path", type=str, default=tempfile.gettempdir(), help="Target root path for downloaded files"
)
parser.add_argument("--version", type=str, default="1.0.82", help="Package version to publish")
parser.add_argument("--do-upload", action="store_true", help="Upload the package to Azure Artifacts")
args = parser.parse_args()

with open("cmake/deps.txt") as file:
text = file.read()

lines = [line for line in text.split("\n") if not line.startswith("#") and ";" in line]

root_path = args.root_path

for line in lines:
url = re.sub("^[^;]+?;https://([^;]+?);.*", r"https://\1", line)
filename = re.sub("^[^;]+?;https://([^;]+?);.*", r"\1", line)
full_path = os.path.join(root_path, filename)
subprocess.run(["curl", "-sSL", "--create-dirs", "-o", full_path, url])

package_name = "onnxruntime_build_dependencies"
version = args.version

# Check if the user is logged in to Azure
result = subprocess.run("az account show", shell=True, capture_output=True, text=True)
if "No subscriptions found" in result.stderr:
# Prompt the user to log in to Azure
print("You are not logged in to Azure. Please log in to continue.")
subprocess.run("az login", shell=True)

# Publish the package to Azure Artifacts if --no-upload is not specified

cmd = f'az artifacts universal publish --organization https://dev.azure.com/onnxruntime --feed onnxruntime --name {package_name} --version {version} --description "onnxruntime build time dependencies" --path {root_path}'
if args.do_upload:
subprocess.run(cmd, shell=True)
else:
print("would have run: " + cmd)

cmd = f'az artifacts universal publish --organization https://dev.azure.com/aiinfra --feed Lotus --name {package_name} --version {version} --description "onnxruntime build time dependencies" --path {root_path}'
if args.do_upload:
subprocess.run(cmd, shell=True)
else:
print("would have run: " + cmd)
8 changes: 6 additions & 2 deletions cmake/external/onnxruntime_external_deps.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,12 @@ if (onnxruntime_BUILD_UNIT_TESTS)
set(gtest_disable_pthreads ON)
endif()
set(INSTALL_GTEST OFF CACHE BOOL "" FORCE)
# Set it to ON will cause crashes in onnxruntime_test_all when onnxruntime_USE_CUDA is ON
set(GTEST_HAS_ABSL OFF CACHE BOOL "" FORCE)
if (CMAKE_SYSTEM_NAME STREQUAL "iOS")
# Needs to update onnxruntime/test/xctest/xcgtest.mm
set(GTEST_HAS_ABSL OFF CACHE BOOL "" FORCE)
else()
set(GTEST_HAS_ABSL ON CACHE BOOL "" FORCE)
endif()
# gtest and gmock
FetchContent_Declare(
googletest
Expand Down
Loading

0 comments on commit 3eb33e9

Please sign in to comment.