forked from tensorflow/tensorflow
-
Notifications
You must be signed in to change notification settings - Fork 95
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2641 from ROCm/develop-upstream-sync-240820
Develop upstream sync 240820
- Loading branch information
Showing
2,567 changed files
with
79,670 additions
and
37,142 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -225,13 +225,16 @@ build:mkl_aarch64_threadpool -c opt | |
| build:cuda --repo_env TF_NEED_CUDA=1 | ||
| build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain | ||
| build:cuda --@local_config_cuda//:enable_cuda | ||
| # Default CUDA and CUDNN versions. | ||
| build:cuda --repo_env=HERMETIC_CUDA_VERSION="12.3.2" | ||
| build:cuda --repo_env=HERMETIC_CUDNN_VERSION="8.9.7.29" | ||
| # This flag is needed to include hermetic CUDA libraries for bazel tests. | ||
| test:cuda --@local_config_cuda//cuda:include_hermetic_cuda_libs=true | ||
|
|
||
| # CUDA: This config refers to building CUDA op kernels with clang. | ||
| build:cuda_clang --config=cuda | ||
| # Enable TensorRT optimizations https://developer.nvidia.com/tensorrt | ||
| build:cuda_clang --config=tensorrt | ||
| build:cuda_clang --action_env=TF_CUDA_CLANG="1" | ||
| build:cuda_clang --@local_config_cuda//:cuda_compiler=clang | ||
| build:cuda_clang --copt=-Qunused-arguments | ||
| # Select supported compute capabilities (supported graphics cards). | ||
| # This is the same as the official TensorFlow builds. | ||
| # See https://developer.nvidia.com/cuda-gpus#compute | ||
|
|
@@ -240,22 +243,22 @@ build:cuda_clang --@local_config_cuda//:cuda_compiler=clang | |
| # release while SASS is only forward compatible inside the current | ||
| # major release. Example: sm_80 kernels can run on sm_89 GPUs but | ||
| # not on sm_90 GPUs. compute_80 kernels though can also run on sm_90 GPUs. | ||
| build:cuda_clang --repo_env=TF_CUDA_COMPUTE_CAPABILITIES="sm_60,sm_70,sm_80,sm_89,compute_90" | ||
| build:cuda_clang --repo_env=HERMETIC_CUDA_COMPUTE_CAPABILITIES="sm_60,sm_70,sm_80,sm_89,compute_90" | ||
| # Set lld as the linker. | ||
| build:cuda_clang --host_linkopt="-fuse-ld=lld" | ||
| build:cuda_clang --host_linkopt="-lm" | ||
| build:cuda_clang --linkopt="-fuse-ld=lld" | ||
| build:cuda_clang --linkopt="-lm" | ||
|
|
||
| # Set up compilation CUDA version and paths and use the CUDA Clang toolchain. | ||
| build:cuda_clang_official --config=cuda_clang | ||
| build:cuda_clang_official --action_env=TF_CUDA_VERSION="12" | ||
| build:cuda_clang_official --action_env=TF_CUDNN_VERSION="8" | ||
| build:cuda_clang_official --action_env=CUDA_TOOLKIT_PATH="/usr/local/cuda-12.3" | ||
| build:cuda_clang_official --action_env=GCC_HOST_COMPILER_PATH="/dt9/usr/bin/gcc" | ||
| build:cuda_clang_official --repo_env=HERMETIC_CUDA_VERSION="12.3.2" | ||
| build:cuda_clang_official --repo_env=HERMETIC_CUDNN_VERSION="8.9.7.29" | ||
| build:cuda_clang_official --action_env=CLANG_CUDA_COMPILER_PATH="/usr/lib/llvm-18/bin/clang" | ||
| build:cuda_clang_official --action_env=LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64" | ||
| build:cuda_clang_official --crosstool_top="@sigbuild-r2.17-clang_config_cuda//crosstool:toolchain" | ||
|
|
||
| # Build with nvcc for CUDA and clang for host | ||
| build:nvcc_clang --config=cuda | ||
| # Unfortunately, cuda_configure.bzl demands this for using nvcc + clang | ||
| build:nvcc_clang --action_env=TF_CUDA_CLANG="1" | ||
| build:nvcc_clang --action_env=TF_NVCC_CLANG="1" | ||
| build:nvcc_clang --@local_config_cuda//:cuda_compiler=nvcc | ||
|
|
||
|
|
@@ -382,6 +385,13 @@ build:windows --features=archive_param_file | |
| build:windows --copt=/d2ReducedOptimizeHugeFunctions | ||
| build:windows --host_copt=/d2ReducedOptimizeHugeFunctions | ||
|
|
||
| # Before VS 2017 15.8, the member "type" would non-conformingly have an | ||
| # alignment of only alignof(max_align_t). VS 2017 15.8 was fixed to handle this | ||
| # correctly, but the fix inherently changes layout and breaks binary | ||
| # compatibility (*only* for uses of aligned_storage with extended alignments). | ||
| build:windows --copt=-D_ENABLE_EXTENDED_ALIGNED_STORAGE | ||
| build:windows --host_copt=-D_ENABLE_EXTENDED_ALIGNED_STORAGE | ||
|
|
||
| # Enable the runfiles symlink tree on Windows. This makes it possible to build | ||
| # the pip package on Windows without an intermediate data-file archive, as the | ||
| # build_pip_package script in its current form (as of Aug 2023) uses the | ||
|
|
@@ -569,10 +579,7 @@ build:rbe_linux_cuda --config=cuda_clang_official | |
| build:rbe_linux_cuda --config=rbe_linux_cpu | ||
| # For Remote build execution -- GPU configuration | ||
| build:rbe_linux_cuda --repo_env=REMOTE_GPU_TESTING=1 | ||
| build:rbe_linux_cuda --repo_env=TF_CUDA_CONFIG_REPO="@sigbuild-r2.17-clang_config_cuda" | ||
| build:rbe_linux_cuda --repo_env=TF_TENSORRT_CONFIG_REPO="@sigbuild-r2.17-clang_config_tensorrt" | ||
| build:rbe_linux_cuda --repo_env=TF_NCCL_CONFIG_REPO="@sigbuild-r2.17-clang_config_nccl" | ||
| test:rbe_linux_cuda --test_env=LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64" | ||
|
|
||
|
|
||
| # ROCm | ||
| # TODO(rocm) Is this actualy used? | ||
|
|
@@ -609,6 +616,9 @@ build:rbe_win_clang --compiler=clang-cl | |
| build:rbe_win_clang --linkopt=/FORCE:MULTIPLE | ||
| build:rbe_win_clang --host_linkopt=/FORCE:MULTIPLE | ||
|
|
||
| # TODO(belitskiy): Rename `rbe_win_clang` to this, once done switching presubmits. | ||
| build:rbe_windows_x86_cpu --config=rbe_win_clang | ||
|
|
||
| # END TF REMOTE BUILD EXECUTION OPTIONS | ||
|
|
||
| # TFLite build configs for generic embedded Linux | ||
|
|
@@ -671,7 +681,6 @@ build:release_cpu_linux_base --linkopt="-fuse-ld=lld" | |
| # Test-related settings below this point. | ||
| test:release_linux_base --build_tests_only --keep_going --test_output=errors --verbose_failures=true | ||
| test:release_linux_base --local_test_jobs=HOST_CPUS | ||
| test:release_linux_base --test_env=LD_LIBRARY_PATH | ||
| # Give only the list of failed tests at the end of the log | ||
| test:release_linux_base --test_summary=short | ||
|
|
||
|
|
@@ -686,7 +695,6 @@ build:release_cpu_linux --config=release_cpu_linux_base | |
| # Set up compilation CUDA version and paths and use the CUDA Clang toolchain. | ||
| # Note that linux cpu and cuda builds share the same toolchain now. | ||
| build:release_gpu_linux --config=cuda_clang_official | ||
| test:release_gpu_linux --test_env=LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64" | ||
| # Local test jobs has to be 4 because parallel_gpu_execute is fragile, I think | ||
| test:release_gpu_linux --test_timeout=300,450,1200,3600 --local_test_jobs=4 --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute | ||
|
|
||
|
|
@@ -717,9 +725,8 @@ build:unsupported_gpu_linux --config=unsupported_cpu_linux | |
| build:unsupported_gpu_linux --action_env=TF_CUDA_VERSION="11" | ||
| build:unsupported_gpu_linux --action_env=TF_CUDNN_VERSION="8" | ||
| build:unsupported_gpu_linux --repo_env=TF_CUDA_COMPUTE_CAPABILITIES="sm_35,sm_50,sm_60,sm_70,sm_75,compute_80" | ||
| build:unsupported_gpu_linux --config=tensorrt | ||
| build:unsupported_gpu_linux --action_env=CUDA_TOOLKIT_PATH="/usr/local/cuda-11.2" | ||
| build:unsupported_gpu_linux --action_env=LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.1/lib64:/usr/local/tensorrt/lib" | ||
| build:unsupported_gpu_linux --action_env=LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.1/lib64" | ||
| build:unsupported_gpu_linux --action_env=GCC_HOST_COMPILER_PATH="/dt9/usr/bin/gcc" | ||
| build:unsupported_gpu_linux [email protected]_manylinux2014-cuda11.2-cudnn8.1-tensorrt7.2_config_cuda//crosstool:toolchain | ||
|
|
||
|
|
@@ -829,17 +836,19 @@ test:macos_x86_wheel_test --config=macos_x86_wheel_test_filters -- //tensorflow/ | |
|
|
||
| # PYCPP TESTS run a suite of Python and C++ tests to verify general correctness over | ||
| # the whole TF code base. These are usually run continuously or upon presubmit. | ||
| # CPU PYCPP: | ||
| # LINUX CPU PYCPP: | ||
| test:linux_cpu_pycpp_test_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only | ||
| test:linux_cpu_pycpp_test_filters --build_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only | ||
| test:linux_cpu_pycpp_test_filters --test_lang_filters=cc,py --test_size_filters=small,medium | ||
| test:linux_cpu_pycpp_test --config=linux_cpu_pycpp_test_filters -- //tensorflow/... -//tensorflow/python/integration_testing/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/... | ||
| # CUDA PYCPP: | ||
|
|
||
| # LINUX CUDA PYCPP: | ||
| test:linux_cuda_pycpp_test_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-benchmark-test,-v1only,gpu,-no_gpu,-no_gpu_presubmit,-no_cuda11 | ||
| test:linux_cuda_pycpp_test_filters --build_tag_filters=-no_oss,-oss_excluded,-oss_serial,-benchmark-test,-v1only,gpu,-no_gpu,-no_gpu_presubmit,-no_cuda11 | ||
| test:linux_cuda_pycpp_test_filters --test_lang_filters=cc,py --test_size_filters=small,medium | ||
| test:linux_cuda_pycpp_test --config=linux_cuda_pycpp_test_filters -- //tensorflow/... -//tensorflow/python/integration_testing/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/... | ||
| # ARM64 PYCPP | ||
|
|
||
| # LINUX ARM64 PYCPP | ||
| # In Linux Arm64 presubmit/continuous build, we cross-compile the binaries on | ||
| # Linux x86 so that we can use RBE. Since tests still need to run on the single | ||
| # host Arm64 machine, the build becomes too slow (~30 min) to be a presubmit. | ||
|
|
@@ -872,6 +881,13 @@ build:macos_x86_pycpp_test --config=macos_x86_pycpp_test_filters -- //tensorflow | |
| # CROSS-COMPILE MACOS X86 PYCPP | ||
| build:cross_compile_macos_x86_pycpp_test --config=macos_x86_pycpp_test | ||
| build:cross_compile_macos_x86_pycpp_test -//tensorflow/core/kernels:quantized_conv_ops_test -//tensorflow/core/kernels:quantized_matmul_op_test -//tensorflow/python/ops:quantized_conv_ops_test -//tensorflow/tools/graph_transforms:transforms_test -//tensorflow/python/tools:aot_compiled_test | ||
| # WINDOWS X86-64 CPU PYCPP | ||
| test:windows_x86_cpu_pycpp_test_filters --test_tag_filters=-no_windows,-windows_excluded,-no_oss,-oss_excluded,-gpu,-tpu,-benchmark-test | ||
| test:windows_x86_cpu_pycpp_test_filters --build_tag_filters=-no_windows,-windows_excluded,-no_oss,-oss_excluded,-benchmark-test | ||
| test:windows_x86_cpu_pycpp_test_filters --test_lang_filters=cc,py --test_size_filters=small,medium --test_timeout="300,450,1200,3600" | ||
| test:windows_x86_cpu_pycpp_test_opts --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions --dynamic_mode=off --build_tests_only | ||
| test:windows_x86_cpu_pycpp_test --config=windows_x86_cpu_pycpp_test_opts --config=windows_x86_cpu_pycpp_test_filters -- //tensorflow/... -//tensorflow/java/... -//tensorflow/lite/... -//tensorflow/compiler/... | ||
|
|
||
| # END TF TEST SUITE OPTIONS | ||
|
|
||
| # START CROSS-COMPILE CONFIGS | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -28,7 +28,7 @@ permissions: | |
| jobs: | ||
| scan-scheduled: | ||
| if: github.repository == 'tensorflow/tensorflow' | ||
| uses: "google/osv-scanner-action/.github/workflows/[email protected].1" | ||
| uses: "google/osv-scanner-action/.github/workflows/[email protected].2" | ||
| with: | ||
| scan-args: |- | ||
| --lockfile=requirements.txt:./requirements_lock_3_9.txt | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.