From 28a58a6350966cb7f810e7dba2f333be647c6e44 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 24 Jan 2024 17:05:33 -0800 Subject: [PATCH 01/19] Adding --gpus all to the docker --- .github/workflows/linux-gpu-x64-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 6699fc7ef..95b976448 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -1,5 +1,5 @@ name: "Linux GPU x64 Build" -on: [ workflow_dispatch ] +on: [ workflow_dispatch, pull_request ] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -43,7 +43,7 @@ jobs: --docker-build-args "--build-arg BUILD_UID=$( id -u )" \ --container-registry onnxruntimebuildcache \ --repository onnxruntimegpubuild - docker run --rm --volume $GITHUB_WORKSPACE:/onnxruntime_src --volume $RUNNER_TEMP:/build -w /build onnxruntimegpubuild bash -c "echo $PATH && /usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" + docker run --gpus all --rm --volume $GITHUB_WORKSPACE:/onnxruntime_src --volume $RUNNER_TEMP:/build -w /build onnxruntimegpubuild bash -c "echo $PATH && /usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" From 1654543e43a0927205d8a6138d559df698ce193d Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 24 Jan 2024 17:22:09 -0800 Subject: [PATCH 02/19] setting cuda_home --- .github/workflows/linux-gpu-x64-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 95b976448..1d0f79a8b 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -43,7 +43,7 @@ jobs: --docker-build-args "--build-arg BUILD_UID=$( id -u )" \ --container-registry onnxruntimebuildcache \ --repository onnxruntimegpubuild - docker run --gpus all --rm --volume $GITHUB_WORKSPACE:/onnxruntime_src --volume $RUNNER_TEMP:/build -w /build onnxruntimegpubuild bash -c "echo $PATH && /usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" + docker run --gpus all --rm --volume $GITHUB_WORKSPACE:/onnxruntime_src --volume $RUNNER_TEMP:/build -w /build onnxruntimegpubuild bash -c "echo $PATH && /usr/bin/cmake -T cuda=/usr/local/cuda-12.2 -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" From 87ee7fc45c2ddebc8e66746f7657d92446b06f35 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 24 Jan 2024 17:45:38 -0800 Subject: [PATCH 03/19] Printing docker enviroemntal variables --- .github/workflows/linux-gpu-x64-build.yml | 13 ++++++++++++- .../docker/inference/x64/default/gpu/Dockerfile | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 1d0f79a8b..3134e456c 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -43,7 +43,18 @@ jobs: --docker-build-args "--build-arg BUILD_UID=$( id -u )" \ --container-registry onnxruntimebuildcache \ --repository onnxruntimegpubuild - docker run --gpus all --rm --volume $GITHUB_WORKSPACE:/onnxruntime_src --volume $RUNNER_TEMP:/build -w /build onnxruntimegpubuild bash -c "echo $PATH && /usr/bin/cmake -T cuda=/usr/local/cuda-12.2 -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" + echo "Printing docker environment variables" + docker run --rm onnxruntimegpubuild env + echo "Running docker image onnxruntimegpubuild" + docker run \ + --gpus all \ + --rm \ + --volume $GITHUB_WORKSPACE:/onnxruntime_src \ + --volume $RUNNER_TEMP:/build \ + -w /build onnxruntimegpubuild bash \ + -c "echo $PATH && /usr/bin/cmake \ + -e CUDA_PATH=/usr/local/cuda-12.2 \ + -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile index 086aafd60..d271ac821 100644 --- a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile @@ -5,7 +5,7 @@ ARG BASEIMAGE=nvidia/cuda:12.2.2-cudnn8-devel-ubi8 FROM $BASEIMAGE -ENV PATH /opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH="${PATH}:/opt/rh/gcc-toolset-12/root/usr/bin" ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 From 8ccafe950555e95702c1bdad5d0dbea7aec9dc6e Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 24 Jan 2024 17:47:06 -0800 Subject: [PATCH 04/19] Printing docker enviroemntal variables --- .github/workflows/linux-gpu-x64-build.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 3134e456c..4498e324f 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -47,15 +47,12 @@ jobs: docker run --rm onnxruntimegpubuild env echo "Running docker image onnxruntimegpubuild" docker run \ - --gpus all \ - --rm \ - --volume $GITHUB_WORKSPACE:/onnxruntime_src \ - --volume $RUNNER_TEMP:/build \ - -w /build onnxruntimegpubuild bash \ - -c "echo $PATH && /usr/bin/cmake \ - -e CUDA_PATH=/usr/local/cuda-12.2 \ - -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" - - - + --gpus all \ + --rm \ + --volume $GITHUB_WORKSPACE:/onnxruntime_src \ + --volume $RUNNER_TEMP:/build \ + -w /build onnxruntimegpubuild bash \ + -c "echo $PATH && /usr/bin/cmake \ + -e CUDA_PATH=/usr/local/cuda-12.2 \ + -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" From 83105d2d2354f1acaee4a741bf3a81bfa54a31e3 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 25 Jan 2024 20:06:50 -0800 Subject: [PATCH 05/19] Set CUDA_HOME --- .github/workflows/linux-gpu-x64-build.yml | 13 ++++++++----- .../docker/inference/x64/default/gpu/Dockerfile | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 4498e324f..c83a6a301 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -35,7 +35,7 @@ jobs: run: | git submodule update --init --recursive - - name: Build with CMake and GCC + - name: Download Docker Image run: | set -e -x python3 tools/ci_build/get_docker_image.py --dockerfile tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile \ @@ -43,16 +43,19 @@ jobs: --docker-build-args "--build-arg BUILD_UID=$( id -u )" \ --container-registry onnxruntimebuildcache \ --repository onnxruntimegpubuild + + - name: Print Docker Environment Variables + run: | echo "Printing docker environment variables" docker run --rm onnxruntimegpubuild env + + - name: Build with Cmake in Docker + run: | echo "Running docker image onnxruntimegpubuild" docker run \ --gpus all \ --rm \ --volume $GITHUB_WORKSPACE:/onnxruntime_src \ --volume $RUNNER_TEMP:/build \ - -w /build onnxruntimegpubuild bash \ - -c "echo $PATH && /usr/bin/cmake \ - -e CUDA_PATH=/usr/local/cuda-12.2 \ - -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" + -w /build onnxruntimegpubuild bash -c "echo $PATH && /usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile index d271ac821..bdbe58f38 100644 --- a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile @@ -8,6 +8,7 @@ FROM $BASEIMAGE ENV PATH="${PATH}:/opt/rh/gcc-toolset-12/root/usr/bin" ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 +ENV CUDA_HOME=/usr/local/cuda ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts From daa6ffefdb9f4ff21e5476b3f686c7a87660d913 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 25 Jan 2024 20:30:05 -0800 Subject: [PATCH 06/19] Reorder PATH --- .github/workflows/linux-gpu-x64-build.yml | 6 +++--- .../linux/docker/inference/x64/default/gpu/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index c83a6a301..65339412b 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -43,10 +43,10 @@ jobs: --docker-build-args "--build-arg BUILD_UID=$( id -u )" \ --container-registry onnxruntimebuildcache \ --repository onnxruntimegpubuild - - - name: Print Docker Environment Variables + + - name: Print Docker Image Environment Variables run: | - echo "Printing docker environment variables" + echo "Printing docker image environment variables" docker run --rm onnxruntimegpubuild env - name: Build with Cmake in Docker diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile index bdbe58f38..adfd6ae94 100644 --- a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile @@ -5,7 +5,7 @@ ARG BASEIMAGE=nvidia/cuda:12.2.2-cudnn8-devel-ubi8 FROM $BASEIMAGE -ENV PATH="${PATH}:/opt/rh/gcc-toolset-12/root/usr/bin" +ENV PATH="/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 ENV CUDA_HOME=/usr/local/cuda From c7561c306757d4b73833441f8d71faabf4759955 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 25 Jan 2024 21:17:21 -0800 Subject: [PATCH 07/19] Update cuda path --- .github/workflows/linux-gpu-x64-build.yml | 4 +--- .../github/linux/docker/inference/x64/default/gpu/Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 65339412b..39111da17 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -16,8 +16,6 @@ jobs: - name: Checkout OnnxRuntime GenAI repo uses: actions/checkout@v2 - - - name: Download OnnxRuntime run: | curl -L -o ${{ env.ort_zip }} ${{ env.ort_url }} @@ -57,5 +55,5 @@ jobs: --rm \ --volume $GITHUB_WORKSPACE:/onnxruntime_src \ --volume $RUNNER_TEMP:/build \ - -w /build onnxruntimegpubuild bash -c "echo $PATH && /usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release && ninja" + -w /build onnxruntimegpubuild bash -c "/usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DUSE_CUDA=1 -DCMAKE_BUILD_TYPE=Release && ninja" diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile index adfd6ae94..309988faa 100644 --- a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile @@ -5,10 +5,11 @@ ARG BASEIMAGE=nvidia/cuda:12.2.2-cudnn8-devel-ubi8 FROM $BASEIMAGE -ENV PATH="/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" +ENV PATH="/usr/local/cuda/binet:/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 ENV CUDA_HOME=/usr/local/cuda +ENV CUDNN_HOME=/usr/lib/x86_64-linux-gnu/ ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts From 85473e3817b47a250f7105c2b52ac91a975f3d62 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 25 Jan 2024 21:33:47 -0800 Subject: [PATCH 08/19] Update LD_LIBRARY_PATH --- .../github/linux/docker/inference/x64/default/gpu/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile index 309988faa..4fbf274e1 100644 --- a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile @@ -10,7 +10,7 @@ ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 ENV CUDA_HOME=/usr/local/cuda ENV CUDNN_HOME=/usr/lib/x86_64-linux-gnu/ - +ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts From 16ec3fd8bbdf4ea985e04e24a09ef30a065eb21c Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 25 Jan 2024 21:48:52 -0800 Subject: [PATCH 09/19] Update LD_LIBRARY_PATH --- .github/workflows/linux-gpu-x64-build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 39111da17..41da72708 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -47,6 +47,11 @@ jobs: echo "Printing docker image environment variables" docker run --rm onnxruntimegpubuild env + - name: Find Cuda Libraries + run: | + echo "Finding cuda libraries" + docker run --rm onnxruntimegpubuild find /usr/local /usr/lib /usr/lib64 -name "libcuda*" + - name: Build with Cmake in Docker run: | echo "Running docker image onnxruntimegpubuild" From 38976ead0b8f6c1e1299fb61817195bbe44f3539 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 25 Jan 2024 22:05:17 -0800 Subject: [PATCH 10/19] Update LD_LIBRARY_PATH --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c8602c3b8..7eb395135 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,10 +75,9 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) "${MODELS_ROOT}/*.cuh" ) list(APPEND generator_srcs ${generator_cuda_srcs}) - add_compile_definitions(USE_CUDA=1) - include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") + link_libraries(CUDA::cuda_library) else() file(GLOB generator_cuda_srcs "${GENERATORS_ROOT}/*_cuda*.*") list(REMOVE_ITEM generator_srcs ${generator_cuda_srcs}) @@ -114,6 +113,7 @@ add_executable (Tests ${test_srcs}) target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) + # pybind11_add_module(python ${python_srcs}) target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) From 538587efc9632ac88576fced6ac47c209fadc8e4 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 25 Jan 2024 23:16:55 -0800 Subject: [PATCH 11/19] Set CMAKE_CUDA_ARCHITECTURES --- .github/workflows/linux-gpu-x64-build.yml | 5 +++-- CMakeLists.txt | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 41da72708..2c6d08cd1 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -50,7 +50,8 @@ jobs: - name: Find Cuda Libraries run: | echo "Finding cuda libraries" - docker run --rm onnxruntimegpubuild find /usr/local /usr/lib /usr/lib64 -name "libcuda*" + docker run --rm onnxruntimegpubuild find / -name "libcuda*" + docker run --rm onnxruntimegpubuild ls /usr/local/cuda - name: Build with Cmake in Docker run: | @@ -60,5 +61,5 @@ jobs: --rm \ --volume $GITHUB_WORKSPACE:/onnxruntime_src \ --volume $RUNNER_TEMP:/build \ - -w /build onnxruntimegpubuild bash -c "/usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DUSE_CUDA=1 -DCMAKE_BUILD_TYPE=Release && ninja" + -w /build onnxruntimegpubuild bash -c "/usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_CUDA_ARCHITECTURES=75 -DUSE_CUDA=1 -DCMAKE_BUILD_TYPE=Release && ninja" diff --git a/CMakeLists.txt b/CMakeLists.txt index 7eb395135..9032e8f93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,6 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) message( STATUS "CMAKE_CUDA_COMPILER_VERSION: ${CMAKE_CUDA_COMPILER_VERSION}") # set(CUDA_PROPAGATE_HOST_FLAGS ON) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcudafe --diag_suppress=2803 --expt-relaxed-constexpr") - file(GLOB generator_cuda_srcs CONFIGURE_DEPENDS "${GENERATORS_ROOT}/*.cu" "${GENERATORS_ROOT}/*.cuh" @@ -77,7 +76,6 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) list(APPEND generator_srcs ${generator_cuda_srcs}) add_compile_definitions(USE_CUDA=1) include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") - link_libraries(CUDA::cuda_library) else() file(GLOB generator_cuda_srcs "${GENERATORS_ROOT}/*_cuda*.*") list(REMOVE_ITEM generator_srcs ${generator_cuda_srcs}) @@ -114,12 +112,19 @@ target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) + # pybind11_add_module(python ${python_srcs}) target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(python PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") +if(USE_CUDA AND CMAKE_CUDA_COMPILER) + set_target_properties(onnxruntime-gena PROPERTIES LINKER_LANGUAGE CUDA) + set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) + set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) +endif() + # Visual C++ - /MP Multiple process build, /WX Warnings as errors # Enable: From 2cf2a9f62079170149035ec5f61dcecb5279ff0e Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 26 Jan 2024 09:51:34 -0800 Subject: [PATCH 12/19] Fix typo --- CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9032e8f93..9f278cde5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,8 +111,7 @@ add_executable (Tests ${test_srcs}) target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) - - +set_directory_properties() # pybind11_add_module(python ${python_srcs}) target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) @@ -120,7 +119,7 @@ target_link_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(python PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") if(USE_CUDA AND CMAKE_CUDA_COMPILER) - set_target_properties(onnxruntime-gena PROPERTIES LINKER_LANGUAGE CUDA) + set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) endif() From 6ef13d18f9d73d58394a90b8ae96a8c3f8ac7e96 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 26 Jan 2024 10:15:54 -0800 Subject: [PATCH 13/19] remove set_directory_properties --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f278cde5..3f9571b8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,6 @@ add_executable (Tests ${test_srcs}) target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) -set_directory_properties() # pybind11_add_module(python ${python_srcs}) target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) From 0fa4f4ebf04a3c1adc151e615169dbce5e20b0a1 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 26 Jan 2024 10:32:17 -0800 Subject: [PATCH 14/19] Remove find cuda libraries --- .github/workflows/linux-gpu-x64-build.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 2c6d08cd1..4bde267a5 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -47,12 +47,6 @@ jobs: echo "Printing docker image environment variables" docker run --rm onnxruntimegpubuild env - - name: Find Cuda Libraries - run: | - echo "Finding cuda libraries" - docker run --rm onnxruntimegpubuild find / -name "libcuda*" - docker run --rm onnxruntimegpubuild ls /usr/local/cuda - - name: Build with Cmake in Docker run: | echo "Running docker image onnxruntimegpubuild" From 25f9cd5df074c36d83112f2d564b6bcddeffcab2 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 26 Jan 2024 10:50:07 -0800 Subject: [PATCH 15/19] Try CMAKE_CUDA_LIBRARIES --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f9571b8f..b6508cf01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,6 +121,11 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) + message(STATUS "Adding CUDA libraries") + message(STATUS "CMAKE_CUDA_LIBRARIES: ${CMAKE_CUDA_LIBRARIES}") + target_link_libraries(onnxruntime-genai PRIVATE ${CMAKE_CUDA_LIBRARIES}) + target_link_libraries(Tests PRIVATE ${CMAKE_CUDA_LIBRARIES}) + target_link_libraries(python PRIVATE ${CMAKE_CUDA_LIBRARIES}) endif() From c3c15844e15c6e6e2f6d16a9968bd0c7da8053b2 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 26 Jan 2024 11:09:22 -0800 Subject: [PATCH 16/19] Try cublasLt cublas cudnn curand cufft --- CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b6508cf01..8330e1650 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,11 +121,9 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) - message(STATUS "Adding CUDA libraries") - message(STATUS "CMAKE_CUDA_LIBRARIES: ${CMAKE_CUDA_LIBRARIES}") - target_link_libraries(onnxruntime-genai PRIVATE ${CMAKE_CUDA_LIBRARIES}) - target_link_libraries(Tests PRIVATE ${CMAKE_CUDA_LIBRARIES}) - target_link_libraries(python PRIVATE ${CMAKE_CUDA_LIBRARIES}) + target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft) + target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft) + target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft) endif() From c25d6b6c180780f08bdbcf34abd5eb790a185759 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 26 Jan 2024 11:46:08 -0800 Subject: [PATCH 17/19] Try CUDAHOSTCXX --- .../github/linux/docker/inference/x64/default/gpu/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile index 4fbf274e1..2ff6c38f5 100644 --- a/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile @@ -11,6 +11,7 @@ ENV LC_ALL=en_US.UTF-8 ENV CUDA_HOME=/usr/local/cuda ENV CUDNN_HOME=/usr/lib/x86_64-linux-gnu/ ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" +ENV CUDAHOSTCXX=/opt/rh/gcc-toolset-12/root/usr/bin/g++ ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts From 99f384bdadc83bca4435e5403f24528be62d24a8 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 26 Jan 2024 11:47:49 -0800 Subject: [PATCH 18/19] set(CMAKE_CUDA_RUNTIME_LIBRARY Shared) --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8330e1650..4dfae2cc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ include(CheckLanguage) check_language(CUDA) if (CMAKE_CUDA_COMPILER) message(STATUS "CUDA found CUDAToolkit_VERSION ${CUDAToolkit_VERSION} with CMAKE_CUDA_COMPILER ${CMAKE_CUDA_COMPILER}" ) + set(CMAKE_CUDA_RUNTIME_LIBRARY Shared) else() message(STATUS "CUDA not found") endif() From 585abd4987cd63189b014812022afd809caf7fd4 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 26 Jan 2024 13:31:16 -0800 Subject: [PATCH 19/19] set(CMAKE_CUDA_RUNTIME_LIBRARY Shared) --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4dfae2cc7..26a90c341 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,9 +122,9 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) - target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft) - target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft) - target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft) + target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) + target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) + target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) endif()