Skip to content

Commit

Permalink
SynapseAi 1.18.0 release
Browse files Browse the repository at this point in the history
 * Update dockerfiles with 1.18.0 content
  • Loading branch information
igor999999 committed Oct 11, 2024
1 parent 7e23f64 commit 45c162d
Show file tree
Hide file tree
Showing 40 changed files with 1,748 additions and 283 deletions.
19 changes: 18 additions & 1 deletion dockerfiles/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,27 @@ This script can be used as reference to build docker images for Gaudi.
make build
```
#### Build triton vllm backend (default OS - ubuntu22.04):
```
cd triton_vllm_backend
make build BUILD_OS=ubuntu22.04
```
3. Build command variables
#### Optional Parameters
* BUILD_OS - set the OS to build (default ubuntu22.04)
* BUILD_DIR - the folder where the build be executed from (default dockerbuild in image folder)
* VERBOSE - set to TRUE to echo the commands (default FALSE)
* DOCKER_CACHE - set to TRUE to use cache for building docker image (default FALSE)
* DOCKER_CACHE - set to TRUE to use cache for building docker image (default FALSE)
4. Instructions for triton-vllm-back-end server
* Run the backend container as described in [habana docs](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Triton_Inference.html?highlight=triton%20inference#run-the-backend-container)
* Start the triton server
```bash
tritonserver --model-repository samples/model_repository
```
The current samples/model_repository/vllm_model contains llama27B 1x.We also have sample model files for llama2 7b/70b and qwen2-7b respectively under samples/model_repository/test_models folder. To use them , copy the model.json and config.pbtxt to vllm_model folder structure.
* To test with client, please follow the instructions [here](https://github.com/triton-inference-server/vllm_backend?tab=readme-ov-file#sending-your-first-inference)
13 changes: 7 additions & 6 deletions dockerfiles/base/Dockerfile.amzn2
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,10 @@ RUN amazon-linux-extras enable python3.8 && \
wget \
lsof \
tar \
mesa-libGL && \
yum clean all && rm -rf /var/cache/yum
mesa-libGL \
sox-devel && \
yum clean all && rm -rf /var/cache/yum && \
rm -f /etc/ssh/ssh_host_*_key*

# Install jemalloc-3.6.0-1.el7.x86_64 package with required /lib64/libjemalloc.so.1 lib need for topologies
RUN yum install -y https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm && \
Expand All @@ -39,7 +41,7 @@ RUN yum install -y sudo system-lsb-core cmake
COPY install_efa.sh .
RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh

ENV LIBFABRIC_VERSION="1.20.0"
ENV LIBFABRIC_VERSION="1.22.0"
ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
ENV MPI_ROOT=/opt/amazon/openmpi
ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
Expand All @@ -55,7 +57,7 @@ ENV RDMA_CORE_LIB=${RDMA_CORE_ROOT}/build/lib
RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
echo "name=Habana AWS Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/AmazonLinux2" >> /etc/yum.repos.d/habanalabs.repo && \
echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/api/v2/repositories/AmazonLinux2/keyPairs/primary/public" >> /etc/yum.repos.d/habanalabs.repo
echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/AmazonLinux2/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo

RUN yum makecache && \
yum install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".amzn2 && \
Expand All @@ -74,7 +76,6 @@ RUN sed -i 's/[ #]\(.*StrictHostKeyChecking \).*/ \1no/g' /etc/ssh/ssh_config &&
sed -i 's/[ #]\(.*ForwardAgent \).*/ \1yes/g' /etc/ssh/ssh_config && \
echo " UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config && \
ssh-keygen -A && \
mkdir -p /var/run/sshd && echo "/usr/sbin/sshd -p 3022" | tee -a ~/.bashrc

# There is no need to store pip installation files inside docker image
Expand All @@ -94,7 +95,7 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
cd / && \
rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main

RUN python3 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
RUN python3 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0

RUN python3 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"

Expand Down
22 changes: 16 additions & 6 deletions dockerfiles/base/Dockerfile.rhel8.6
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ RUN dnf install -y \
llvm \
lsof \
python38-devel \
bzip2 \
bzip2-devel \
openssh-clients \
libjpeg-devel \
openssh-server \
Expand All @@ -50,7 +52,8 @@ RUN dnf install -y \
# update pkgs (except OS version) for resolving potentials CVEs
dnf versionlock add redhat-release* && \
dnf update -y && \
dnf clean all && rm -rf /var/cache/yum
dnf clean all && rm -rf /var/cache/yum && \
rm -f /etc/ssh/ssh_host_*_key*

# CVE-2023-47038 RHSA-2024:3128
RUN dnf module reset perl -y && \
Expand All @@ -67,20 +70,22 @@ RUN echo "[appstream]" > /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
COPY install_efa.sh .
RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh

ENV OPENMPI_VERSION=4.1.6
ENV LIBFABRIC_VERSION="1.20.0"
ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
ENV MPI_ROOT=/opt/amazon/openmpi
ENV MPI_ROOT=/opt/habanalabs/openmpi
ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
ENV PATH=${LIBFABRIC_ROOT}/bin:${MPI_ROOT}/bin:$PATH
ENV OPAL_PREFIX=${MPI_ROOT}
ENV MPICC=${MPI_ROOT}/bin/mpicc
ENV RDMAV_FORK_SAFE=1
ENV FI_EFA_USE_DEVICE_RDMA=1
ENV FI_EFA_USE_DEVICE_RDMA=0
ENV OMPI_MCA_btl=^openib

RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
echo "name=Habana RH8 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/rhel/8/8.6" >> /etc/yum.repos.d/habanalabs.repo && \
echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/api/v2/repositories/rhel/keyPairs/primary/public" >> /etc/yum.repos.d/habanalabs.repo
echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/rhel/8/8.6/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo

RUN echo "[powertools]" > /etc/yum.repos.d/powertools.repo && \
echo "name=powertools" >> /etc/yum.repos.d/powertools.repo && \
Expand Down Expand Up @@ -109,6 +114,12 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
make && make install && cd / && rm -rf /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 /tmp/libfabric-${LIBFABRIC_VERSION}

RUN wget -q -O /tmp/openmpi-${OPENMPI_VERSION}.tar.gz https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz && \
tar -xzf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz -C /tmp && \
cd /tmp/openmpi-${OPENMPI_VERSION} && \
./configure --prefix=${MPI_ROOT} --with-libfabric=$LIBFABRIC_ROOT --with-verbs && \
make -j$(nproc) && make install && cd / && rm -rf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz /tmp/openmpi-${OPENMPI_VERSION}

RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
unzip /tmp/main.zip -d /tmp && \
cd /tmp/hccl_ofi_wrapper-main && \
Expand All @@ -117,7 +128,7 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main

ENV PYTHON_VERSION=3.8
RUN python3.8 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
RUN python3.8 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0

RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 && \
alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 && \
Expand All @@ -131,7 +142,6 @@ RUN mkdir -p /var/run/sshd && \
sed -i 's/#\(ForwardAgent \).*/\1yes/g' /etc/ssh/ssh_config && \
echo " UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config && \
ssh-keygen -A && \
mkdir -p /var/run/sshd && echo "/usr/sbin/sshd -p 3022" | tee -a ~/.bashrc

ENV GC_KERNEL_PATH=/usr/lib/habanalabs/libtpc_kernels.so
Expand Down
27 changes: 18 additions & 9 deletions dockerfiles/base/Dockerfile.rhel9.2
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ RUN dnf install -y \
wget \
git \
libffi-devel \
bzip2 \
bzip2-devel \
zlib-devel \
mesa-libGL \
Expand All @@ -61,7 +62,8 @@ RUN dnf install -y \
# update pkgs (except OS version) for resolving potentials CVEs
dnf versionlock add redhat-release* && \
dnf update -y && \
dnf clean all && rm -rf /var/cache/yum
dnf clean all && rm -rf /var/cache/yum && \
rm -f /etc/ssh/ssh_host_*_key*

ENV PYTHON_VERSION=3.10
COPY install-python310.sh .
Expand All @@ -71,29 +73,31 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
COPY install_efa.sh .
RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh

ENV OPENMPI_VERSION=4.1.6
ENV LIBFABRIC_VERSION="1.20.0"
ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
ENV MPI_ROOT=/opt/amazon/openmpi
ENV MPI_ROOT=/opt/habanalabs/openmpi
ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
ENV PATH=${LIBFABRIC_ROOT}/bin:${MPI_ROOT}/bin:$PATH
ENV OPAL_PREFIX=${MPI_ROOT}
ENV MPICC=${MPI_ROOT}/bin/mpicc
ENV RDMAV_FORK_SAFE=1
ENV FI_EFA_USE_DEVICE_RDMA=1
ENV FI_EFA_USE_DEVICE_RDMA=0
ENV OMPI_MCA_btl=^openib

RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
echo "name=Habana RH9 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/rhel/9/9.2" >> /etc/yum.repos.d/habanalabs.repo && \
echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/api/v2/repositories/rhel/keyPairs/primary/public" >> /etc/yum.repos.d/habanalabs.repo && \
echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/rhel/9/9.2/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo && \
echo "gpgcheck=1" >> /etc/yum.repos.d/habanalabs.repo

# for Habana GPG key with SHA-1 signature
RUN update-crypto-policies --set DEFAULT:SHA1

RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el9 \
habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
rm -f /etc/yum.repos.d/habanalabs.repo && rm -f /etc/yum.repos.d/habana.repo && rm -rf /tmp/* && \
dnf clean all && rm -rf /var/cache/yum

Expand All @@ -111,14 +115,20 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
make && make install && cd / && rm -rf /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 /tmp/libfabric-${LIBFABRIC_VERSION}

RUN wget -q -O /tmp/openmpi-${OPENMPI_VERSION}.tar.gz https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz && \
tar -xzf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz -C /tmp && \
cd /tmp/openmpi-${OPENMPI_VERSION} && \
./configure --prefix=${MPI_ROOT} --with-libfabric=$LIBFABRIC_ROOT --with-verbs && \
make -j$(nproc) && make install && cd / && rm -rf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz /tmp/openmpi-${OPENMPI_VERSION}

RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
unzip /tmp/main.zip -d /tmp && \
cd /tmp/hccl_ofi_wrapper-main && \
make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
cd / && \
rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main

RUN python3.10 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
RUN python3.10 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0

RUN ln -s /usr/bin/python3 /usr/bin/python

Expand All @@ -130,7 +140,6 @@ RUN mkdir -p /var/run/sshd && \
sed -i 's/#\(ForwardAgent \).*/\1yes/g' /etc/ssh/ssh_config && \
echo " UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config && \
ssh-keygen -A && \
mkdir -p /var/run/sshd && echo "/usr/sbin/sshd -p 3022" | tee -a ~/.bashrc

ENV GC_KERNEL_PATH=/usr/lib/habanalabs/libtpc_kernels.so
Expand Down
37 changes: 23 additions & 14 deletions dockerfiles/base/Dockerfile.rhel9.4
Original file line number Diff line number Diff line change
Expand Up @@ -51,28 +51,30 @@ RUN dnf install -y \
lsof \
python3-devel \
openssh-clients \
openssl-1:3.0.7-27.el9 \
openssl-devel-1:3.0.7-27.el9 \
openssl-1:3.0.7-28.el9_4 \
openssl-devel-1:3.0.7-28.el9_4 \
libjpeg-devel \
openssh-server \
lsb_release \
wget \
git \
libffi-devel \
bzip2 \
bzip2-devel \
zlib-devel \
mesa-libGL \
iproute \
python3.11 \
python3.11-pip \
python3.11-devel \
python3.11-rpm \
ffmpeg-free \
perl-Net-SSLeay-1.92-2.el9 \
python3-dnf-plugin-versionlock && \
# update pkgs (except OS version) for resolving potentials CVEs
dnf versionlock add redhat-release* openssl* perl-Net-SSLeay && \
dnf versionlock add redhat-release* openssl* libcurl-minimal curl-minimal ima-evm-utils python3-rpm rpm* && \
dnf update -y && \
dnf clean all && rm -rf /var/cache/yum
dnf clean all && rm -rf /var/cache/yum && \
rm -f /etc/ssh/ssh_host_*_key*

RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2 && \
alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 && \
Expand All @@ -84,29 +86,31 @@ RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2 && \
COPY install_efa.sh .
RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh

ENV LIBFABRIC_VERSION="1.20.0"
ENV OPENMPI_VERSION=4.1.6
ENV LIBFABRIC_VERSION="1.22.0"
ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
ENV MPI_ROOT=/opt/amazon/openmpi
ENV MPI_ROOT=/opt/habanalabs/openmpi
ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
ENV PATH=${LIBFABRIC_ROOT}/bin:${MPI_ROOT}/bin:$PATH
ENV OPAL_PREFIX=${MPI_ROOT}
ENV MPICC=${MPI_ROOT}/bin/mpicc
ENV RDMAV_FORK_SAFE=1
ENV FI_EFA_USE_DEVICE_RDMA=1
ENV FI_EFA_USE_DEVICE_RDMA=0
ENV OMPI_MCA_btl=^openib

RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
echo "name=Habana RH9 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/rhel/9/9.4" >> /etc/yum.repos.d/habanalabs.repo && \
echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/api/v2/repositories/rhel/keyPairs/primary/public" >> /etc/yum.repos.d/habanalabs.repo && \
echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/rhel/9/9.4/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo && \
echo "gpgcheck=1" >> /etc/yum.repos.d/habanalabs.repo

# for Habana GPG key with SHA-1 signature
RUN update-crypto-policies --set DEFAULT:SHA1

RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el9 \
habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
rm -f /etc/yum.repos.d/habanalabs.repo && rm -f /etc/yum.repos.d/habana.repo && rm -rf /tmp/* && \
dnf clean all && rm -rf /var/cache/yum

Expand All @@ -124,14 +128,20 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
make && make install && cd / && rm -rf /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 /tmp/libfabric-${LIBFABRIC_VERSION}

RUN wget -q -O /tmp/openmpi-${OPENMPI_VERSION}.tar.gz https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz && \
tar -xzf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz -C /tmp && \
cd /tmp/openmpi-${OPENMPI_VERSION} && \
./configure --prefix=${MPI_ROOT} --with-libfabric=$LIBFABRIC_ROOT --with-verbs && \
make -j$(nproc) && make install && cd / && rm -rf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz /tmp/openmpi-${OPENMPI_VERSION}

RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
unzip /tmp/main.zip -d /tmp && \
cd /tmp/hccl_ofi_wrapper-main && \
make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
cd / && \
rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main

RUN python3.11 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
RUN python3.11 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0

RUN ln -s /usr/bin/python3 /usr/bin/python

Expand All @@ -143,7 +153,6 @@ RUN mkdir -p /var/run/sshd && \
sed -i 's/#\(ForwardAgent \).*/\1yes/g' /etc/ssh/ssh_config && \
echo " UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config && \
ssh-keygen -A && \
mkdir -p /var/run/sshd && echo "/usr/sbin/sshd -p 3022" | tee -a ~/.bashrc

ENV GC_KERNEL_PATH=/usr/lib/habanalabs/libtpc_kernels.so
Expand Down
Loading

0 comments on commit 45c162d

Please sign in to comment.