Merge pull request #973 from anarkiwi/orin

build Orin torchserve.

.github/workflows/docker-extras.yml

@@ -40,6 +40,40 @@ jobs:
         push: true
         tags: iqtlabs/gamutrf-torchserve:${{ steps.change_version.outputs.VERSION }}
       if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push'
+  buildx-orin-torchserve:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Get the version
+      id: get_version
+      run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3)
+    - name: Change for main
+      id: change_version
+      run: if [ "${{ steps.get_version.outputs.VERSION }}" == "main" ]; then echo ::set-output name=VERSION::latest; else echo ::set-output name=VERSION::${{ steps.get_version.outputs.VERSION }}; fi
+    - name: Set up qemu
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: all
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Docker Login
+      env:
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
+      run: |
+        echo "${DOCKER_PASSWORD}" | docker login --username "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+      if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push'
+    - name: Build and push platforms
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: docker/Dockerfile.orin-torchserve
+        platforms: linux/arm64
+        push: true
+        tags: iqtlabs/gamutrf-orin-torchserve:${{ steps.change_version.outputs.VERSION }}
+      if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push'
   buildx-cuda-torchserve:
     runs-on: ubuntu-latest
     steps:

.github/workflows/docker-test.yml

@@ -1,6 +1,26 @@
 name: docker-test
 on: [push, pull_request]
 jobs:
+  test-orin-torchserve:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up qemu
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: all
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Build
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: docker/Dockerfile.orin-torchserve
+        platforms: linux/arm64
+        push: false
   test-gamutrf-torchserve-image:
     runs-on: ubuntu-latest
     steps:

.github/workflows/docker.yml

@@ -75,7 +75,6 @@ jobs:
       run: |
         echo "${DOCKER_PASSWORD}" | docker login --username "${{ secrets.DOCKER_USERNAME }}" --password-stdin
       if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push'
-
     - name: Build and push platforms
       uses: docker/build-push-action@v5
       with:

docker/Dockerfile.orin-torchserve

@@ -0,0 +1,9 @@
+FROM nvcr.io/nvidia/l4t-pytorch:r35.2.1-pth2.0-py3
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /root
+COPY torchserve/install-orin-torchserve.sh /torchserve/install-orin-torchserve.sh
+RUN /torchserve/install-orin-torchserve.sh
+RUN /usr/local/bin/torchserve --help
+COPY torchserve/config.properties /torchserve/config.properties
+COPY torchserve/torchserve-entrypoint.sh /torchserve/torchserve-entrypoint.sh
+CMD ["/torchserve/torchserve-entrypoint.sh"]

tests/test_torchserve.sh

@@ -12,7 +12,8 @@ wget https://raw.githubusercontent.com/pytorch/serve/master/examples/object_dete
 torch-model-archiver --force --model-name yolov8n --version 1.0 --serialized-file yolov8n.pt --handler custom_handler.py -r requirements.txt
 rm -rf model_store && mkdir model_store
 mv yolov8n.mar model_store/
-docker run -v $(pwd)/model_store:/model_store --net host --entrypoint timeout -d iqtlabs/gamutrf-torchserve 180s /torchserve/torchserve-entrypoint.sh --models yolov8n=yolov8n.mar
+# TODO: --runtime nvidia is required for Orin
+docker run -v $(pwd)/model_store:/model_store -p 8080:8080 --rm --name testts --entrypoint timeout -d iqtlabs/gamutrf-torchserve 180s /torchserve/torchserve-entrypoint.sh --models yolov8n=yolov8n.mar
 # TODO: use gamutRF test spectogram image
 wget https://github.com/pytorch/serve/raw/master/examples/object_detector/yolo/yolov8/persons.jpg
 wget -q --retry-connrefused --retry-on-host-error --body-file=persons.jpg --method=PUT -O- --header='Content-Type: image/jpg' http://127.0.0.1:8080/predictions/yolov8n | jq

torchserve/install-orin-torchserve.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+apt-get update && \
+  apt-get install -y \
+    git \
+    python3-pip \
+    openjdk-17-jdk \
+    wget \
+    numactl
+  apt-get purge -y opencv-python && \
+  pip install "opencv-python<4.8.0.76" && \
+  echo /usr/local/lib/python3.8/dist-packages/opencv_python.libs > /etc/ld.so.conf.d/opencv.conf && \
+  ldconfig && \
+  pip config set global.no-cache-dir false && \
+  git clone https://github.com/pytorch/serve -b v0.9.0 && \
+  cd serve && \
+  pip install -r requirements/common.txt && \
+  pip install -r requirements/common_gpu.txt && \
+  pip install -r requirements/torch_common.txt && \
+  pip install . && \
+  cd .. && \
+  rm -rf serve