Ga build docker images (quic#3351)

* Implement pyproject.toml to build AIMET package There are 3 dynamic fields in metadata: - version - dependencies - name (not PEP compatible!) A plugin of scikit-build-core build system generates dependencies and package name based on `CMAKE_ARGS` environment variable. Signed-off-by: Evgeny Mironov <[email protected]> * Build all docker images from the single Dockerfile Docker images contain dependencies to build and run tests. Signed-off-by: Evgeny Mironov <[email protected]> * Build docker images on a CI Signed-off-by: Evgeny Mironov <[email protected]> --------- Signed-off-by: Evgeny Mironov <[email protected]> Co-authored-by: Evgeny Mironov <[email protected]>
dwelsch-esi · Sep 19, 2024 · 6ad7365 · 6ad7365
1 parent 75f0682
commit 6ad7365
Show file tree

Hide file tree

Showing 10 changed files with 471 additions and 0 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,9 @@
+# Ignore everything
+*
+
+# Except few files and directories
+!Jenkins/fast-release/environment.devenv.yml
+!packaging/dependencies
+!packaging/plugins
+!packaging/version.txt
+!pyproject.toml
diff --git a/.github/actions/docker-build-image/action.yml b/.github/actions/docker-build-image/action.yml
@@ -0,0 +1,77 @@
+name: "Build docker image if tag is not 'latest'"
+
+description: "Build docker image if tag is not 'latest'"
+
+inputs:
+  docker-registry:
+    description: "Docker registry"
+    required: true
+  docker-login:
+    description: "Docker login"
+    required: true
+  docker-password:
+    description: "Docker password"
+    required: true
+  dockercontext:
+    description: "Docker build context"
+    required: false
+    default: "."
+  dockerfile:
+    description: "Dockerfile"
+    required: false
+    default: "Dockerfile"
+  image-name:
+    description: "Docker image name"
+    required: false
+    default: "${{ github.event.repository.name }}"
+  image-tag:
+    description: "Docker image tag"
+    required: false
+    default: "latest"
+  build-args:
+    description: "Docker build argunebts"
+    required: false
+    default: ""
+
+outputs:
+  docker-image:
+    description: "Docker image"
+    value: ${{ steps.image.outputs.value }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Set docker image tag"
+      id: image
+      shell: bash
+      run: echo "value=${{ inputs.docker-registry }}/${{ inputs.image-name }}:${{ inputs.image-tag }}" >> $GITHUB_OUTPUT
+
+    - name: "Set DOCKER_CONFIG because buildx stores data there"
+      if: inputs.image-tag != 'latest'
+      shell: bash
+      run: echo "DOCKER_CONFIG=./.docker" >> $GITHUB_ENV
+
+    - name: "Set up Docker Buildx"
+      if: inputs.image-tag != 'latest'
+      uses: docker/setup-buildx-action@v3
+      with:
+        buildkitd-flags: --debug
+        driver: docker
+
+    - name: "Login to docker registry"
+      if: inputs.image-tag != 'latest'
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ inputs.docker-registry }}
+        username: ${{ inputs.docker-login }}
+        password: ${{ inputs.docker-password }}
+
+    - name: "Build and push"
+      if: inputs.image-tag != 'latest'
+      uses: docker/build-push-action@v6
+      with:
+        context: ${{ inputs.dockercontext }}
+        file: ${{ inputs.dockerfile }}
+        tags: ${{ steps.image.outputs.value }}
+        build-args: ${{ inputs.build-args }}
+        push: true
diff --git a/.github/actions/docker-tag/action.yml b/.github/actions/docker-tag/action.yml
@@ -0,0 +1,54 @@
+name: "Docker image tag"
+
+description: "Get a docker image tag based on changes in a docker build context"
+
+inputs:
+  dockercontext:
+    description: "Docker build context"
+    required: false
+    default: "."
+  dockerfile:
+    description: "Dockerfile"
+    required: false
+    default: "Dockerfile"
+  dockerignore:
+    description: "Dockerignore file"
+    required: false
+    default: ".dockerignore"
+  defaulttag:
+    description: "A default docker image tag"
+    required: false
+    default: "latest"
+
+outputs:
+  tag:
+    description: "Docker image tag"
+    value: ${{ steps.tag.outputs.value }}
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v4
+
+    - name: "Download '.dockerignore' file parser"
+      shell: bash
+      run: |
+        curl -L "https://github.com/johnstairs/dockerignore-filter/releases/download/v0.1.6/dockerignore-filter_Linux_x86_64" -o dockerignore-filter
+        chmod +x dockerignore-filter
+
+    - name: "Get list of files from the docker build context (including Dockerfile and .dockerignore)"
+      shell: bash
+      run: echo "DOCKER_BUILD_CONTEXT_FILES=$(find ${{ inputs.dockercontext }} -type f | ./dockerignore-filter ${{ inputs.dockercontext }}/${{ inputs.dockerignore }} | tr '\n' ' ') ${{ inputs.dockercontext }}/${{ inputs.dockerfile }} ${{ inputs.dockercontext }}/${{ inputs.dockerignore }}" >> $GITHUB_ENV
+
+    - name: "Get list of changes files in the docker build context"
+      shell: bash
+      run: |
+        git branch --delete --force ${{ github.event.repository.default_branch }} || true
+        git fetch --no-tags --force --prune --no-recurse-submodules --depth=1 origin ${{ github.event.repository.default_branch }}:refs/remotes/origin/${{ github.event.repository.default_branch }}
+        echo "DOCKER_BUILD_CONTEXT_FILES=$(git diff --name-only origin/${{ github.event.repository.default_branch }} -- $DOCKER_BUILD_CONTEXT_FILES | tr '\n' ' ')" >> $GITHUB_ENV
+
+    - name: "Set a docker image tag"
+      id: tag
+      shell: bash
+      run: echo "value=$([[ ! -z \"$DOCKER_BUILD_CONTEXT_FILES\" ]] && echo $(git rev-parse --short HEAD) || echo ${{ inputs.defaulttag }})" >> $GITHUB_OUTPUT
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,11 @@
+name: CI pipeline
+
+on:
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  pipeline:
+    if: github.server_url != 'https://github.com'
+    uses: ./.github/workflows/pipeline.yml
+    secrets: inherit
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
@@ -0,0 +1,101 @@
+name: CI Pipeline
+
+on:
+  workflow_call:
+
+jobs:
+  docker-tag:
+    name: Check if 'latest' tag could be used (no build docker images)
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+    steps:
+      - run: |
+          sudo apt update -qq
+          sudo apt install -y git curl jq sudo ca-certificates
+          sudo update-ca-certificates
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/docker-tag
+        id: tag
+        with:
+          dockerfile: Jenkins/fast-release/Dockerfile.ci
+
+  variants:
+    name: Define supported AIMET variants
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.final.outputs.value }}
+    steps:
+      - run: |
+          sudo apt update -qq
+          sudo apt install -y git curl jq sudo ca-certificates
+          sudo update-ca-certificates
+
+      - name: Torch variants
+        run: |
+          set -x
+          VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"",       "VER_TORCH":"2.1.2", "VER_ONNX":"", "VER_CUDA":""       },
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"",       "VER_TORCH":"2.1.2", "VER_ONNX":"", "VER_CUDA":"11.8.0" }
+            ]')
+          echo "VALUE=$VALUE" >> $GITHUB_ENV
+
+      - name: Tensorflow variants
+        run: |
+          set -x
+          VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"2.10.1", "VER_TORCH":"",      "VER_ONNX":"", "VER_CUDA":"" },
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"2.10.1", "VER_TORCH":"",      "VER_ONNX":"", "VER_CUDA":"11.8.0" }
+            ]')
+          echo "VALUE=$VALUE" >> $GITHUB_ENV
+
+      - name: ONNX variants
+        run: |
+          set -x
+          VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"",       "VER_TORCH":"1.13.1", "VER_ONNX":"1.14.1", "VER_CUDA":""       },
+              { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"",       "VER_TORCH":"1.13.1", "VER_ONNX":"1.14.1", "VER_CUDA":"11.7.1" }
+            ]')
+          echo "VALUE=$VALUE" >> $GITHUB_ENV
+
+      - name: (Last step) Generate few extra properties for each variant
+        id: final
+        run: |
+          set -x
+          VALUE=$(echo "$VALUE" | jq -c '.include[] |= . + {
+            "runs-on":(if .VER_CUDA != "" then "k8s-gpu" else "ubuntu-latest" end),
+            "id":(""
+                    +(if .VER_TENSORFLOW != "" then "tf-" else "" end)
+                    +(if .VER_ONNX != "" then "onnx-" elif .VER_TORCH != "" then "torch-" else "" end)
+                    +(if .VER_CUDA != "" then "gpu" else "cpu" end)
+                 )
+            }')
+          echo "value=$VALUE" >> $GITHUB_OUTPUT
+
+  docker-build-image:
+    name: Docker image ${{ matrix.id }}
+    runs-on: ubuntu-latest
+    needs: [docker-tag, variants]
+    strategy:
+      matrix: ${{ fromJSON(needs.variants.outputs.matrix) }}
+    steps:
+      - run: |
+          sudo apt update -qq
+          sudo apt install -y git curl jq sudo ca-certificates
+          sudo update-ca-certificates
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/docker-build-image
+        with:
+          dockerfile: Jenkins/fast-release/Dockerfile.ci
+          docker-login: ${{ secrets.DOCKER_LOGIN }}
+          docker-password: ${{ secrets.DOCKER_CREDENTIALS }}
+          docker-registry: ${{ secrets.DOCKER_REGISTRY }}
+          image-name: "${{ secrets.DOCKER_IMAGE }}-${{ matrix.id }}"
+          image-tag: ${{ needs.docker-tag.outputs.tag }}
+          build-args: |
+            VER_PYTHON=${{ matrix.VER_PYTHON }}
+            VER_CUDA=${{ matrix.VER_CUDA }}
+            VER_TORCH=${{ matrix.VER_TORCH }}
+            VER_TENSORFLOW=${{ matrix.VER_TENSORFLOW }}
+            VER_ONNX=${{ matrix.VER_ONNX }}
+
diff --git a/Jenkins/fast-release/Dockerfile.ci b/Jenkins/fast-release/Dockerfile.ci
@@ -0,0 +1,73 @@
+ARG BASE_IMAGE=ubuntu:22.04
+FROM $BASE_IMAGE AS build
+
+ENV CONDA_PREFIX=/opt/conda
+ENV CONDA=${CONDA_PREFIX}/bin/conda
+ENV CONDA_DEFAULT_ENV=dev
+
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive apt install --yes --no-install-recommends \
+        acl \
+        ca-certificates \
+        curl \
+        # manylinux2014 requires gcc 10 and cuda doesn't support gcc>11
+        g++-10 \
+        git \
+        jq \
+        make \
+        sudo \
+    && rm -rf /var/lib/apt/lists/* \
+    && echo '%users ALL = (ALL) NOPASSWD: ALL' > /etc/sudoers.d/passwordless \
+    && curl -o /tmp/conda.sh -L 'https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh' \
+    && mkdir -m 777 -p ${CONDA_PREFIX} \
+    && setfacl -d -m o::rwx ${CONDA_PREFIX} \
+    && bash /tmp/conda.sh -u -b -p ${CONDA_PREFIX} \
+    && rm /tmp/conda.sh \
+    && ${CONDA} config --set channel_priority strict \
+    && ${CONDA} init --no-user --system --all \
+    && ${CONDA} install -y conda-devenv \
+    && ${CONDA} clean --yes --all --verbose \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \
+    && git config --system --add safe.directory '*'
+
+ARG VER_PYTHON=3.8
+ARG VER_CUDA=11.7.1
+ARG VER_TORCH=""
+ARG VER_TENSORFLOW=""
+ARG VER_ONNX=""
+
+COPY Jenkins/fast-release/environment.devenv.yml /tmp/
+RUN export PATH=$PATH:${CONDA_PREFIX}/bin PIP_NO_CACHE_DIR=1 \
+    && ${CONDA} devenv \
+        --env-var ENV_NAME="${CONDA_DEFAULT_ENV}" \
+        --env-var VER_PYTHON="${VER_PYTHON}" \
+        --env-var VER_CUDA="${VER_CUDA}" \
+        --file /tmp/environment.devenv.yml \
+        --output-file /tmp/environment.yml \
+    && cat /tmp/environment.yml \
+    && ${CONDA} clean --yes --all --verbose \
+    && echo "conda activate ${CONDA_DEFAULT_ENV}" >> /etc/profile.d/conda.sh \
+    && rm -rf ~/.conda*
+
+RUN --mount=type=bind,target=/workspace \
+    echo "Install all required dependencies" \
+    && export PIP_CACHE_DIR=/tmp/pip-cache BUILD_DIR=/tmp/build \
+    && mkdir -p $BUILD_DIR \
+    && export PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl" \
+    && export CMAKE_ARGS="\
+        -DENABLE_TENSORFLOW=$([ -z ${VER_TENSORFLOW} ]; echo $?) \
+        -DENABLE_TORCH=$([ -z ${VER_TORCH} ]; echo $?) \
+        -DENABLE_ONNX=$([ -z ${VER_ONNX} ]; echo $?) \
+        -DENABLE_CUDA=$([ -z ${VER_CUDA} ]; echo $?) \
+       " \
+    && ${CONDA} run --name ${CONDA_DEFAULT_ENV} --live-stream \
+        python3 -m pip install --dry-run --report $BUILD_DIR/pip-report.json -C build-dir="$BUILD_DIR/{wheel_tag}" /workspace \
+    && ${CONDA} run --name ${CONDA_DEFAULT_ENV} --live-stream \
+        python3 -m pip install -c /workspace/packaging/dependencies/constraints.txt $(jq -r '.install[0].metadata.requires_dist[] | split(";") | .[0]' $BUILD_DIR/pip-report.json) \
+    && rm -rf $PIP_CACHE_DIR $BUILD_DIR
+
+ENV PYTHONPYCACHEPREFIX=/tmp
+
+ENTRYPOINT ["/bin/bash", "--login", "-c", "${0#--} \"$@\""]
+CMD ["/bin/bash"]
+
diff --git a/Jenkins/fast-release/environment.devenv.yml b/Jenkins/fast-release/environment.devenv.yml
@@ -0,0 +1,29 @@
+{% set ENV_NAME         = os.environ.get('ENV_NAME', 'dev') %}
+
+{% set VER_PYTHON       = os.environ.get('VER_PYTHON') %}
+{% set VER_CUDA         = os.environ.get('VER_CUDA') %}
+
+{% set CUDA_CHANNEL     = 'nvidia/label/cuda-' + VER_CUDA %}
+{% set CU               = 'cu' + ''.join(VER_CUDA.split('.')[:-1]) if VER_CUDA != '' else 'cpu' %}
+
+
+name: {{ ENV_NAME }}
+
+{% if CU != 'cpu' %}
+channels:
+  - {{ CUDA_CHANNEL }}
+{% endif %}
+
+dependencies:
+  - auditwheel
+  - patchelf
+  - python={{ VER_PYTHON }}
+  - python-build
+  - pip
+{% if CU != 'cpu' %}
+  - cuda-gdb
+  - cuda-nvcc
+  - cuda-nvtx
+  - cuda-libraries-dev
+  - cudnn
+{% endif %}
diff --git a/packaging/dependencies/constraints.txt b/packaging/dependencies/constraints.txt
@@ -0,0 +1,3 @@
+# pytorch uses setuptools to build C++ extensions
+# [ImportError: cannot import name 'packaging' from 'pkg_resources']
+setuptools<70