From 6ad73652b9560505dc8b337be3ba83f3d08be25b Mon Sep 17 00:00:00 2001 From: Hitarth Mehta Date: Wed, 18 Sep 2024 18:42:23 -0700 Subject: [PATCH] Ga build docker images (#3351) * Implement pyproject.toml to build AIMET package There are 3 dynamic fields in metadata: - version - dependencies - name (not PEP compatible!) A plugin of scikit-build-core build system generates dependencies and package name based on `CMAKE_ARGS` environment variable. Signed-off-by: Evgeny Mironov * Build all docker images from the single Dockerfile Docker images contain dependencies to build and run tests. Signed-off-by: Evgeny Mironov * Build docker images on a CI Signed-off-by: Evgeny Mironov --------- Signed-off-by: Evgeny Mironov Co-authored-by: Evgeny Mironov --- .dockerignore | 9 ++ .github/actions/docker-build-image/action.yml | 77 +++++++++++++ .github/actions/docker-tag/action.yml | 54 ++++++++++ .github/workflows/ci.yml | 11 ++ .github/workflows/pipeline.yml | 101 ++++++++++++++++++ Jenkins/fast-release/Dockerfile.ci | 73 +++++++++++++ Jenkins/fast-release/environment.devenv.yml | 29 +++++ packaging/dependencies/constraints.txt | 3 + packaging/plugins/local/aimet.py | 74 +++++++++++++ pyproject.toml | 40 +++++++ 10 files changed, 471 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/actions/docker-build-image/action.yml create mode 100644 .github/actions/docker-tag/action.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/pipeline.yml create mode 100644 Jenkins/fast-release/Dockerfile.ci create mode 100644 Jenkins/fast-release/environment.devenv.yml create mode 100644 packaging/dependencies/constraints.txt create mode 100644 packaging/plugins/local/aimet.py create mode 100644 pyproject.toml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..e776926377e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +# Ignore everything +* + +# Except few files and directories +!Jenkins/fast-release/environment.devenv.yml +!packaging/dependencies +!packaging/plugins +!packaging/version.txt +!pyproject.toml diff --git a/.github/actions/docker-build-image/action.yml b/.github/actions/docker-build-image/action.yml new file mode 100644 index 00000000000..d974fd78cf9 --- /dev/null +++ b/.github/actions/docker-build-image/action.yml @@ -0,0 +1,77 @@ +name: "Build docker image if tag is not 'latest'" + +description: "Build docker image if tag is not 'latest'" + +inputs: + docker-registry: + description: "Docker registry" + required: true + docker-login: + description: "Docker login" + required: true + docker-password: + description: "Docker password" + required: true + dockercontext: + description: "Docker build context" + required: false + default: "." + dockerfile: + description: "Dockerfile" + required: false + default: "Dockerfile" + image-name: + description: "Docker image name" + required: false + default: "${{ github.event.repository.name }}" + image-tag: + description: "Docker image tag" + required: false + default: "latest" + build-args: + description: "Docker build argunebts" + required: false + default: "" + +outputs: + docker-image: + description: "Docker image" + value: ${{ steps.image.outputs.value }} + +runs: + using: "composite" + steps: + - name: "Set docker image tag" + id: image + shell: bash + run: echo "value=${{ inputs.docker-registry }}/${{ inputs.image-name }}:${{ inputs.image-tag }}" >> $GITHUB_OUTPUT + + - name: "Set DOCKER_CONFIG because buildx stores data there" + if: inputs.image-tag != 'latest' + shell: bash + run: echo "DOCKER_CONFIG=./.docker" >> $GITHUB_ENV + + - name: "Set up Docker Buildx" + if: inputs.image-tag != 'latest' + uses: docker/setup-buildx-action@v3 + with: + buildkitd-flags: --debug + driver: docker + + - name: "Login to docker registry" + if: inputs.image-tag != 'latest' + uses: docker/login-action@v3 + with: + registry: ${{ inputs.docker-registry }} + username: ${{ inputs.docker-login }} + password: ${{ inputs.docker-password }} + + - name: "Build and push" + if: inputs.image-tag != 'latest' + uses: docker/build-push-action@v6 + with: + context: ${{ inputs.dockercontext }} + file: ${{ inputs.dockerfile }} + tags: ${{ steps.image.outputs.value }} + build-args: ${{ inputs.build-args }} + push: true diff --git a/.github/actions/docker-tag/action.yml b/.github/actions/docker-tag/action.yml new file mode 100644 index 00000000000..fde3e545d3f --- /dev/null +++ b/.github/actions/docker-tag/action.yml @@ -0,0 +1,54 @@ +name: "Docker image tag" + +description: "Get a docker image tag based on changes in a docker build context" + +inputs: + dockercontext: + description: "Docker build context" + required: false + default: "." + dockerfile: + description: "Dockerfile" + required: false + default: "Dockerfile" + dockerignore: + description: "Dockerignore file" + required: false + default: ".dockerignore" + defaulttag: + description: "A default docker image tag" + required: false + default: "latest" + +outputs: + tag: + description: "Docker image tag" + value: ${{ steps.tag.outputs.value }} + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - name: "Download '.dockerignore' file parser" + shell: bash + run: | + curl -L "https://github.com/johnstairs/dockerignore-filter/releases/download/v0.1.6/dockerignore-filter_Linux_x86_64" -o dockerignore-filter + chmod +x dockerignore-filter + + - name: "Get list of files from the docker build context (including Dockerfile and .dockerignore)" + shell: bash + run: echo "DOCKER_BUILD_CONTEXT_FILES=$(find ${{ inputs.dockercontext }} -type f | ./dockerignore-filter ${{ inputs.dockercontext }}/${{ inputs.dockerignore }} | tr '\n' ' ') ${{ inputs.dockercontext }}/${{ inputs.dockerfile }} ${{ inputs.dockercontext }}/${{ inputs.dockerignore }}" >> $GITHUB_ENV + + - name: "Get list of changes files in the docker build context" + shell: bash + run: | + git branch --delete --force ${{ github.event.repository.default_branch }} || true + git fetch --no-tags --force --prune --no-recurse-submodules --depth=1 origin ${{ github.event.repository.default_branch }}:refs/remotes/origin/${{ github.event.repository.default_branch }} + echo "DOCKER_BUILD_CONTEXT_FILES=$(git diff --name-only origin/${{ github.event.repository.default_branch }} -- $DOCKER_BUILD_CONTEXT_FILES | tr '\n' ' ')" >> $GITHUB_ENV + + - name: "Set a docker image tag" + id: tag + shell: bash + run: echo "value=$([[ ! -z \"$DOCKER_BUILD_CONTEXT_FILES\" ]] && echo $(git rev-parse --short HEAD) || echo ${{ inputs.defaulttag }})" >> $GITHUB_OUTPUT + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..2c2f812a7ee --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,11 @@ +name: CI pipeline + +on: + pull_request: + branches: [ develop ] + +jobs: + pipeline: + if: github.server_url != 'https://github.com' + uses: ./.github/workflows/pipeline.yml + secrets: inherit diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 00000000000..4d807174d27 --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,101 @@ +name: CI Pipeline + +on: + workflow_call: + +jobs: + docker-tag: + name: Check if 'latest' tag could be used (no build docker images) + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - run: | + sudo apt update -qq + sudo apt install -y git curl jq sudo ca-certificates + sudo update-ca-certificates + - uses: actions/checkout@v4 + - uses: ./.github/actions/docker-tag + id: tag + with: + dockerfile: Jenkins/fast-release/Dockerfile.ci + + variants: + name: Define supported AIMET variants + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.final.outputs.value }} + steps: + - run: | + sudo apt update -qq + sudo apt install -y git curl jq sudo ca-certificates + sudo update-ca-certificates + + - name: Torch variants + run: | + set -x + VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [ + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"", "VER_TORCH":"2.1.2", "VER_ONNX":"", "VER_CUDA":"" }, + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"", "VER_TORCH":"2.1.2", "VER_ONNX":"", "VER_CUDA":"11.8.0" } + ]') + echo "VALUE=$VALUE" >> $GITHUB_ENV + + - name: Tensorflow variants + run: | + set -x + VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [ + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"2.10.1", "VER_TORCH":"", "VER_ONNX":"", "VER_CUDA":"" }, + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"2.10.1", "VER_TORCH":"", "VER_ONNX":"", "VER_CUDA":"11.8.0" } + ]') + echo "VALUE=$VALUE" >> $GITHUB_ENV + + - name: ONNX variants + run: | + set -x + VALUE=$(echo "${VALUE:-"{}"}" | jq -c '.include += [ + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"", "VER_TORCH":"1.13.1", "VER_ONNX":"1.14.1", "VER_CUDA":"" }, + { "VER_PYTHON":"3.10", "VER_TENSORFLOW":"", "VER_TORCH":"1.13.1", "VER_ONNX":"1.14.1", "VER_CUDA":"11.7.1" } + ]') + echo "VALUE=$VALUE" >> $GITHUB_ENV + + - name: (Last step) Generate few extra properties for each variant + id: final + run: | + set -x + VALUE=$(echo "$VALUE" | jq -c '.include[] |= . + { + "runs-on":(if .VER_CUDA != "" then "k8s-gpu" else "ubuntu-latest" end), + "id":("" + +(if .VER_TENSORFLOW != "" then "tf-" else "" end) + +(if .VER_ONNX != "" then "onnx-" elif .VER_TORCH != "" then "torch-" else "" end) + +(if .VER_CUDA != "" then "gpu" else "cpu" end) + ) + }') + echo "value=$VALUE" >> $GITHUB_OUTPUT + + docker-build-image: + name: Docker image ${{ matrix.id }} + runs-on: ubuntu-latest + needs: [docker-tag, variants] + strategy: + matrix: ${{ fromJSON(needs.variants.outputs.matrix) }} + steps: + - run: | + sudo apt update -qq + sudo apt install -y git curl jq sudo ca-certificates + sudo update-ca-certificates + - uses: actions/checkout@v4 + - uses: ./.github/actions/docker-build-image + with: + dockerfile: Jenkins/fast-release/Dockerfile.ci + docker-login: ${{ secrets.DOCKER_LOGIN }} + docker-password: ${{ secrets.DOCKER_CREDENTIALS }} + docker-registry: ${{ secrets.DOCKER_REGISTRY }} + image-name: "${{ secrets.DOCKER_IMAGE }}-${{ matrix.id }}" + image-tag: ${{ needs.docker-tag.outputs.tag }} + build-args: | + VER_PYTHON=${{ matrix.VER_PYTHON }} + VER_CUDA=${{ matrix.VER_CUDA }} + VER_TORCH=${{ matrix.VER_TORCH }} + VER_TENSORFLOW=${{ matrix.VER_TENSORFLOW }} + VER_ONNX=${{ matrix.VER_ONNX }} + diff --git a/Jenkins/fast-release/Dockerfile.ci b/Jenkins/fast-release/Dockerfile.ci new file mode 100644 index 00000000000..9db9fdeba49 --- /dev/null +++ b/Jenkins/fast-release/Dockerfile.ci @@ -0,0 +1,73 @@ +ARG BASE_IMAGE=ubuntu:22.04 +FROM $BASE_IMAGE AS build + +ENV CONDA_PREFIX=/opt/conda +ENV CONDA=${CONDA_PREFIX}/bin/conda +ENV CONDA_DEFAULT_ENV=dev + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install --yes --no-install-recommends \ + acl \ + ca-certificates \ + curl \ + # manylinux2014 requires gcc 10 and cuda doesn't support gcc>11 + g++-10 \ + git \ + jq \ + make \ + sudo \ + && rm -rf /var/lib/apt/lists/* \ + && echo '%users ALL = (ALL) NOPASSWD: ALL' > /etc/sudoers.d/passwordless \ + && curl -o /tmp/conda.sh -L 'https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh' \ + && mkdir -m 777 -p ${CONDA_PREFIX} \ + && setfacl -d -m o::rwx ${CONDA_PREFIX} \ + && bash /tmp/conda.sh -u -b -p ${CONDA_PREFIX} \ + && rm /tmp/conda.sh \ + && ${CONDA} config --set channel_priority strict \ + && ${CONDA} init --no-user --system --all \ + && ${CONDA} install -y conda-devenv \ + && ${CONDA} clean --yes --all --verbose \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ + && git config --system --add safe.directory '*' + +ARG VER_PYTHON=3.8 +ARG VER_CUDA=11.7.1 +ARG VER_TORCH="" +ARG VER_TENSORFLOW="" +ARG VER_ONNX="" + +COPY Jenkins/fast-release/environment.devenv.yml /tmp/ +RUN export PATH=$PATH:${CONDA_PREFIX}/bin PIP_NO_CACHE_DIR=1 \ + && ${CONDA} devenv \ + --env-var ENV_NAME="${CONDA_DEFAULT_ENV}" \ + --env-var VER_PYTHON="${VER_PYTHON}" \ + --env-var VER_CUDA="${VER_CUDA}" \ + --file /tmp/environment.devenv.yml \ + --output-file /tmp/environment.yml \ + && cat /tmp/environment.yml \ + && ${CONDA} clean --yes --all --verbose \ + && echo "conda activate ${CONDA_DEFAULT_ENV}" >> /etc/profile.d/conda.sh \ + && rm -rf ~/.conda* + +RUN --mount=type=bind,target=/workspace \ + echo "Install all required dependencies" \ + && export PIP_CACHE_DIR=/tmp/pip-cache BUILD_DIR=/tmp/build \ + && mkdir -p $BUILD_DIR \ + && export PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl" \ + && export CMAKE_ARGS="\ + -DENABLE_TENSORFLOW=$([ -z ${VER_TENSORFLOW} ]; echo $?) \ + -DENABLE_TORCH=$([ -z ${VER_TORCH} ]; echo $?) \ + -DENABLE_ONNX=$([ -z ${VER_ONNX} ]; echo $?) \ + -DENABLE_CUDA=$([ -z ${VER_CUDA} ]; echo $?) \ + " \ + && ${CONDA} run --name ${CONDA_DEFAULT_ENV} --live-stream \ + python3 -m pip install --dry-run --report $BUILD_DIR/pip-report.json -C build-dir="$BUILD_DIR/{wheel_tag}" /workspace \ + && ${CONDA} run --name ${CONDA_DEFAULT_ENV} --live-stream \ + python3 -m pip install -c /workspace/packaging/dependencies/constraints.txt $(jq -r '.install[0].metadata.requires_dist[] | split(";") | .[0]' $BUILD_DIR/pip-report.json) \ + && rm -rf $PIP_CACHE_DIR $BUILD_DIR + +ENV PYTHONPYCACHEPREFIX=/tmp + +ENTRYPOINT ["/bin/bash", "--login", "-c", "${0#--} \"$@\""] +CMD ["/bin/bash"] + diff --git a/Jenkins/fast-release/environment.devenv.yml b/Jenkins/fast-release/environment.devenv.yml new file mode 100644 index 00000000000..cae92095d17 --- /dev/null +++ b/Jenkins/fast-release/environment.devenv.yml @@ -0,0 +1,29 @@ +{% set ENV_NAME = os.environ.get('ENV_NAME', 'dev') %} + +{% set VER_PYTHON = os.environ.get('VER_PYTHON') %} +{% set VER_CUDA = os.environ.get('VER_CUDA') %} + +{% set CUDA_CHANNEL = 'nvidia/label/cuda-' + VER_CUDA %} +{% set CU = 'cu' + ''.join(VER_CUDA.split('.')[:-1]) if VER_CUDA != '' else 'cpu' %} + + +name: {{ ENV_NAME }} + +{% if CU != 'cpu' %} +channels: + - {{ CUDA_CHANNEL }} +{% endif %} + +dependencies: + - auditwheel + - patchelf + - python={{ VER_PYTHON }} + - python-build + - pip +{% if CU != 'cpu' %} + - cuda-gdb + - cuda-nvcc + - cuda-nvtx + - cuda-libraries-dev + - cudnn +{% endif %} diff --git a/packaging/dependencies/constraints.txt b/packaging/dependencies/constraints.txt new file mode 100644 index 00000000000..9433039b385 --- /dev/null +++ b/packaging/dependencies/constraints.txt @@ -0,0 +1,3 @@ +# pytorch uses setuptools to build C++ extensions +# [ImportError: cannot import name 'packaging' from 'pkg_resources'] +setuptools<70 diff --git a/packaging/plugins/local/aimet.py b/packaging/plugins/local/aimet.py new file mode 100644 index 00000000000..804209770e6 --- /dev/null +++ b/packaging/plugins/local/aimet.py @@ -0,0 +1,74 @@ +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +from __future__ import annotations + +import itertools +import os +import pathlib +import shlex + +__all__ = ["dynamic_metadata"] + + +def __dir__() -> list[str]: + return __all__ + + +def is_cmake_option_enabled(option_name: str) -> bool: + """Returns True if CMAKE_ARGS environment variable contains `-D{option_name}=ON/YES/1/TRUE` and False otherwise.""" + cmake_args = {k:v for k,v in (arg.split("=", 1) for arg in shlex.split(os.environ.get("CMAKE_ARGS", "")))} + return not cmake_args.get(f"-D{option_name}", "").upper() in {"OFF", "NO", "FALSE", "0", "N" } + + +def get_aimet_variant() -> str: + """Return a variant based on CMAKE_ARGS environment variable""" + enable_cuda = is_cmake_option_enabled("ENABLE_CUDA") + enable_torch = is_cmake_option_enabled("ENABLE_TORCH") + enable_tensorflow = is_cmake_option_enabled("ENABLE_TENSORFLOW") + enable_onnx = is_cmake_option_enabled("ENABLE_ONNX") + + variant = "" + if enable_tensorflow: + variant += "tf-" + if enable_torch: + variant += "torch-" + if enable_onnx: + variant = "onnx-" + variant += "gpu" if enable_cuda else "cpu" + return variant + + +def get_aimet_dependencies() -> list[str]: + """Read dependencies form the corresponded files and return them as a list (!) of strings""" + deps_path = pathlib.Path("packaging", "dependencies", get_aimet_variant()) + deps_files = [deps_path.parent / "reqs_pip_common.txt", *deps_path.glob("reqs_pip_*.txt")] + print(f"CMAKE_ARGS='{os.environ.get('CMAKE_ARGS', '')}'") + print(f"Read dependencies for variant '{get_aimet_variant()}' from the following files: {deps_files}") + deps = {d for d in itertools.chain.from_iterable(line.replace(" -f ", "\n-f ").split("\n") for f in deps_files for line in f.read_text(encoding="utf8").splitlines()) if not d.startswith(("#", "-f"))} + return list(deps) + + +def get_version() -> str: + return pathlib.Path("packaging", "version.txt").read_text(encoding="utf8") + + +def dynamic_metadata( + field: str, + settings: dict[str, object] | None = None, +) -> str: + if settings: + raise ValueError("No inline configuration is supported") + if field == "name": + return f"aimet-{get_aimet_variant()}" + if field == "dependencies": + return get_aimet_dependencies() + if field == "version": + return get_version() + raise ValueError(f"Unsupported field '{field}'") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..b71f55e42d4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = [ + "scikit-build-core[wheels]>=0.9", +] +build-backend = "scikit_build_core.build" + +[project] +#name = "aimet" +requires-python = ">=3.8" +dynamic = ["name", "dependencies", "version"] + +[project.optional-dependencies] +dev = [ + # duplicate build-system.requires for editable mode (non-isolated) + "scikit-build-core[wheels]>=0.9", + # and the rest +] +test = [ + "pytest", +] +docs = [ +] + +[tool.scikit-build] +experimental = true +metadata.name = { provider = "aimet", provider-path = "packaging/plugins/local" } +metadata.dependencies = { provider = "aimet", provider-path = "packaging/plugins/local" } +metadata.version = { provider="aimet", provider-path = "packaging/plugins/local" } +build-dir = "build" +sdist.cmake = false +logging.level = "DEBUG" +strict-config = false +wheel.license-files=[] +wheel.packages=["TrainingExtensions/common/src/python/aimet_common"] + +[tool.scikit-build.cmake.define] +CMAKE_BUILD_TYPE="RelWithDebInfo" +CMAKE_CUDA_ARCHITECTURES="70;75;80" +CMAKE_CUDA_FLAGS="--threads=8" +