diff --git a/Makefile b/Makefile index 77ad298..11df315 100644 --- a/Makefile +++ b/Makefile @@ -24,25 +24,27 @@ MAKEFLAGS += --no-print-directory MAKEFILE_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) # build architecture -DEFAULT_ARCH := $(shell dpkg --print-architecture) +DEFAULT_ARCH := $(shell dpkg --print-architecture 2> /dev/null || uname -m) ARCH := $(if $(ARCH),$(ARCH),$(DEFAULT_ARCH)) # defaults -DEFAULT_TF_VERSION := 2.11.0 -TF_VERSIONS := 2.11.0 2.10.1 2.10.0 2.9.3 2.9.2 2.9.1 2.9.0 2.8.4 2.8.3 2.8.2 2.8.1 2.8.0 2.7.4 2.7.3 2.7.2 2.7.1 2.7.0 2.6.5 2.6.4 2.6.3 2.6.2 2.6.1 2.6.0 2.5.3 2.5.2 2.5.1 2.5.0 2.4.4 2.4.3 2.4.2 2.4.1 2.4.0 2.3.4 2.3.3 2.3.2 2.3.1 2.3.0 2.2.3 2.2.2 2.2.1 2.2.0 2.1.4 2.1.3 2.1.2 2.1.1 2.1.0 2.0.4 2.0.3 2.0.2 2.0.1 2.0.0 -DEFAULT_JOBS := $(shell nproc) +DEFAULT_TF_VERSION := 2.13.0 +TF_VERSIONS := 2.13.0 2.12.1 2.12.0 2.11.1 2.11.0 2.10.1 2.10.0 2.9.3 2.9.2 2.9.1 2.9.0 2.8.4 2.8.3 2.8.2 2.8.1 2.8.0 2.7.4 2.7.3 2.7.2 2.7.1 2.7.0 2.6.5 2.6.4 2.6.3 2.6.2 2.6.1 2.6.0 2.5.3 2.5.2 2.5.1 2.5.0 2.4.4 2.4.3 2.4.2 2.4.1 2.4.0 2.3.4 2.3.3 2.3.2 2.3.1 2.3.0 2.2.3 2.2.2 2.2.1 2.2.0 2.1.4 2.1.3 2.1.2 2.1.1 2.1.0 2.0.4 2.0.3 2.0.2 2.0.1 2.0.0 +DEFAULT_JOBS := $(shell nproc 2> /dev/null || sysctl -n hw.ncpu) DEFAULT_GPU := 1 ifeq ($(ARCH), arm64) - DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES := 5.3,6.0,6.1,7.0,7.2,7.5,8.0,8.6,8.7 + DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES := 5.3,6.2,7.2,8.7 else - DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES := 5.3,6.0,6.1,7.0,7.2,7.5,8.0,8.6 + DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES := 6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0 endif +DEFAULT_BUILD_PIP_PACKAGE := 0 # arguments TF_VERSION := $(if $(TF_VERSION),$(TF_VERSION),$(DEFAULT_TF_VERSION)) JOBS := $(if $(JOBS),$(JOBS),$(DEFAULT_JOBS)) GPU := $(if $(GPU),$(GPU),$(DEFAULT_GPU)) TF_CUDA_COMPUTE_CAPABILITIES := $(if $(TF_CUDA_COMPUTE_CAPABILITIES),$(TF_CUDA_COMPUTE_CAPABILITIES),$(DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES)) +BUILD_PIP_PACKAGE := $(if $(BUILD_PIP_PACKAGE),$(BUILD_PIP_PACKAGE),$(DEFAULT_BUILD_PIP_PACKAGE)) # variables ifeq ($(GPU), 1) @@ -118,15 +120,15 @@ clean-images: clean-official-devel-images clean-cpp-images clean-libtensorflow-c $(MAKEFILE_DIR)/scripts/$@.sh .PHONY: 1-build-official-devel-image -1-build-official-devel-image: 0-download-official-dockerfiles +1-build-official-devel-image: $(MAKEFILE_DIR)/scripts/$@.sh .PHONY: 2-build-cpp-image -2-build-cpp-image: 1-build-official-devel-image +2-build-cpp-image: $(MAKEFILE_DIR)/scripts/$@.sh .PHONY: 3-export-libtensorflow-cc -3-export-libtensorflow-cc: 2-build-cpp-image +3-export-libtensorflow-cc: $(MAKEFILE_DIR)/scripts/$@.sh .PHONY: 4-test-libtensorflow-cc diff --git a/README.md b/README.md index 5d362d7..f9fed37 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ docker run --rm \ rwthika/tensorflow-cc:latest \ ./build-and-run.sh -# Hello from TensorFlow C++ 2.11.0! +# Hello from TensorFlow C++ 2.13.0! # # A = # 1 2 @@ -65,16 +65,16 @@ The pre-built `libtensorflow_cc.so` library and accompanying headers are package #### GPU ```bash -wget https://github.com/ika-rwth-aachen/libtensorflow_cc/releases/download/v2.11.0/libtensorflow-cc_2.11.0-gpu_$(dpkg --print-architecture).deb -sudo dpkg -i libtensorflow-cc_2.11.0-gpu_$(dpkg --print-architecture).deb +wget https://github.com/ika-rwth-aachen/libtensorflow_cc/releases/download/v2.13.0/libtensorflow-cc_2.13.0-gpu_$(dpkg --print-architecture).deb +sudo dpkg -i libtensorflow-cc_2.13.0-gpu_$(dpkg --print-architecture).deb ldconfig ``` #### CPU ```bash -wget https://github.com/ika-rwth-aachen/libtensorflow_cc/releases/download/v2.11.0/libtensorflow-cc_2.11.0_$(dpkg --print-architecture).deb -sudo dpkg -i libtensorflow-cc_2.11.0_$(dpkg --print-architecture).deb +wget https://github.com/ika-rwth-aachen/libtensorflow_cc/releases/download/v2.13.0/libtensorflow-cc_2.13.0_$(dpkg --print-architecture).deb +sudo dpkg -i libtensorflow-cc_2.13.0_$(dpkg --print-architecture).deb ldconfig ``` @@ -101,15 +101,18 @@ The amd64 images are based on the [official TensorFlow Docker images](https://hu | TensorFlow Version | CPU/GPU | Architecture | Image:Tag | | :---: | :---: | :---: | --- | -| 2.11.0 | GPU | amd64, arm64 | `rwthika/tensorflow-cc:latest-gpu` | -| 2.11.0 | CPU | amd64, arm64 | `rwthika/tensorflow-cc:latest` | +| 2.13.0 | GPU | amd64 | `rwthika/tensorflow-cc:latest-gpu` | +| 2.13.0 | CPU | amd64, arm64 | `rwthika/tensorflow-cc:latest` |
All TensorFlow Versions (GPU) | TensorFlow Version | Architecture | Image:Tag | | :---: | :---: | --- | -| latest | amd64, arm64 | `rwthika/tensorflow-cc:latest-gpu` | +| latest | amd64 | `rwthika/tensorflow-cc:latest-gpu` | +| 2.13.0 | amd64 | `rwthika/tensorflow-cc:2.13.0-gpu` | +| 2.12.0 | amd64 | `rwthika/tensorflow-cc:2.12.0-gpu` | +| 2.11.1 | amd64 | `rwthika/tensorflow-cc:2.11.0-gpu` | | 2.11.0 | amd64, arm64 | `rwthika/tensorflow-cc:2.11.0-gpu` | | 2.10.1 | amd64, arm64 | `rwthika/tensorflow-cc:2.10.1-gpu` | | 2.10.0 | amd64, arm64 | `rwthika/tensorflow-cc:2.10.0-gpu` | @@ -136,6 +139,10 @@ The amd64 images are based on the [official TensorFlow Docker images](https://hu | TensorFlow Version | Architecture | Image:Tag | | :---: | :---: | --- | | latest | amd64, arm64 | `rwthika/tensorflow-cc:latest` | +| 2.13.0 | amd64, arm64 | `rwthika/tensorflow-cc:2.13.0` | +| 2.12.1 | arm64 | `rwthika/tensorflow-cc:2.12.1` | +| 2.12.0 | amd64, arm64 | `rwthika/tensorflow-cc:2.12.0` | +| 2.11.1 | amd64, arm64 | `rwthika/tensorflow-cc:2.11.1` | | 2.11.0 | amd64, arm64 | `rwthika/tensorflow-cc:2.11.0` | | 2.10.1 | amd64, arm64 | `rwthika/tensorflow-cc:2.10.1` | | 2.10.0 | amd64, arm64 | `rwthika/tensorflow-cc:2.10.0` | @@ -178,7 +185,7 @@ The amd64 images are based on the [official TensorFlow Docker images](https://hu If you would like to build the deb-package and Docker images yourself, use the [`Makefile`](Makefile) as instructed below. -All `make` targets support the flags `TF_VERSION` (defaults to `2.11.0`), `GPU` (defaults to `1`), and `ARCH` (defaults to host architecture) in order to build a specific TensorFlow version in CPU/GPU mode for a specific architecture. +All `make` targets support the flags `TF_VERSION` (defaults to `2.13.0`), `GPU` (defaults to `1`), and `ARCH` (defaults to host architecture) in order to build a specific TensorFlow version in CPU/GPU mode for a specific architecture. The flag `BUILD_PIP_PACKAGE` (defaults to `0`) enables you to also build the pip-package alongside. All `make` targets listed below also have a counterpart named `-all`, which can be used to build multiple TensorFlow versions one after the other using the `TF_VERSIONS` flag like so: @@ -242,6 +249,14 @@ make 5-print-versions | Version | Architecture | Step 1 (CPU) | Step 2 (CPU) | Step 4 (CPU) | Step 1 (GPU) | Step 2 (GPU) | Step 4 (GPU) | Notes | | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | +| 2.13.0 | amd64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | +| 2.13.0 | arm64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | - | linker error at the end of building `libtensorflow_cc.so`; CPU image without Python TensorFlow | +| 2.12.1 | amd64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | missing image `tensorflow/tensorflow:2.12.1` | +| 2.12.1 | arm64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | - | linker error at the end of building `libtensorflow_cc.so`; CPU image without Python TensorFlow | +| 2.12.0 | amd64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | +| 2.12.0 | arm64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | - | linker error at the end of building `libtensorflow_cc.so`; CPU image without Python TensorFlow | +| 2.11.1 | amd64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | +| 2.11.1 | arm64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | - | linker error at the end of building `libtensorflow_cc.so`; CPU image without Python TensorFlow | | 2.11.0 | amd64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | | 2.11.0 | arm64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | | 2.10.1 | amd64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | @@ -310,6 +325,14 @@ make 5-print-versions | TensorFlow | Architecture | Ubuntu | GCC | Bazel | Python | protobuf | CUDA | cuDNN | TensorRT | GPU Compute Capability | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 2.13.0 | amd64 | 20.04 | 9.4.0/11.4.0 | 6.3.2 | 3.8.10 | 3.21.9 | 11.8.89 | 8.6.0 | 8.5.3 | 9.0, 8.9, 8.6, 8.0, 7.5, 7.0, 6.1, 6.0 | +| 2.13.0 | arm64 | 20.04 | 9.4.0/11.4.0 | 6.3.2 | 3.8.10 | 3.21.9 | - | - | - | - | +| 2.12.1 | amd64 | 20.04 | 9.4.0/11.4.0 | 6.3.2 | 3.8.10 | 3.21.9 | 11.8.89 | 8.6.0 | 8.5.3 | 9.0, 8.9, 8.6, 8.0, 7.5, 7.0, 6.1, 6.0 | +| 2.12.1 | arm64 | 20.04 | 9.4.0/11.4.0 | 6.3.2 | 3.8.10 | 3.21.9 | - | - | - | - | +| 2.12.0 | amd64 | 20.04 | 9.4.0/11.4.0 | 6.3.2 | 3.8.10 | 3.21.9 | 11.8.89 | 8.6.0 | 8.5.3 | 9.0, 8.9, 8.6, 8.0, 7.5, 7.0, 6.1, 6.0 | +| 2.12.0 | arm64 | 20.04 | 9.4.0/11.4.0 | 6.3.2 | 3.8.10 | 3.21.9 | - | - | - | - | +| 2.11.1 | amd64 | 20.04 | 9.4.0 | 6.3.2 | 3.8.10 | 3.9.2 | 11.2.152 | 8.1.0 | 7.2.2 | 8.6, 8.0, 7.5, 7.0, 6.1, 6.0 | +| 2.11.1 | arm64 | 20.04 | 9.4.0 | 6.3.2 | 3.8.10 | 3.9.2 | - | - | - | - | | 2.11.0 | amd64 | 20.04 | 9.4.0 | 6.0.0 | 3.8.10 | 3.9.2 | 11.2.152 | 8.1.0 | 7.2.2 | 8.6, 8.0, 7.5, 7.2, 7.0, 6.1, 6.0, 5.3 | | 2.11.0 | arm64 | 20.04 | 9.4.0 | 6.0.0 | 3.8.10 | 3.9.2 | 11.4.239 | 8.4.1 | 8.4.1 | 8.7, 8.6, 8.0, 7.5, 7.2, 7.0, 6.1, 6.0, 5.3 | | 2.10.1 | amd64 | 20.04 | 9.4.0 | 6.0.0 | 3.8.10 | 3.9.2 | 11.2.152 | 8.1.0 | 7.2.2 | 8.6, 8.0, 7.5, 7.2, 7.0, 6.1, 6.0, 5.3 | diff --git a/docker/.Dockerfiles/.gitignore b/docker/.Dockerfiles/.gitignore new file mode 100644 index 0000000..f6e6930 --- /dev/null +++ b/docker/.Dockerfiles/.gitignore @@ -0,0 +1,7 @@ +/* +!.gitignore +!/2.11.0/ +!/2.11.1/ +!/2.12.0/ +!/2.12.1/ +!/2.13.0/ diff --git a/docker/.Dockerfiles/2.11.0/bashrc b/docker/.Dockerfiles/2.11.0/bashrc new file mode 100644 index 0000000..d9dd7a6 --- /dev/null +++ b/docker/.Dockerfiles/2.11.0/bashrc @@ -0,0 +1,53 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +# If not running interactively, don't do anything +[ -z "$PS1" ] && return + +export PS1="\[\e[31m\]tf-docker\[\e[m\] \[\e[33m\]\w\[\e[m\] > " +export TERM=xterm-256color +alias grep="grep --color=auto" +alias ls="ls --color=auto" + +echo -e "\e[1;31m" +cat< /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-arm64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.Dockerfiles/2.11.0/dockerfiles/devel-cpu-ubuntu20.Dockerfile b/docker/.Dockerfiles/2.11.0/dockerfiles/devel-cpu-ubuntu20.Dockerfile new file mode 100644 index 0000000..dfd1ac6 --- /dev/null +++ b/docker/.Dockerfiles/2.11.0/dockerfiles/devel-cpu-ubuntu20.Dockerfile @@ -0,0 +1,108 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# +# THIS IS A GENERATED DOCKERFILE. +# +# This file was assembled from multiple pieces, whose use is documented +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. + +ARG UBUNTU_VERSION=20.04 + +FROM ubuntu:${UBUNTU_VERSION} AS base + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + clang-format \ + curl \ + git \ + libcurl3-dev \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + rsync \ + software-properties-common \ + sudo \ + unzip \ + zip \ + zlib1g-dev \ + openjdk-8-jdk \ + openjdk-8-jre-headless \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV CI_BUILD_PYTHON python + +# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version +ARG CACHE_STOP=1 +# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1 +ARG CHECKOUT_TF_SRC=0 +# In case of Python 2.7+ we need to add passwd entries for user and group id +RUN chmod a+w /etc/passwd /etc/group +RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow_src || true + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip + +RUN python3 -m pip --no-cache-dir install --upgrade \ + "pip<20.3" \ + setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which python3) /usr/local/bin/python + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + wget \ + openjdk-8-jdk \ + python3-dev \ + virtualenv \ + swig + +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip --no-cache-dir install \ + Pillow \ + h5py \ + tb-nightly \ + matplotlib \ + mock \ + numpy \ + scipy \ + scikit-learn \ + pandas \ + future \ + portpicker \ + enum34 \ + 'protobuf < 4' + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.Dockerfiles/2.11.0/dockerfiles/devel-cpu-ubuntu22.Dockerfile b/docker/.Dockerfiles/2.11.0/dockerfiles/devel-cpu-ubuntu22.Dockerfile new file mode 100644 index 0000000..8a18292 --- /dev/null +++ b/docker/.Dockerfiles/2.11.0/dockerfiles/devel-cpu-ubuntu22.Dockerfile @@ -0,0 +1,108 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# +# THIS IS A GENERATED DOCKERFILE. +# +# This file was assembled from multiple pieces, whose use is documented +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. + +ARG UBUNTU_VERSION=22.04 + +FROM ubuntu:${UBUNTU_VERSION} AS base + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + clang-format \ + curl \ + git \ + libcurl3-dev \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + rsync \ + software-properties-common \ + sudo \ + unzip \ + zip \ + zlib1g-dev \ + openjdk-8-jdk \ + openjdk-8-jre-headless \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV CI_BUILD_PYTHON python + +# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version +ARG CACHE_STOP=1 +# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1 +ARG CHECKOUT_TF_SRC=0 +# In case of Python 2.7+ we need to add passwd entries for user and group id +RUN chmod a+w /etc/passwd /etc/group +RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow_src || true + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip + +RUN python3 -m pip --no-cache-dir install --upgrade \ + "pip<20.3" \ + setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which python3) /usr/local/bin/python + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + wget \ + openjdk-8-jdk \ + python3-dev \ + virtualenv \ + swig + +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip --no-cache-dir install \ + Pillow \ + h5py \ + tb-nightly \ + matplotlib \ + mock \ + numpy \ + scipy \ + scikit-learn \ + pandas \ + future \ + portpicker \ + enum34 \ + 'protobuf < 4' + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.Dockerfiles/2.11.0/dockerfiles/devel-gpu+cu118-ubuntu20.Dockerfile b/docker/.Dockerfiles/2.11.0/dockerfiles/devel-gpu+cu118-ubuntu20.Dockerfile new file mode 100644 index 0000000..1f19c79 --- /dev/null +++ b/docker/.Dockerfiles/2.11.0/dockerfiles/devel-gpu+cu118-ubuntu20.Dockerfile @@ -0,0 +1,152 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# +# THIS IS A GENERATED DOCKERFILE. +# +# This file was assembled from multiple pieces, whose use is documented +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. + +ARG UBUNTU_VERSION=20.04 + +ARG ARCH= +ARG CUDA=11.8 +FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}.0-devel-ubuntu${UBUNTU_VERSION} as base +# ARCH and CUDA are specified again because the FROM directive resets ARGs +# (but their default value is retained if set previously) +ARG ARCH +ARG CUDA +ARG CUDNN=8.6.0.163-1 +ARG CUDNN_MAJOR_VERSION=8 +ARG LIB_DIR_PREFIX=x86_64 +ARG LIBNVINFER=8.5.3-1 +ARG LIBNVINFER_MAJOR_VERSION=8 + +# Needed for string substitution +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub && \ + apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + clang-format \ + cuda-command-line-tools-${CUDA/./-} \ + libcublas-${CUDA/./-} \ + libcublas-dev-${CUDA/./-} \ + cuda-nvprune-${CUDA/./-} \ + cuda-nvrtc-${CUDA/./-} \ + cuda-nvrtc-dev-${CUDA/./-} \ + cuda-cudart-dev-${CUDA/./-} \ + libcufft-dev-${CUDA/./-} \ + libcurand-dev-${CUDA/./-} \ + libcusolver-dev-${CUDA/./-} \ + libcusparse-dev-${CUDA/./-} \ + libcudnn8=${CUDNN}+cuda${CUDA} \ + libcudnn8-dev=${CUDNN}+cuda${CUDA} \ + libcurl3-dev \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + rsync \ + software-properties-common \ + unzip \ + zip \ + zlib1g-dev \ + wget \ + git \ + && \ + find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete + +# Install TensorRT if not building for PowerPC +# NOTE: libnvinfer uses cuda11.1 versions +RUN apt-get update && \ + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub && \ + echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/tensorRT.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + libnvinfer-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Configure the build for our CUDA configuration. +ENV LD_LIBRARY_PATH /usr/local/cuda-11.8/targets/x86_64-linux/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/include/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs:/usr/local/cuda-11.8/lib64:/usr/local/cuda-11.8/lib64 +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_VERSION=${CUDA} +ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION} +# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version +ARG CACHE_STOP=1 +# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1 +ARG CHECKOUT_TF_SRC=0 +RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow_src || true + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip + +RUN python3 -m pip --no-cache-dir install --upgrade \ + "pip<20.3" \ + setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which python3) /usr/local/bin/python + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + wget \ + openjdk-8-jdk \ + python3-dev \ + virtualenv \ + swig + +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip --no-cache-dir install \ + Pillow \ + h5py \ + tb-nightly \ + matplotlib \ + mock \ + numpy \ + scipy \ + scikit-learn \ + pandas \ + future \ + portpicker \ + enum34 \ + 'protobuf < 4' + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.Dockerfiles/2.11.0/dockerfiles/devel-gpu+cu118-ubuntu22.Dockerfile b/docker/.Dockerfiles/2.11.0/dockerfiles/devel-gpu+cu118-ubuntu22.Dockerfile new file mode 100644 index 0000000..fa81f63 --- /dev/null +++ b/docker/.Dockerfiles/2.11.0/dockerfiles/devel-gpu+cu118-ubuntu22.Dockerfile @@ -0,0 +1,152 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# +# THIS IS A GENERATED DOCKERFILE. +# +# This file was assembled from multiple pieces, whose use is documented +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. + +ARG UBUNTU_VERSION=22.04 + +ARG ARCH= +ARG CUDA=11.8 +FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}.0-devel-ubuntu${UBUNTU_VERSION} as base +# ARCH and CUDA are specified again because the FROM directive resets ARGs +# (but their default value is retained if set previously) +ARG ARCH +ARG CUDA +ARG CUDNN=8.6.0.163-1 +ARG CUDNN_MAJOR_VERSION=8 +ARG LIB_DIR_PREFIX=x86_64 +ARG LIBNVINFER=8.5.3-1 +ARG LIBNVINFER_MAJOR_VERSION=8 + +# Needed for string substitution +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub && \ + apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + clang-format \ + cuda-command-line-tools-${CUDA/./-} \ + libcublas-${CUDA/./-} \ + libcublas-dev-${CUDA/./-} \ + cuda-nvprune-${CUDA/./-} \ + cuda-nvrtc-${CUDA/./-} \ + cuda-nvrtc-dev-${CUDA/./-} \ + cuda-cudart-dev-${CUDA/./-} \ + libcufft-dev-${CUDA/./-} \ + libcurand-dev-${CUDA/./-} \ + libcusolver-dev-${CUDA/./-} \ + libcusparse-dev-${CUDA/./-} \ + libcudnn8=${CUDNN}+cuda${CUDA} \ + libcudnn8-dev=${CUDNN}+cuda${CUDA} \ + libcurl3-dev \ + libfreetype6-dev \ + libhdf5-serial-dev \ + libzmq3-dev \ + pkg-config \ + rsync \ + software-properties-common \ + unzip \ + zip \ + zlib1g-dev \ + wget \ + git \ + && \ + find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete + +# Install TensorRT if not building for PowerPC +# NOTE: libnvinfer uses cuda11.1 versions +RUN apt-get update && \ + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub && \ + echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/tensorRT.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + libnvinfer-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Configure the build for our CUDA configuration. +ENV LD_LIBRARY_PATH /usr/local/cuda-11.8/targets/x86_64-linux/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/include/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs:/usr/local/cuda-11.8/lib64:/usr/local/cuda-11.8/lib64 +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_VERSION=${CUDA} +ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION} +# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version +ARG CACHE_STOP=1 +# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1 +ARG CHECKOUT_TF_SRC=0 +RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow_src || true + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip + +RUN python3 -m pip --no-cache-dir install --upgrade \ + "pip<20.3" \ + setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which python3) /usr/local/bin/python + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + wget \ + openjdk-8-jdk \ + python3-dev \ + virtualenv \ + swig + +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip --no-cache-dir install \ + Pillow \ + h5py \ + tb-nightly \ + matplotlib \ + mock \ + numpy \ + scipy \ + scikit-learn \ + pandas \ + future \ + portpicker \ + enum34 \ + 'protobuf < 4' + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.Dockerfiles/2.11.1/bashrc b/docker/.Dockerfiles/2.11.1/bashrc new file mode 100644 index 0000000..d9dd7a6 --- /dev/null +++ b/docker/.Dockerfiles/2.11.1/bashrc @@ -0,0 +1,53 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +# If not running interactively, don't do anything +[ -z "$PS1" ] && return + +export PS1="\[\e[31m\]tf-docker\[\e[m\] \[\e[33m\]\w\[\e[m\] > " +export TERM=xterm-256color +alias grep="grep --color=auto" +alias ls="ls --color=auto" + +echo -e "\e[1;31m" +cat< /etc/apt/sources.list.d/tensorRT.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.0 \ + libnvinfer-dev=${LIBNVINFER}+cuda11.0 \ + libnvinfer-plugin-dev=${LIBNVINFER}+cuda11.0 \ + libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.0 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*; } + +# Configure the build for our CUDA configuration. +ENV LD_LIBRARY_PATH /usr/local/cuda-11.0/targets/x86_64-linux/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/include/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs:/usr/local/cuda-11.0/lib64:/usr/local/cuda-11.2/lib64 +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_VERSION=${CUDA} +ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION} +# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version +ARG CACHE_STOP=1 +# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1 +ARG CHECKOUT_TF_SRC=0 +RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow_src || true + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip + +RUN python3 -m pip --no-cache-dir install --upgrade \ + "pip<20.3" \ + setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which python3) /usr/local/bin/python + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + wget \ + openjdk-8-jdk \ + python3-dev \ + virtualenv \ + swig + +RUN python3 -m pip --no-cache-dir install \ + Pillow \ + h5py \ + tb-nightly \ + matplotlib \ + mock \ + 'numpy<1.19.0' \ + scipy \ + scikit-learn \ + pandas \ + future \ + portpicker \ + enum34 \ + 'protobuf < 4' + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.Dockerfiles/2.12.0/bashrc b/docker/.Dockerfiles/2.12.0/bashrc new file mode 100644 index 0000000..d9dd7a6 --- /dev/null +++ b/docker/.Dockerfiles/2.12.0/bashrc @@ -0,0 +1,53 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +# If not running interactively, don't do anything +[ -z "$PS1" ] && return + +export PS1="\[\e[31m\]tf-docker\[\e[m\] \[\e[33m\]\w\[\e[m\] > " +export TERM=xterm-256color +alias grep="grep --color=auto" +alias ls="ls --color=auto" + +echo -e "\e[1;31m" +cat< /etc/apt/sources.list.d/tensorRT.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + libnvinfer-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*; } + +# Configure the build for our CUDA configuration. +ENV LD_LIBRARY_PATH /usr/local/cuda-11.8/targets/x86_64-linux/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/include/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs:/usr/local/cuda-11.8/lib64:/usr/local/cuda-11.2/lib64 +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_VERSION=${CUDA} +ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION} +# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version +ARG CACHE_STOP=1 +# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1 +ARG CHECKOUT_TF_SRC=0 +RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow_src || true + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip + +RUN python3 -m pip --no-cache-dir install --upgrade \ + "pip<20.3" \ + setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which python3) /usr/local/bin/python + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + wget \ + openjdk-8-jdk \ + python3-dev \ + virtualenv \ + swig + +RUN python3 -m pip --no-cache-dir install \ + Pillow \ + h5py \ + keras_preprocessing \ + tb-nightly \ + matplotlib \ + mock \ + 'numpy<1.19.0' \ + scipy \ + scikit-learn \ + pandas \ + future \ + portpicker \ + enum34 + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.Dockerfiles/2.12.1/bashrc b/docker/.Dockerfiles/2.12.1/bashrc new file mode 100644 index 0000000..d9dd7a6 --- /dev/null +++ b/docker/.Dockerfiles/2.12.1/bashrc @@ -0,0 +1,53 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +# If not running interactively, don't do anything +[ -z "$PS1" ] && return + +export PS1="\[\e[31m\]tf-docker\[\e[m\] \[\e[33m\]\w\[\e[m\] > " +export TERM=xterm-256color +alias grep="grep --color=auto" +alias ls="ls --color=auto" + +echo -e "\e[1;31m" +cat< /etc/apt/sources.list.d/tensorRT.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + libnvinfer-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*; } + +# Configure the build for our CUDA configuration. +ENV LD_LIBRARY_PATH /usr/local/cuda-11.8/targets/x86_64-linux/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/include/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs:/usr/local/cuda-11.8/lib64:/usr/local/cuda-11.2/lib64 +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_VERSION=${CUDA} +ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION} +# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version +ARG CACHE_STOP=1 +# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1 +ARG CHECKOUT_TF_SRC=0 +RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow_src || true + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip + +RUN python3 -m pip --no-cache-dir install --upgrade \ + "pip<20.3" \ + setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which python3) /usr/local/bin/python + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + wget \ + openjdk-8-jdk \ + python3-dev \ + virtualenv \ + swig + +RUN python3 -m pip --no-cache-dir install \ + Pillow \ + h5py \ + keras_preprocessing \ + tb-nightly \ + matplotlib \ + mock \ + 'numpy<1.19.0' \ + scipy \ + scikit-learn \ + pandas \ + future \ + portpicker \ + enum34 + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.Dockerfiles/2.13.0/bashrc b/docker/.Dockerfiles/2.13.0/bashrc new file mode 100644 index 0000000..d9dd7a6 --- /dev/null +++ b/docker/.Dockerfiles/2.13.0/bashrc @@ -0,0 +1,53 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +# If not running interactively, don't do anything +[ -z "$PS1" ] && return + +export PS1="\[\e[31m\]tf-docker\[\e[m\] \[\e[33m\]\w\[\e[m\] > " +export TERM=xterm-256color +alias grep="grep --color=auto" +alias ls="ls --color=auto" + +echo -e "\e[1;31m" +cat< /etc/apt/sources.list.d/tensorRT.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + libnvinfer-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin-dev=${LIBNVINFER}+cuda11.8 \ + libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.8 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*; } + +# Configure the build for our CUDA configuration. +ENV LD_LIBRARY_PATH /usr/local/cuda-11.8/targets/x86_64-linux/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/include/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH:/usr/local/cuda/lib64/stubs:/usr/local/cuda-11.8/lib64:/usr/local/cuda-11.2/lib64 +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_VERSION=${CUDA} +ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION} +# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version +ARG CACHE_STOP=1 +# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1 +ARG CHECKOUT_TF_SRC=0 +RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow_src || true + +# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure +# dynamic linker run-time bindings +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ + && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ + && ldconfig + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip + +RUN python3 -m pip --no-cache-dir install --upgrade \ + "pip<20.3" \ + setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which python3) /usr/local/bin/python + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + wget \ + openjdk-8-jdk \ + python3-dev \ + virtualenv \ + swig + +RUN python3 -m pip --no-cache-dir install \ + Pillow \ + h5py \ + keras_preprocessing \ + tb-nightly \ + matplotlib \ + mock \ + 'numpy<1.19.0' \ + scipy \ + scikit-learn \ + pandas \ + future \ + portpicker \ + enum34 + +# Installs bazelisk +RUN mkdir /bazel && \ + curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ + mkdir /bazelisk && \ + curl -fSsL -o /bazelisk/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazelisk/master/LICENSE" && \ + curl -fSsL -o /usr/bin/bazel "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" && \ + chmod +x /usr/bin/bazel + +COPY bashrc /etc/bash.bashrc +RUN chmod a+rwx /etc/bash.bashrc diff --git a/docker/.gitignore b/docker/.gitignore index f928c21..5a9df23 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,2 +1 @@ -.Dockerfiles/ .log/ \ No newline at end of file diff --git a/docker/DEBIAN/control b/docker/DEBIAN/control index 0b94a9e..dc8f226 100644 --- a/docker/DEBIAN/control +++ b/docker/DEBIAN/control @@ -2,4 +2,5 @@ Package: libtensorflow-cc Version: TF_VERSION Architecture: TARGETARCH Maintainer: Lennart Reiher +Uploaders: Lennart Reiher , Jean-Pierre Busch Description: TensorFlow C++ Library diff --git a/docker/Dockerfile b/docker/Dockerfile index af86fd3..e8837ac 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -39,11 +39,18 @@ RUN if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ]; then \ git checkout FETCH_HEAD -- tensorflow/BUILD; \ fi +# fix GPU-version build issue in 2.11.1, 2.12.0, 2.12.1, 2.13.0 +# https://github.com/tensorflow/tensorflow/issues/60398#issuecomment-1537040602 +COPY patch/ patch/ +RUN if [ "${TF_VERSION}" = "2.11.1" ] || [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then \ + git apply patch/${TF_VERSION}/workspace2.bzl.patch; \ + fi + # configure compilation -ARG TF_CUDA_COMPUTE_CAPABILITIES=5.3,6.0,6.1,7.0,7.2,7.5,8.0,8.6 ENV PYTHON_BIN_PATH=/usr/bin/python3 ENV PYTHON_LIB_PATH=/usr/lib/python3/dist-packages ENV TF_NEED_ROCM=0 +ARG TF_CUDA_COMPUTE_CAPABILITIES=6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0 ENV TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} ENV TF_CUDA_CLANG=0 ENV GCC_HOST_COMPILER_PATH=/usr/bin/gcc @@ -52,17 +59,17 @@ ENV TF_SET_ANDROID_WORKSPACE=0 RUN ./configure # build C++ library -# fix runtime issue in 2.10.0, 2.10.1, 2.11.0 +# fix runtime issue in 2.10.0, 2.10.1, 2.11.0, 2.11.1, 2.12.0, 2.12.1, 2.13.0 by building non-monolithic # https://github.com/tensorflow/tensorflow/issues/59081 ARG JOBS="auto" RUN if [ "${GPU_POSTFIX}" = "-gpu" ]; then \ - if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ]; then \ + if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ] || [ "${TF_VERSION}" = "2.11.1" ] || [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then \ bazel build --jobs ${JOBS} --config=cuda --config=opt --verbose_failures tensorflow:libtensorflow_cc.so tensorflow:install_headers; \ else \ bazel build --jobs ${JOBS} --config=cuda --config=opt --config=monolithic --verbose_failures tensorflow:libtensorflow_cc.so tensorflow:install_headers; \ fi; \ else \ - if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ]; then \ + if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ] || [ "${TF_VERSION}" = "2.11.1" ] || [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then \ bazel build --jobs ${JOBS} --config=opt --verbose_failures tensorflow:libtensorflow_cc.so tensorflow:install_headers; \ else \ bazel build --jobs ${JOBS} --config=opt --config=monolithic --verbose_failures tensorflow:libtensorflow_cc.so tensorflow:install_headers; \ @@ -70,17 +77,42 @@ RUN if [ "${GPU_POSTFIX}" = "-gpu" ]; then \ fi # move libtensorflow_cc to separate folder for easier Dockerfile COPY -RUN mkdir bazel-bin/tensorflow/lib && \ +RUN mkdir -p bazel-bin/tensorflow/lib && \ mv bazel-bin/tensorflow/libtensorflow_cc.so* bazel-bin/tensorflow/lib/ && \ mv bazel-bin/tensorflow/libtensorflow_framework.so* bazel-bin/tensorflow/lib/ && \ - ln -s libtensorflow_framework.so.2 bazel-bin/tensorflow/lib/libtensorflow_framework.so && \ + ln -sf libtensorflow_framework.so.2 bazel-bin/tensorflow/lib/libtensorflow_framework.so && \ rm bazel-bin/tensorflow/lib/*params +# build pip package +# fix runtime issue in 2.10.0, 2.10.1, 2.11.0, 2.11.1, 2.12.0, 2.12.1, 2.13.0 by building non-monolithic +# https://github.com/tensorflow/tensorflow/issues/59081 +ARG BUILD_PIP_PACKAGE=0 +RUN if [ "${BUILD_PIP_PACKAGE}" = "1" ]; then \ + if [ "${GPU_POSTFIX}" = "-gpu" ]; then \ + if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ] || [ "${TF_VERSION}" = "2.11.1" ] || [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then \ + bazel build --jobs ${JOBS} --config=cuda --config=opt --verbose_failures tensorflow/tools/pip_package:build_pip_package; \ + else \ + bazel build --jobs ${JOBS} --config=cuda --config=opt --config=monolithic --verbose_failures tensorflow/tools/pip_package:build_pip_package; \ + fi; \ + else \ + if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ] || [ "${TF_VERSION}" = "2.11.1" ] || [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then \ + bazel build --jobs ${JOBS} --config=opt --verbose_failures tensorflow/tools/pip_package:build_pip_package; \ + else \ + bazel build --jobs ${JOBS} --config=opt --config=monolithic --verbose_failures tensorflow/tools/pip_package:build_pip_package; \ + fi; \ + fi; \ + pip install patchelf && \ + ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow/pip ; \ + else \ + mkdir -p /tmp/tensorflow/pip && \ + touch /tmp/tensorflow/pip/tensorflow-dummy.whl ; \ + fi + # build protobuf from source, same version as TensorFlow is using WORKDIR / -RUN apt-get install -y autoconf automake libtool curl make g++ unzip && \ +RUN apt-get install -y autoconf automake libtool curl make g++ unzip wget && \ PROTOBUF_URL=$(grep -Eho "https://github.com/protocolbuffers/protobuf/archive/.*\.(zip|tar.gz)" tensorflow/tensorflow/workspace*.bzl) && \ - PROTOBUF_VERSION=$(echo $PROTOBUF_URL | grep -Eo "(.?\..?\..?|[0-9a-f]{40})") && \ + PROTOBUF_VERSION=$(echo $PROTOBUF_URL | grep -Eo "([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|[0-9a-f]{40})") && \ wget -q $PROTOBUF_URL -O protobuf.archive && \ (unzip -q protobuf.archive || tar -xzf protobuf.archive) && \ rm protobuf.archive && \ @@ -119,18 +151,45 @@ WORKDIR / RUN dpkg-deb --build --root-owner-group libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX} && \ rm -rf libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX}/ +# copy pip package, if built +ARG BUILD_PIP_PACKAGE=0 +COPY --from=build /tmp/tensorflow/pip/tensorflow*.whl / + # --- final stage with TensorFlow Python and C++ Library ----------------------- FROM --platform=amd64 tensorflow/tensorflow:${TF_VERSION}${GPU_POSTFIX} as final-amd64 ARG TARGETARCH -FROM --platform=arm64 nvcr.io/nvidia/l4t-tensorflow:r35.1.0-tf2.9-py3 as final-arm64 +# FROM ubuntu +FROM --platform=arm64 ubuntu:focal as final-arm64 +ARG TARGETARCH + +FROM --platform=arm64 nvcr.io/nvidia/l4t-tensorflow:r35.1.0-tf2.9-py3 as final-arm64-gpu ARG TARGETARCH -FROM "final-${TARGETARCH}" as final +FROM "final-${TARGETARCH}${GPU_POSTFIX}" as final +ARG DEBIAN_FRONTEND=noninteractive ARG TF_VERSION ARG GPU_POSTFIX +# make sure gcc is installed +RUN apt-get update && \ + apt-get install -y build-essential && \ + rm -rf /var/lib/apt/lists/* + +# install gcc 11 for 2.12.0, 2.12.1, 2.13.0 +# https://stackoverflow.com/a/67453352/7264974 +RUN if [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then \ + apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:ubuntu-toolchain-r/test && \ + apt-get install -y gcc-11 g++-11 && \ + rm -rf /var/lib/apt/lists/* && \ + update-alternatives --remove-all cpp && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 --slave /usr/bin/g++ g++ /usr/bin/g++-9 --slave /usr/bin/gcov gcov /usr/bin/gcov-9 --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-9 --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-9 --slave /usr/bin/cpp cpp /usr/bin/cpp-9 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11 --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 --slave /usr/bin/cpp cpp /usr/bin/cpp-11 ; \ + fi + # install TensorFlow C++ API incl. protobuf COPY --from=deb-package /libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX}.deb /tmp RUN dpkg -i /tmp/libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX}.deb && \ diff --git a/docker/patch/2.11.1/workspace2.bzl.patch b/docker/patch/2.11.1/workspace2.bzl.patch new file mode 100644 index 0000000..956eb74 --- /dev/null +++ b/docker/patch/2.11.1/workspace2.bzl.patch @@ -0,0 +1,17 @@ +diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl +index 18274158..306f2431 100644 +--- a/tensorflow/workspace2.bzl ++++ b/tensorflow/workspace2.bzl +@@ -171,9 +171,9 @@ def _tf_repositories(): + name = "cudnn_frontend_archive", + build_file = "//third_party:cudnn_frontend.BUILD", + patch_file = ["//third_party:cudnn_frontend_header_fix.patch"], +- sha256 = "6ca6e7d4affdff59c749865d6d0428c849968b0873a1d1b849f56d7be624f27b", +- strip_prefix = "cudnn-frontend-0.7.1", +- urls = tf_mirror_urls("https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v0.7.1.zip"), ++ sha256 = "d8dba9e2607a0c256aa8eacb45b39986ab6f3f24a4d431d4397047a3cb0cd4fb", ++ strip_prefix = "cudnn-frontend-0.9", ++ urls = tf_mirror_urls("https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v0.9.zip"), + ) + + tf_http_archive( diff --git a/docker/patch/2.12.0/workspace2.bzl.patch b/docker/patch/2.12.0/workspace2.bzl.patch new file mode 100644 index 0000000..64aa087 --- /dev/null +++ b/docker/patch/2.12.0/workspace2.bzl.patch @@ -0,0 +1,17 @@ +diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl +index 1261273b..0a9d755a 100644 +--- a/tensorflow/workspace2.bzl ++++ b/tensorflow/workspace2.bzl +@@ -172,9 +172,9 @@ def _tf_repositories(): + name = "cudnn_frontend_archive", + build_file = "//third_party:cudnn_frontend.BUILD", + patch_file = ["//third_party:cudnn_frontend_header_fix.patch"], +- sha256 = "3c7b842cd67989810955b220fa1116e7e2ed10660a8cfb632118146a64992c30", +- strip_prefix = "cudnn-frontend-0.7.3", +- urls = tf_mirror_urls("https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v0.7.3.zip"), ++ sha256 = "d8dba9e2607a0c256aa8eacb45b39986ab6f3f24a4d431d4397047a3cb0cd4fb", ++ strip_prefix = "cudnn-frontend-0.9", ++ urls = tf_mirror_urls("https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v0.9.zip"), + ) + + tf_http_archive( diff --git a/docker/patch/2.12.1/workspace2.bzl.patch b/docker/patch/2.12.1/workspace2.bzl.patch new file mode 100644 index 0000000..64aa087 --- /dev/null +++ b/docker/patch/2.12.1/workspace2.bzl.patch @@ -0,0 +1,17 @@ +diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl +index 1261273b..0a9d755a 100644 +--- a/tensorflow/workspace2.bzl ++++ b/tensorflow/workspace2.bzl +@@ -172,9 +172,9 @@ def _tf_repositories(): + name = "cudnn_frontend_archive", + build_file = "//third_party:cudnn_frontend.BUILD", + patch_file = ["//third_party:cudnn_frontend_header_fix.patch"], +- sha256 = "3c7b842cd67989810955b220fa1116e7e2ed10660a8cfb632118146a64992c30", +- strip_prefix = "cudnn-frontend-0.7.3", +- urls = tf_mirror_urls("https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v0.7.3.zip"), ++ sha256 = "d8dba9e2607a0c256aa8eacb45b39986ab6f3f24a4d431d4397047a3cb0cd4fb", ++ strip_prefix = "cudnn-frontend-0.9", ++ urls = tf_mirror_urls("https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v0.9.zip"), + ) + + tf_http_archive( diff --git a/docker/patch/2.13.0/workspace2.bzl.patch b/docker/patch/2.13.0/workspace2.bzl.patch new file mode 100644 index 0000000..ad3c568 --- /dev/null +++ b/docker/patch/2.13.0/workspace2.bzl.patch @@ -0,0 +1,17 @@ +diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl +index da9295ad..86a3df2f 100644 +--- a/tensorflow/workspace2.bzl ++++ b/tensorflow/workspace2.bzl +@@ -174,9 +174,9 @@ def _tf_repositories(): + name = "cudnn_frontend_archive", + build_file = "//third_party:cudnn_frontend.BUILD", + patch_file = ["//third_party:cudnn_frontend_header_fix.patch"], +- sha256 = "bfcf778030831f325cfc13ae5995388cc834fbff2995a297ba580d9ec65ca3b6", +- strip_prefix = "cudnn-frontend-0.8", +- urls = tf_mirror_urls("https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v0.8.zip"), ++ sha256 = "d8dba9e2607a0c256aa8eacb45b39986ab6f3f24a4d431d4397047a3cb0cd4fb", ++ strip_prefix = "cudnn-frontend-0.9", ++ urls = tf_mirror_urls("https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v0.9.zip"), + ) + + tf_http_archive( diff --git a/scripts/.common.sh b/scripts/.common.sh index a91ede8..15478e5 100644 --- a/scripts/.common.sh +++ b/scripts/.common.sh @@ -21,10 +21,10 @@ set -e set -o pipefail -DEFAULT_TF_VERSION="2.11.0" -DEFAULT_JOBS=$(nproc) +DEFAULT_TF_VERSION="2.13.0" +DEFAULT_JOBS=$(nproc 2> /dev/null || sysctl -n hw.ncpu) DEFAULT_GPU=1 -DEFAULT_ARCH=$(dpkg --print-architecture) +DEFAULT_ARCH=$(dpkg --print-architecture 2> /dev/null || uname -m) TF_VERSION=${TF_VERSION:-${DEFAULT_TF_VERSION}} JOBS=${JOBS:-${DEFAULT_JOBS}} @@ -33,9 +33,9 @@ GPU=${GPU:-${DEFAULT_GPU}} ARCH=${ARCH:-${DEFAULT_ARCH}} if [ "$ARCH" = "arm64" ]; then - DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES=5.3,6.0,6.1,7.0,7.2,7.5,8.0,8.6,8.7 + DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES=5.3,6.2,7.2,8.7 else - DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES=5.3,6.0,6.1,7.0,7.2,7.5,8.0,8.6 + DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES=6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0 fi TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES:-${DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES}} diff --git a/scripts/1-build-official-devel-image.sh b/scripts/1-build-official-devel-image.sh index 3df21eb..6fdab50 100755 --- a/scripts/1-build-official-devel-image.sh +++ b/scripts/1-build-official-devel-image.sh @@ -23,21 +23,30 @@ source $(dirname "$0")/.common.sh CPU_GPU_POSTFIX=${GPU_POSTFIX:--cpu} BUILD_DIR=${DOWNLOAD_DOCKERFILE_DIR} +BSD_SED_ARG="" +if [ "$(uname -s)" = "Darwin" ]; then + BSD_SED_ARG=".bak" +fi +TF_VERSION_MAJOR=$(echo "$TF_VERSION" | cut -d. -f1) +TF_VERSION_MINOR=$(echo "$TF_VERSION" | cut -d. -f2) +TF_VERSION_PATCH=$(echo "$TF_VERSION" | cut -d. -f3) if [ "$ARCH" = "amd64" ]; then DOCKERFILE=${DOWNLOAD_DOCKERFILE_DIR}/dockerfiles/devel${CPU_GPU_POSTFIX}.Dockerfile elif [ "$ARCH" = "arm64" ]; then - DOCKERFILE=${DOWNLOAD_DOCKERFILE_DIR}/dockerfiles/arm64v8/devel-cpu-arm64v8.Dockerfile - if [ "$TF_VERSION" = "2.8.4" ]; then - sed -i "s/ubuntu:\${UBUNTU_VERSION}/nvcr.io\/nvidia\/l4t-tensorflow:r34.1.1-tf2.8-py3/" $DOCKERFILE + if [[ "$TF_VERSION_MAJOR" -lt "2" || ( "$TF_VERSION_MAJOR" -eq "2" && "$TF_VERSION_MINOR" -le "10" ) ]]; then + DOCKERFILE=${DOWNLOAD_DOCKERFILE_DIR}/dockerfiles/arm64v8/devel-cpu-arm64v8.Dockerfile + if [ "$TF_VERSION" = "2.8.4" ]; then + sed -i $BSD_SED_ARG "s/ubuntu:\${UBUNTU_VERSION}/nvcr.io\/nvidia\/l4t-tensorflow:r34.1.1-tf2.8-py3/" $DOCKERFILE + else + sed -i $BSD_SED_ARG "s/ubuntu:\${UBUNTU_VERSION}/nvcr.io\/nvidia\/l4t-tensorflow:r35.1.0-tf2.9-py3/" $DOCKERFILE + fi + # replace sklearn (deprecated) with scikit-learn https://pypi.org/project/sklearn/ + sed -i $BSD_SED_ARG "s/sklearn/scikit-learn/" $DOCKERFILE else - sed -i "s/ubuntu:\${UBUNTU_VERSION}/nvcr.io\/nvidia\/l4t-tensorflow:r35.1.0-tf2.9-py3/" $DOCKERFILE + DOCKERFILE=${DOWNLOAD_DOCKERFILE_DIR}/dockerfiles/arm64v8/devel${CPU_GPU_POSTFIX}-arm64v8.Dockerfile fi fi -# replace sklearn (deprecated) with scikit-learn -# https://pypi.org/project/sklearn/ -sed -i "s/sklearn/scikit-learn/" $DOCKERFILE - echo "Building ${IMAGE_DEVEL_ARCH} ... " docker build -t ${IMAGE_DEVEL_ARCH} -f ${DOCKERFILE} ${BUILD_DIR} | tee ${LOG_FILE} diff --git a/scripts/2-build-cpp-image.sh b/scripts/2-build-cpp-image.sh index b7e4517..98977de 100755 --- a/scripts/2-build-cpp-image.sh +++ b/scripts/2-build-cpp-image.sh @@ -22,5 +22,5 @@ source $(dirname "$0")/.common.sh echo "Building ${IMAGE_CPP} ... " -docker build --build-arg TARGETARCH=$ARCH --build-arg TF_VERSION=${TF_VERSION} --build-arg JOBS=${JOBS} --build-arg GPU_POSTFIX=${GPU_POSTFIX} --build-arg TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} -t ${IMAGE_CPP_ARCH} ${DOCKER_DIR} | tee ${LOG_FILE} +docker build --build-arg TARGETARCH=$ARCH --build-arg TF_VERSION=${TF_VERSION} --build-arg JOBS=${JOBS} --build-arg GPU_POSTFIX=${GPU_POSTFIX} --build-arg TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} --build-arg BUILD_PIP_PACKAGE=${BUILD_PIP_PACKAGE} -t ${IMAGE_CPP_ARCH} ${DOCKER_DIR} | tee ${LOG_FILE} docker tag ${IMAGE_CPP_ARCH} ${IMAGE_CPP} \ No newline at end of file diff --git a/scripts/3-export-libtensorflow-cc.sh b/scripts/3-export-libtensorflow-cc.sh index bcdeadb..df152d5 100755 --- a/scripts/3-export-libtensorflow-cc.sh +++ b/scripts/3-export-libtensorflow-cc.sh @@ -24,12 +24,20 @@ source $(dirname "$0")/.common.sh EXPORT_DIR=${REPOSITORY_DIR}/libtensorflow-cc CONTAINER_FILE="libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX}.deb" EXPORT_FILE="libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX}_${ARCH}.deb" + +EXPORT_DIR_PIP=${REPOSITORY_DIR}/tensorflow-wheel +CONTAINER_FILE_PIP="tensorflow-${TF_VERSION}*.whl" + STAGE="deb-package" echo "Building ${IMAGE_LIBTENSORFLOW_CC_ARCH} ... " -docker build --build-arg TARGETARCH=$ARCH --build-arg TF_VERSION=${TF_VERSION} --build-arg JOBS=${JOBS} --build-arg GPU_POSTFIX=${GPU_POSTFIX} --build-arg TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} --target ${STAGE} -t ${IMAGE_LIBTENSORFLOW_CC_ARCH} ${DOCKER_DIR} | tee ${LOG_FILE} +docker build --build-arg TARGETARCH=$ARCH --build-arg TF_VERSION=${TF_VERSION} --build-arg JOBS=${JOBS} --build-arg GPU_POSTFIX=${GPU_POSTFIX} --build-arg TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} --build-arg BUILD_PIP_PACKAGE=${BUILD_PIP_PACKAGE} --target ${STAGE} -t ${IMAGE_LIBTENSORFLOW_CC_ARCH} ${DOCKER_DIR} | tee ${LOG_FILE} echo "Exporting to $(realpath ${EXPORT_DIR})/${EXPORT_FILE} ... " -TMP_CONTAINER=$(docker create ${IMAGE_LIBTENSORFLOW_CC_ARCH}) +TMP_CONTAINER=$(docker run -d --rm ${IMAGE_LIBTENSORFLOW_CC_ARCH} sleep infinity) docker cp ${TMP_CONTAINER}:/${CONTAINER_FILE} ${EXPORT_DIR}/${EXPORT_FILE} -docker rm -v ${TMP_CONTAINER} +if [ "$BUILD_PIP_PACKAGE" = "1" ]; then + echo "Exporting to $(realpath ${EXPORT_DIR_PIP}) ... " + docker exec ${TMP_CONTAINER} bash -c "ls /${CONTAINER_FILE_PIP}" | while read f; do docker cp ${TMP_CONTAINER}:/$f ${EXPORT_DIR_PIP}; done +fi +docker kill ${TMP_CONTAINER} diff --git a/scripts/4-test-libtensorflow-cc.sh b/scripts/4-test-libtensorflow-cc.sh index 61fc703..3d71d4a 100755 --- a/scripts/4-test-libtensorflow-cc.sh +++ b/scripts/4-test-libtensorflow-cc.sh @@ -23,7 +23,7 @@ source $(dirname "$0")/.common.sh EXAMPLE_DIR=${REPOSITORY_DIR}/example EXAMPLE_MOUNT="/example" -if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ]; then +if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ] || [ "${TF_VERSION}" = "2.11.1" ] || [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then CMD="./build-and-run.sh" else CMD="./build-and-run-monolithic.sh" diff --git a/tensorflow-wheel/.gitignore b/tensorflow-wheel/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/tensorflow-wheel/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file