add option to build and export pip package

Makefile

@@ -37,12 +37,14 @@ ifeq ($(ARCH), arm64)
 else
 	DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES := 6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0
 endif
+DEFAULT_BUILD_PIP_PACKAGE := 0
 
 # arguments
 TF_VERSION := $(if $(TF_VERSION),$(TF_VERSION),$(DEFAULT_TF_VERSION))
 JOBS := $(if $(JOBS),$(JOBS),$(DEFAULT_JOBS))
 GPU := $(if $(GPU),$(GPU),$(DEFAULT_GPU))
 TF_CUDA_COMPUTE_CAPABILITIES := $(if $(TF_CUDA_COMPUTE_CAPABILITIES),$(TF_CUDA_COMPUTE_CAPABILITIES),$(DEFAULT_TF_CUDA_COMPUTE_CAPABILITIES))
+BUILD_PIP_PACKAGE := $(if $(BUILD_PIP_PACKAGE),$(BUILD_PIP_PACKAGE),$(DEFAULT_BUILD_PIP_PACKAGE))
 
 # variables
 ifeq ($(GPU), 1)

README.md

@@ -185,7 +185,7 @@ The amd64 images are based on the [official TensorFlow Docker images](https://hu
 
 If you would like to build the deb-package and Docker images yourself, use the [`Makefile`](Makefile) as instructed below.
 
-All `make` targets support the flags `TF_VERSION` (defaults to `2.13.0`), `GPU` (defaults to `1`), and `ARCH` (defaults to host architecture) in order to build a specific TensorFlow version in CPU/GPU mode for a specific architecture.
+All `make` targets support the flags `TF_VERSION` (defaults to `2.13.0`), `GPU` (defaults to `1`), and `ARCH` (defaults to host architecture) in order to build a specific TensorFlow version in CPU/GPU mode for a specific architecture. The flag `BUILD_PIP_PACKAGE` (defaults to `0`) enables you to also build the pip-package alongside.
 
 All `make` targets listed below also have a counterpart named `<target>-all`, which can be used to build multiple TensorFlow versions one after the other using the `TF_VERSIONS` flag like so:
 

docker/Dockerfile

@@ -77,12 +77,35 @@ RUN if [ "${GPU_POSTFIX}" = "-gpu" ]; then \
     fi
 
 # move libtensorflow_cc to separate folder for easier Dockerfile COPY
-RUN mkdir bazel-bin/tensorflow/lib && \
+RUN mkdir -p bazel-bin/tensorflow/lib && \
+    mkdir -p bazel-bin/tensorflow/pip && \
     mv bazel-bin/tensorflow/libtensorflow_cc.so* bazel-bin/tensorflow/lib/ && \
     mv bazel-bin/tensorflow/libtensorflow_framework.so* bazel-bin/tensorflow/lib/ && \
-    ln -s libtensorflow_framework.so.2 bazel-bin/tensorflow/lib/libtensorflow_framework.so && \
+    ln -sf libtensorflow_framework.so.2 bazel-bin/tensorflow/lib/libtensorflow_framework.so && \
     rm bazel-bin/tensorflow/lib/*params
 
+# build pip package
+# fix runtime issue in 2.10.0, 2.10.1, 2.11.0, 2.11.1, 2.12.0, 2.12.1, 2.13.0 by building non-monolithic
+# https://github.com/tensorflow/tensorflow/issues/59081
+ARG BUILD_PIP_PACKAGE=0
+RUN if [ "${BUILD_PIP_PACKAGE}" = "1" ]; then \
+        if [ "${GPU_POSTFIX}" = "-gpu" ]; then \
+            if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ] || [ "${TF_VERSION}" = "2.11.1" ] || [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then \
+                bazel build --jobs ${JOBS} --config=cuda --config=opt --verbose_failures tensorflow/tools/pip_package:build_pip_package; \
+            else \
+                bazel build --jobs ${JOBS} --config=cuda --config=opt --config=monolithic --verbose_failures tensorflow/tools/pip_package:build_pip_package; \
+            fi; \
+        else \
+            if [ "${TF_VERSION}" = "2.10.0" ] || [ "${TF_VERSION}" = "2.10.1" ] || [ "${TF_VERSION}" = "2.11.0" ] || [ "${TF_VERSION}" = "2.11.1" ] || [ "${TF_VERSION}" = "2.12.0" ] || [ "${TF_VERSION}" = "2.12.1" ] || [ "${TF_VERSION}" = "2.13.0" ]; then \
+                bazel build --jobs ${JOBS} --config=opt --verbose_failures tensorflow/tools/pip_package:build_pip_package; \
+            else \
+                bazel build --jobs ${JOBS} --config=opt --config=monolithic --verbose_failures tensorflow/tools/pip_package:build_pip_package; \
+            fi; \
+        fi; \
+        pip install patchelf && \
+        ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow/pip ; \
+    fi
+
 # build protobuf from source, same version as TensorFlow is using
 WORKDIR /
 RUN apt-get install -y autoconf automake libtool curl make g++ unzip && \
@@ -126,6 +149,10 @@ WORKDIR /
 RUN dpkg-deb --build --root-owner-group libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX} && \
     rm -rf libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX}/
 
+# copy pip package, if built
+ARG BUILD_PIP_PACKAGE=0
+COPY --from=build /tmp/tensorflow/pip/tensorflow*.whl /
+
 # --- final stage with TensorFlow Python and C++ Library -----------------------
 FROM --platform=amd64 tensorflow/tensorflow:${TF_VERSION}${GPU_POSTFIX} as final-amd64
 ARG TARGETARCH

scripts/2-build-cpp-image.sh

@@ -22,5 +22,5 @@
 source  $(dirname "$0")/.common.sh
 
 echo "Building ${IMAGE_CPP} ... "
-docker build --build-arg TARGETARCH=$ARCH --build-arg TF_VERSION=${TF_VERSION} --build-arg JOBS=${JOBS} --build-arg GPU_POSTFIX=${GPU_POSTFIX} --build-arg TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} -t ${IMAGE_CPP_ARCH} ${DOCKER_DIR} | tee ${LOG_FILE}
+docker build --build-arg TARGETARCH=$ARCH --build-arg TF_VERSION=${TF_VERSION} --build-arg JOBS=${JOBS} --build-arg GPU_POSTFIX=${GPU_POSTFIX} --build-arg TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} --build-arg BUILD_PIP_PACKAGE=${BUILD_PIP_PACKAGE} -t ${IMAGE_CPP_ARCH} ${DOCKER_DIR} | tee ${LOG_FILE}
 docker tag ${IMAGE_CPP_ARCH} ${IMAGE_CPP}

scripts/3-export-libtensorflow-cc.sh

@@ -24,12 +24,20 @@ source  $(dirname "$0")/.common.sh
 EXPORT_DIR=${REPOSITORY_DIR}/libtensorflow-cc
 CONTAINER_FILE="libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX}.deb"
 EXPORT_FILE="libtensorflow-cc_${TF_VERSION}${GPU_POSTFIX}_${ARCH}.deb"
+
+EXPORT_DIR_PIP=${REPOSITORY_DIR}/tensorflow-wheel
+CONTAINER_FILE_PIP="tensorflow-${TF_VERSION}*.whl"
+
 STAGE="deb-package"
 
 echo "Building ${IMAGE_LIBTENSORFLOW_CC_ARCH} ... "
-docker build --build-arg TARGETARCH=$ARCH --build-arg TF_VERSION=${TF_VERSION} --build-arg JOBS=${JOBS} --build-arg GPU_POSTFIX=${GPU_POSTFIX} --build-arg TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} --target ${STAGE} -t ${IMAGE_LIBTENSORFLOW_CC_ARCH} ${DOCKER_DIR} | tee ${LOG_FILE}
+docker build --build-arg TARGETARCH=$ARCH --build-arg TF_VERSION=${TF_VERSION} --build-arg JOBS=${JOBS} --build-arg GPU_POSTFIX=${GPU_POSTFIX} --build-arg TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES} --build-arg BUILD_PIP_PACKAGE=${BUILD_PIP_PACKAGE} --target ${STAGE} -t ${IMAGE_LIBTENSORFLOW_CC_ARCH} ${DOCKER_DIR} | tee ${LOG_FILE}
 
 echo "Exporting to $(realpath ${EXPORT_DIR})/${EXPORT_FILE} ... "
-TMP_CONTAINER=$(docker create ${IMAGE_LIBTENSORFLOW_CC_ARCH})
+TMP_CONTAINER=$(docker run -d --rm ${IMAGE_LIBTENSORFLOW_CC_ARCH} sleep infinity)
 docker cp ${TMP_CONTAINER}:/${CONTAINER_FILE} ${EXPORT_DIR}/${EXPORT_FILE}
-docker rm -v ${TMP_CONTAINER}
+if [ "$BUILD_PIP_PACKAGE" = "1" ]; then
+    echo "Exporting to $(realpath ${EXPORT_DIR_PIP}) ... "
+    docker exec ${TMP_CONTAINER} bash -c "ls /${CONTAINER_FILE_PIP}" | while read f; do docker cp ${TMP_CONTAINER}:/$f ${EXPORT_DIR_PIP}; done
+fi
+docker kill ${TMP_CONTAINER}

tensorflow-wheel/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore