From 050af9a30cbfc300216a27562925dfb58f410fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20Kerbiriou?= Date: Wed, 29 Mar 2017 17:23:26 +0200 Subject: [PATCH] Add docker and compilation script for wheels manylinux1 --- .gitignore | 2 +- docker/Dockerfile.manylinux | 133 ++++++++++++++++++++++++++++++++ docker/build-manylinux-inner.sh | 43 +++++++++++ docker/build-manylinux.sh | 40 ++++++++++ 4 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile.manylinux create mode 100644 docker/build-manylinux-inner.sh create mode 100644 docker/build-manylinux.sh diff --git a/.gitignore b/.gitignore index 349a30c..cef90d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/build +/build*/ tags **/.ipynb_checkpoints/ **/*.tar* diff --git a/docker/Dockerfile.manylinux b/docker/Dockerfile.manylinux new file mode 100644 index 0000000..f84b2af --- /dev/null +++ b/docker/Dockerfile.manylinux @@ -0,0 +1,133 @@ +####################################################################################### +# +# Dockerfile to start a pyGATB/Ptyhon compiling machine using these dependencies: +# +# --> gcc 4.9 +# --> CMake 3.7.2 +# --> Python 3.4 +# --> Cython 0.25 +# +# See below to change these values. +# +#-------------------------------------------------------------------------------------- +# +# Use: +# +# ### To build the container, use: +# +# docker build -f Dockerfile.py -t py_gatb_machine . +# +# ### To run the container. +# +# Running the container means that you want to compile pyGATB. For that +# purpose, docker run expects some information, as illustrated in this +# command: +# +# docker run +# -i -t +# -e "GIT_BRANCH=master" <-- +# -v /path/to/py-gatb-source:/tmp/py-gatb-code <-- source code +# -v /path/to/py-gatb-build:/tmp/py-gatb-build <-- compiled code (optional) +# py_gatb_machine <-- container to start +# /tmp/py-gatb-code/pyGATB/docker/py-gatb-compile.sh <-- script to run +# -DCMAKE_CXX_FLAGS_RELEASE="-march=native -Ofast -DNDEBUG" <-- cmake arguments +# +# First of all, we have retain that the code is not compiled within the +# container. Instead we use two external volumes bound to the container using +# two docker run "-v" arguments. These two volumes simply target: +# +# 1. a directory containing the pyGATB source code, i.e. a "git clone" of +# pyGATB repository; +# 2. a directory containing the compiled code. +# +# Using such a design, you can work with an existing clone of pyGATB +# repository and you can easily access the compiled code. +# +# pyGATB source code directory (hereafter denoted as "py-gatb-source") must +# exist on the host system, but it can be empty. In such a case, the container +# will do the git clone. Thus, py-gatb-source is passed to docker run as +# follows: +# +# -v /full/path/to/your/py-gatb-source:/tmp/py-gatb-code +# +# (do not modify "/tmp/py-gatb-code": this is the mount path within the +# container) +# +# pyGATB compiled code directory (hereafter denoted as "py-gatb-build") +# must also exist on the host system. In all case, the container will erase its +# content before running the code compiling procedure. Thus, py-gatb-build +# is passed to docker run as follows: +# +# -v /full/path/to/your/py-gatb-build:/tmp/py-gatb-build +# +# (do not modify "/tmp/py-gatb-build": this is the mount path within the +# container) +# +# Finally, the docker run also accepts an optional environment variable: the +# pyGATB branch to compile. Simply pass that information using the "-e" +# argument of docker run as follows: +# +# -e "GIT_BRANCH=master" +# +# replace "master" by an appropriate value, i.e. a git branch or tag. +# +# If "-e" is not provided to docker run, then the master branch of pyGATB +# is compiled. +# +# All in all, the pyGATB compiler machine can be started as follows: +# +# docker run --name py_gatb_machine \ +# -i -t \ <-- remove if running from Jenkins/slave +# (TTY not allowed) +# -e "GIT_BRANCH=master" +# -v /path/to/py-gatb-source:/tmp/py-gatb-code \ +# -v /path/to/py-gatb-build:/tmp/py-gatb-build \ +# py_gatb_machine +# +# +# Sample command from the real life: docker run --name py_gatb_machine -i -t -e "GIT_BRANCH=master" -v /Users/pdurand/tmp/py-gatb/docker:/tmp/py-gatb-code -v /Users/pdurand/tmp/py-gatb/docker:/tmp/py-gatb-build py_gatb_machine +# +# ### Additional notes +# +# Root access inside the container: +# +# - if running: docker exec -it py_gatb_machine bash +# +# - if not yet running: docker run --rm -i -t py_gatb_machine bash +# +####################################################################################### + +# ### +# Base commands +# +# We use a manylinux docker image, which is a CentOS 5 with many CPython +# version installed. +# +FROM quay.io/pypa/manylinux1_x86_64:latest + +# who to blame? +LABEL mainteners="Patrick Durand ; Maël Kerbiriou " + +# ### +# Configuring gcc and cmake release +# +ENV CMAKE_SERIES=3.7 \ + CMAKE_VERSION=3.7.2 \ +# How many (make) jobs to run in parallel ? + PARALLEL_OPT="-j4" + +# ### +# Build Dependencies +# + +# Install library dependencies +RUN yum install -y zlib-devel \ +# Install cython + && /opt/python/cp36-cp36m/bin/pip install --no-cache cython \ + && cp /opt/python/cp36-cp36m/bin/cython /usr/local/bin/ \ +# CMake installation + && cd /opt \ + && export CMAKE_URL="http://cmake.org/files/v${CMAKE_SERIES}/cmake-${CMAKE_VERSION}.tar.gz" \ + && wget --no-check-certificate ${CMAKE_URL} -O - | tar xzf - \ + && cd cmake-${CMAKE_VERSION} \ + && ./bootstrap && make ${PARALLEL_OPT} && make install && cd /opt && rm -rf cmake-${CMAKE_VERSION} diff --git a/docker/build-manylinux-inner.sh b/docker/build-manylinux-inner.sh new file mode 100644 index 0000000..f917d86 --- /dev/null +++ b/docker/build-manylinux-inner.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -eo pipefail + +# Where to find the pyGATB repository +[ -z "$PYGATB_SOURCE" ] && PYGATB_SOURCE="/tmp/pyGATB" +# Build directory +[ -z "$PYGATB_BUILD" ] && PYGATB_BUILD="${PYGATB_SOURCE}/build-manylinux" +# Where to put the wheels +[ -z "$WHEELHOUSE" ] && WHEELHOUSE="${PYGATB_BUILD}/wheelhouse" +[ -z "$PARALLEL_OPT" ] && PARALLEL_OPT="-j4" + +PYGATB_VERSION=$(cat "$PYGATB_SOURCE/src/VERSION") + + +# Prepare a fresh build directory +mkdir -p "$PYGATB_BUILD" +cd "$PYGATB_BUILD" +if [[ ! -f keep-build ]]; then + find . -mindepth 1 -delete +fi + + +for PY_MINOR in 3 4 5 6 +do + PY_BIN_DIR=/opt/python/cp3${PY_MINOR}-cp3${PY_MINOR}m/bin/ + PY_INCLUDE_DIR=/opt/python/cp3${PY_MINOR}-cp3${PY_MINOR}m/include/python3.${PY_MINOR}m/ + cmake . "$PYGATB_SOURCE" -DCMAKE_BUILD_TYPE=Release -DENABLE_LTO=ON \ + -DPYTHON_EXECUTABLE="${PY_BIN_DIR}/python" \ + -DPYTHON_INCLUDE_DIR="$PY_INCLUDE_DIR" + + make $PARALLEL_OPT + + "${PY_BIN_DIR}/pip" wheel . -w "$WHEELHOUSE" + + auditwheel repair "${WHEELHOUSE}/pyGATB-${PYGATB_VERSION}-cp3${PY_MINOR}-cp3${PY_MINOR}m-linux_x86_64.whl" + ls "${WHEELHOUSE}/pyGATB-${PYGATB_VERSION}-cp3${PY_MINOR}-cp3${PY_MINOR}m-manylinux1_x86_64.whl" + rm "${WHEELHOUSE}/pyGATB-${PYGATB_VERSION}-cp3${PY_MINOR}-cp3${PY_MINOR}m-linux_x86_64.whl" +done + + +# PYGATB_BUILD="$PYGATB_SOURCE/build- +# +# cmake . ../pyGATB -DCMAKE_BUILD_TYPE=Release -DENABLE_LTO=ON -DPYTHON_EXECUTABLE=/opt/python/cp3{PY_MINOR}-cp3{PY_MINOR}m/bin/python -DPYTHON_INCLUDE_DIR=/opt/python/cp3{PY_MINOR}-cp3{PY_MINOR}m/include/python3{PY_MINOR}m/ && make -j4 /opt/python/cp3{PY_MINOR}-3{PY_MINOR}m/bin/pip wheel . diff --git a/docker/build-manylinux.sh b/docker/build-manylinux.sh new file mode 100644 index 0000000..49f8829 --- /dev/null +++ b/docker/build-manylinux.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -eo pipefail + + +cd $(dirname $0) + +# Where to find the pyGATB repo (directory on the host containing the repo dir) +[ -z "$PYGATB_SOURCE" ] && PYGATB_SOURCE=$(realpath "..") +[ -z "$PARALLEL_OPT" ] && PARALLEL_OPT="-j4" + + + +# Run compilation (kill switch: "docker container stop pygatb-alpine-compilation") +docker run --rm --name "pygatb-alpine-compilation" \ + -v "$PYGATB_SOURCE:/tmp/pyGATB" \ + -e "PARALLEL_OPT=$PARALLEL_OPT" \ + -it pygatb/manylinux1_x86_64 bash /tmp/pyGATB/docker/build-manylinux-inner.sh + +exit 1 + +# Extract the egg for subsequent docker builds +PYGATB_EGG=($PYGATB_BUILD/$PYGATB_BUILD_DIRNAME/dist/pyGATB*.egg) +cp "$PYGATB_EGG" . +PYGATB_EGG=$(basename "$PYGATB_EGG") + +# Copy samples in work dir +cp -r "$PYGATB_SOURCE/pyGATB/samples" . + +# Runtime container +docker build -f Dockerfile.alpine_runtime -t pygatb/alpine_runtime --build-arg "PYGATB_EGG=$PYGATB_EGG" . + +rm "$PYGATB_EGG" +rm -rf samples + +# Jupyter notebook container + +docker build -f Dockerfile.alpine_notebook -t pygatb/alpine_notebook . + +OUT_ARCHIVE="$IMAGES_ARCHIVE_DIR/docker_images.tar.xz" +rm -f "$OUT_ARCHIVE" +docker save pygatb/alpine_notebook pygatb/alpine_runtime | xz > $OUT_ARCHIVE