From 92bfe5f26631b23d172c8be12fd9562df183d3e4 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 2 Jan 2025 16:59:14 +0100 Subject: [PATCH 1/3] dockerfile to work on centos7 locally --- docker/centos-local-build/README.MD | 63 ++++++++++++++++++++++++ docker/centos-local-build/build.sh | 73 ++++++++++++++++++++++++++++ docker/centos-local-build/dockerfile | 12 +++++ docker/centos-local-build/run.sh | 6 +++ 4 files changed, 154 insertions(+) create mode 100644 docker/centos-local-build/README.MD create mode 100644 docker/centos-local-build/build.sh create mode 100644 docker/centos-local-build/dockerfile create mode 100755 docker/centos-local-build/run.sh diff --git a/docker/centos-local-build/README.MD b/docker/centos-local-build/README.MD new file mode 100644 index 000000000..528d59213 --- /dev/null +++ b/docker/centos-local-build/README.MD @@ -0,0 +1,63 @@ +# Locally generate a centos 7 asset + +The goal of this image is to provide a CentOS 7 environment to build xpansion and generate the asset. +One use case is when wanting to test things on OPF environments. This alleviates the need to have a CentOS 7 machine or rely on CI pipelines to generate the asset. + +## Prerequisites + +Docker installed, proxy configured if necessary. + +## Usage + +Simple use: + + ```./run.sh ``` + +## Content + +* dockerfile: Dockerfile to build the image +* build.sh: Script to build and generate asset +* run.sh: Script to "one line" the process + +## How does it work? + +The dockerfile contains the instruction to build an image with the stables requirements to build Xpansion. +The build.sh script will be part of the image and set as entrypoint of containers created from this image. +The build will happen in a container. This is to avoid rebuilding the image everytime the source code changes. +The build.sh script and thus running a container require two arguments: one for source code and one for the output directory. + +## Output directory + +The output directory is in fact a binary_directory. It will contain the generated asset but also binary cache for subsequent builds. + +## Advance usage + +* Build the image. Try to set the current directory to the dockerfile directory. Docker use the current directory as context, and it may slow down your build process. + + docker build -t . + +* Run the container with the image. The container will be removed after the build. + + docker run \ + --mount type=bind,src=,dst=/mnt/sources \ + --mount type=bind,src=,dst=/mnt/caches \ + \ + /mnt/sources /mnt/caches + +* Run a container but don't do anything. This is useful to run the build yourself + + docker run -it \ + --entry-point /bin/bash \ + --mount type=bind,src=,dst=/mnt/sources \ + --mount type=bind,src=,dst=/mnt/caches \ + + +* Build inside the container. + + source ./build.sh /mnt/sources /mnt/caches +`source` command is used to load the proper environment variables and run the script a first time. Subsequent build can be done with the usual cmake commands. + +## Improvements + +Some improvements can be made to the process. It would be nice to have md5sum of downloaded dependencies to prevent redownloading them and of course store them in the cach directory. + diff --git a/docker/centos-local-build/build.sh b/docker/centos-local-build/build.sh new file mode 100644 index 000000000..9843fc211 --- /dev/null +++ b/docker/centos-local-build/build.sh @@ -0,0 +1,73 @@ +#!/bin/bash +#Define source directory +xpansion_sources=$1 +ln -s $xpansion_sources /workspace/antares-xpansion + +#Define binary cache directory +cache_dir=$2 +ccache_cache_dir=$(realpath "$cache_dir/ccache") +vcpkg_cache_dir=$(realpath "$cache_dir/vcpkg_cache") +build_dir=$(realpath "$cache_dir/build") +install_dir=$(realpath "$cache_dir/install") +mkdir $ccache_cache_dir +mkdir $vcpkg_cache_dir +export CCACHE_DIR=$ccache_cache_dir/ccache/.ccache +export CCACHE_BASEDIR=$ccache_cache_dir/ccache +export CCACHE_COMPRESS=1 +export PATH="/usr/lib/ccache:$PATH" + +export VCPKG_ROOT=/workspace/antares-xpansion/vcpkg +ORTOOLS_TAG=$(cat /workspace/antares-xpansion/antares-version.json | jq -r '."or-tools-rte"' ) +echo "OR-Tools tag :" $ORTOOLS_TAG +URL_ORTOOLS=https://github.com/rte-france/or-tools-rte/releases/download/v$ORTOOLS_TAG/ortools_cxx_centos7_static_sirius.zip +echo "Downloading " $URL_ORTOOLS +mkdir -p ortools +pushd ortools +wget -O ortools.zip $URL_ORTOOLS +echo "Done" +unzip -q ortools.zip +rm -f ortools.zip +popd + +ANTARES_VERSION=$(cat /workspace/antares-xpansion/antares-version.json | jq -r '."antares_version"' ) +echo "ANTARES_VERSION=$ANTARES_VERSION" +mkdir -p deps +URL_ANTARES=https://github.com/AntaresSimulatorTeam/Antares_Simulator/releases/download/v$ANTARES_VERSION/antares-${ANTARES_VERSION}-CentOS-7.9.2009.tar.gz +wget $URL_ANTARES +tar -xvf antares-${ANTARES_VERSION}-CentOS-7.9.2009.tar.gz -C deps --strip-components=1 &&\ +rm -rf antares-${ANTARES_VERSION}-CentOS-7.9.2009.tar.gz + +pip3 install --upgrade pip +pip3 install wheel +pip3 install -r /workspace/antares-xpansion/requirements-tests.txt + +source /opt/rh/devtoolset-11/enable +source /opt/rh/rh-git227/enable +export VCPKG_BINARY_SOURCES="clear;files,$vcpkg_cache_dir,readwrite" +cmake -B $build_dir -S /workspace/antares-xpansion \ + -DBUILD_TESTING=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_PREFIX_PATH="/workspace/deps;/workspace/ortools/install" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$install_dir \ + -DALLOW_RUN_AS_ROOT=ON \ + -DVCPKG_TARGET_TRIPLET=x64-linux-release \ + -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_INSTALL_OPTIONS="--x-buildtrees-root=$build_dir/vcpkg_buildtrees" + +cmake --build $build_dir --config Release -j`nproc` + +cd $build_dir +cmake --install . +cd $install_dir + +rm -f ./antares-xpansion-launcher* +pyinstaller -F /workspace/antares-xpansion/src/python/launch.py -n antares-xpansion-launcher --add-data "/workspace/antares-xpansion/src/python/config.yaml:." --add-data "./bin/:bin" +mv ./dist/antares-xpansion-launcher* . +rm -rf bin +rm -rf build +rm -rf dist +rm -f *.spec +cd .. +tar -czf antares-xpansion-centos.tar.gz -C $install_dir . --exclude='examples' \ No newline at end of file diff --git a/docker/centos-local-build/dockerfile b/docker/centos-local-build/dockerfile new file mode 100644 index 000000000..6895f3042 --- /dev/null +++ b/docker/centos-local-build/dockerfile @@ -0,0 +1,12 @@ +FROM antaresrte/xpansion-centos7:1.1.1 + +CMD ["/bin/bash"] + +RUN mkdir /workspace +WORKDIR /workspace + +ADD build.sh /workspace +RUN chmod +x /workspace/build.sh + +#Every container will run this +ENTRYPOINT ["/workspace/build.sh"] \ No newline at end of file diff --git a/docker/centos-local-build/run.sh b/docker/centos-local-build/run.sh new file mode 100755 index 000000000..ba47b0b08 --- /dev/null +++ b/docker/centos-local-build/run.sh @@ -0,0 +1,6 @@ +docker build -t xpansion/centos7 . +docker run \ +--mount type=bind,src=/home/marechaljas/CLionProjects/antares-xpansion,dst=/mnt/sources \ +--mount type=bind,src=/home/marechaljas/CLionProjects/docker_caches,dst=/mnt/caches \ +xpansion/centos7 \ +/mnt/sources /mnt/caches \ No newline at end of file From 1bbf405f7a236b82f8f486b0be06ff755cd048af Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Fri, 3 Jan 2025 10:39:23 +0100 Subject: [PATCH 2/3] Update run.sh now accepting parameters instead of hard coded path --- docker/centos-local-build/run.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/centos-local-build/run.sh b/docker/centos-local-build/run.sh index ba47b0b08..fbe451ed8 100755 --- a/docker/centos-local-build/run.sh +++ b/docker/centos-local-build/run.sh @@ -1,6 +1,10 @@ +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi docker build -t xpansion/centos7 . docker run \ ---mount type=bind,src=/home/marechaljas/CLionProjects/antares-xpansion,dst=/mnt/sources \ ---mount type=bind,src=/home/marechaljas/CLionProjects/docker_caches,dst=/mnt/caches \ +--mount type=bind,src=$1,dst=/mnt/sources \ +--mount type=bind,src=$2,dst=/mnt/caches \ xpansion/centos7 \ /mnt/sources /mnt/caches \ No newline at end of file From c45a860825da32872ed4a1e8d5c0b50f99c05440 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Tue, 7 Jan 2025 16:05:20 +0100 Subject: [PATCH 3/3] Fix permissions of asset output --- docker/centos-local-build/build.sh | 12 +++++++----- docker/centos-local-build/dockerfile | 5 +++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docker/centos-local-build/build.sh b/docker/centos-local-build/build.sh index 9843fc211..ca1989c46 100644 --- a/docker/centos-local-build/build.sh +++ b/docker/centos-local-build/build.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -x #Define source directory xpansion_sources=$1 ln -s $xpansion_sources /workspace/antares-xpansion @@ -16,6 +17,10 @@ export CCACHE_BASEDIR=$ccache_cache_dir/ccache export CCACHE_COMPRESS=1 export PATH="/usr/lib/ccache:$PATH" +pip3 install --upgrade pip +pip3 install wheel +pip3 install -r /workspace/antares-xpansion/requirements-tests.txt + export VCPKG_ROOT=/workspace/antares-xpansion/vcpkg ORTOOLS_TAG=$(cat /workspace/antares-xpansion/antares-version.json | jq -r '."or-tools-rte"' ) echo "OR-Tools tag :" $ORTOOLS_TAG @@ -37,10 +42,6 @@ wget $URL_ANTARES tar -xvf antares-${ANTARES_VERSION}-CentOS-7.9.2009.tar.gz -C deps --strip-components=1 &&\ rm -rf antares-${ANTARES_VERSION}-CentOS-7.9.2009.tar.gz -pip3 install --upgrade pip -pip3 install wheel -pip3 install -r /workspace/antares-xpansion/requirements-tests.txt - source /opt/rh/devtoolset-11/enable source /opt/rh/rh-git227/enable export VCPKG_BINARY_SOURCES="clear;files,$vcpkg_cache_dir,readwrite" @@ -70,4 +71,5 @@ rm -rf build rm -rf dist rm -f *.spec cd .. -tar -czf antares-xpansion-centos.tar.gz -C $install_dir . --exclude='examples' \ No newline at end of file +tar -czf antares-xpansion-centos.tar.gz -C $install_dir . --exclude='examples' +chmod 777 antares-xpansion-centos.tar.gz \ No newline at end of file diff --git a/docker/centos-local-build/dockerfile b/docker/centos-local-build/dockerfile index 6895f3042..b4e3fef3e 100644 --- a/docker/centos-local-build/dockerfile +++ b/docker/centos-local-build/dockerfile @@ -2,11 +2,12 @@ FROM antaresrte/xpansion-centos7:1.1.1 CMD ["/bin/bash"] -RUN mkdir /workspace +RUN mkdir /workspace \ + && chmod a+w /workspace + WORKDIR /workspace ADD build.sh /workspace RUN chmod +x /workspace/build.sh - #Every container will run this ENTRYPOINT ["/workspace/build.sh"] \ No newline at end of file