From 9027f4a2858264853f757ef95ca4c2a566489f39 Mon Sep 17 00:00:00 2001
From: Henri Menke <henri@henrimenke.de>
Date: Fri, 1 Dec 2023 08:41:45 +0100
Subject: [PATCH] Use ccache inside of docker build

---
 .github/workflows/docker.yml |  6 +++++
 Docker/github_ci_dockerfile  | 38 +++++++++++++++++++++---------
 Docker/mpich_dockerfile      | 25 +++++++++++++-------
 Docker/openmpi_dockerfile    | 45 +++++++++++++++++++++++++-----------
 4 files changed, 82 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 0f0007b1..534820b3 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -41,6 +41,12 @@ jobs:
         with:
           images: ghcr.io/${{ github.repository }}_${{ matrix.name }}
 
+      - name: Inject ccache into Docker
+        uses: reproducible-containers/buildkit-cache-dance@v2
+        with:
+          cache-source: var-cache-ccache
+          cache-target: /var/cache/ccache
+
       - name: Build and push Docker image
         uses: docker/build-push-action@v5
         with:
diff --git a/Docker/github_ci_dockerfile b/Docker/github_ci_dockerfile
index 63b202f7..d729d016 100644
--- a/Docker/github_ci_dockerfile
+++ b/Docker/github_ci_dockerfile
@@ -1,6 +1,7 @@
 FROM ubuntu:22.04 AS base
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+      ccache \
       cmake \
       g++-12 \
       gfortran-12 \
@@ -64,11 +65,20 @@ RUN pip3 install --no-cache-dir \
 
 FROM base AS pkgs
 
-ENV OMP_NUM_THREADS=1
-ENV MKL_NUM_THREADS=1
-ENV OMPI_ALLOW_RUN_AS_ROOT=1
-ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
-ENV OMPI_MCA_rmaps_base_oversubscribe="yes"
+ENV CMAKE_C_COMPILER_LAUNCHER=ccache \
+    CMAKE_CXX_COMPILER_LAUNCHER=ccache \
+    CCACHE_COMPILERCHECK=content \
+    CCACHE_BASEDIR=/triqs \
+    CCACHE_DIR=/var/cache/ccache \
+    CCACHE_MAXSIZE=500M \
+    CCACHE_SLOPPINESS=pch_defines,time_macros,include_file_mtime,include_file_ctime \
+    CCACHE_COMPRESS="1" \
+    CCACHE_COMPRESSLEVEL="1" \
+    OMP_NUM_THREADS=1 \
+    MKL_NUM_THREADS=1 \
+    OMPI_ALLOW_RUN_AS_ROOT=1 \
+    OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \
+    OMPI_MCA_rmaps_base_oversubscribe="yes"
 ARG NCORES=8
 
 # create source dirs
@@ -88,37 +98,43 @@ ENV CPATH=/triqs/include:${CPATH} \
     CTEST_PARALLEL_LEVEL=${NCORES} \
     TRIQS_ROOT=/triqs
 
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/triqs \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/triqs \
     && cd triqs && mkdir -p build && cd build \
     && cmake ../ -DCMAKE_INSTALL_PREFIX=/triqs \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # dft_tools
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/dft_tools.git \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/dft_tools.git \
     && cd dft_tools && mkdir -p build && cd build \
     && cmake ../ \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # cthyb
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/cthyb.git \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/cthyb.git \
     && cd cthyb && mkdir -p build && cd build \
     && cmake ../ \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # hubbardI
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hubbardI.git \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hubbardI.git \
     && cd hubbardI && mkdir -p build && cd build \
     && cmake ../ \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # Hartree-Fock solver
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hartree_fock.git \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hartree_fock.git \
     && cd hartree_fock && mkdir -p build && cd build \
     && cmake ../ \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # maxent needed for doc build
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/maxent.git \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/maxent.git \
     && cd maxent && mkdir -p build && cd build \
     && cmake ../ \
     && make -j$NCORES && ctest -j$NCORES && make install
diff --git a/Docker/mpich_dockerfile b/Docker/mpich_dockerfile
index 97c5d032..56ebf481 100644
--- a/Docker/mpich_dockerfile
+++ b/Docker/mpich_dockerfile
@@ -14,6 +14,7 @@ RUN apt-get update && \
     libmpich-dev \
     mpich \
     build-essential \
+    ccache \
     cmake \
 # OpenMP problem
     libmkl-gnu-thread \
@@ -130,49 +131,57 @@ ENV TRIQS_ROOT=/triqs
 # triqs
 RUN cd / && mkdir -p triqs && mkdir -p source
 
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/triqs triqs.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/triqs triqs.src \
     && mkdir -p triqs.build && cd triqs.build \
     && cmake ../triqs.src -DCMAKE_INSTALL_PREFIX=/triqs -DBLA_VENDOR=Intel10_64_dyn \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # dft_tools
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/dft_tools.git dft_tools.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/dft_tools.git dft_tools.src \
     && mkdir -p dft_tools.build && cd dft_tools.build \
     && cmake ../dft_tools.src -DCMAKE_INSTALL_PREFIX=/triqs \
     && make -j$NCORES && ctest -j$NCORES test && make install
 
 # cthyb
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/cthyb.git cthyb.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/cthyb.git cthyb.src \
     && mkdir -p cthyb.build && cd cthyb.build \
     && cmake ../cthyb.src -DCMAKE_INSTALL_PREFIX=/triqs \
     && make -j$NCORES && make -j$NCORES test && make install
 
 # maxent
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/maxent.git maxent.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/maxent.git maxent.src \
     && mkdir -p maxent.build && cd maxent.build \
     && cmake ../maxent.src -DCMAKE_INSTALL_PREFIX=/triqs \
     && make -j$NCORES && make -j$NCORES test && make install
 
 # ctint
 # ADD ctint-3.0.x.tar.gz /source/ctint.src
-# RUN cd /source && mkdir -p ctint.build && cd ctint.build \
+# RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && mkdir -p ctint.build && cd ctint.build \
 #     && cmake ../ctint.src -DCMAKE_INSTALL_PREFIX=/triqs \
 #     && make -j$NCORES && make -j$NCORES test && make install
 
 # hubbardI
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hubbardI hubbardI.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hubbardI hubbardI.src \
     && mkdir -p hubbardI.build && cd hubbardI.build \
     && cmake ../hubbardI.src -DCMAKE_INSTALL_PREFIX=/triqs \
     && make -j$NCORES && make -j$NCORES test && make install
 
 # hartree_fock
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hartree_fock.git hartree_fock.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hartree_fock.git hartree_fock.src \
     && mkdir -p hartree_fock.build && cd hartree_fock.build \
     && cmake ../hartree_fock.src -DCMAKE_INSTALL_PREFIX=/triqs \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # solid_dmft
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/flatironinstitute/solid_dmft.git solid_dmft.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/flatironinstitute/solid_dmft.git solid_dmft.src \
     && mkdir -p solid_dmft.build && cd solid_dmft.build \
     && cmake ../solid_dmft.src -DCMAKE_INSTALL_PREFIX=/triqs \
     && make -j$NCORES && make -j$NCORES test && make install
diff --git a/Docker/openmpi_dockerfile b/Docker/openmpi_dockerfile
index 6e62442e..1a515a1b 100644
--- a/Docker/openmpi_dockerfile
+++ b/Docker/openmpi_dockerfile
@@ -9,6 +9,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -
 
 # This platform includes dependencies for building docs
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      ccache \
       cmake \
       g++ \
       gfortran \
@@ -106,10 +107,19 @@ ENV PYTHON_VERSION=3.10 \
     CC=clang-${LLVM} CXX=clang++-${LLVM} CXXFLAGS="-stdlib=libc++"
 RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM} 60 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM} --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${LLVM}
 
-ENV OMP_NUM_THREADS=1
-ENV MKL_NUM_THREADS=1
-ENV MKL_INTERFACE_LAYER=GNU,LP64
-ENV MKL_THREADING_LAYER=SEQUENTIAL
+ENV CMAKE_C_COMPILER_LAUNCHER=ccache \
+    CMAKE_CXX_COMPILER_LAUNCHER=ccache \
+    CCACHE_COMPILERCHECK=content \
+    CCACHE_BASEDIR=/triqs \
+    CCACHE_DIR=/var/cache/ccache \
+    CCACHE_MAXSIZE=500M \
+    CCACHE_SLOPPINESS=pch_defines,time_macros,include_file_mtime,include_file_ctime \
+    CCACHE_COMPRESS="1" \
+    CCACHE_COMPRESSLEVEL="1" \
+    OMP_NUM_THREADS=1 \
+    MKL_NUM_THREADS=1 \
+    MKL_INTERFACE_LAYER=GNU,LP64 \
+    MKL_THREADING_LAYER=SEQUENTIAL
 ARG NCORES=10
 
 # create source dirs
@@ -127,7 +137,8 @@ RUN cd /source && git clone -b develop --depth 1 https://github.com/wannier-deve
     && cp src/obj/* /triqs/include/
 
 # install qe
-RUN cd /source && wget --quiet https://gitlab.com/QEF/q-e/-/archive/qe-7.2/q-e-qe-7.2.tar.gz \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && wget --quiet https://gitlab.com/QEF/q-e/-/archive/qe-7.2/q-e-qe-7.2.tar.gz \
    && tar -xf q-e-qe-7.2.tar.gz \
    && cd q-e-qe-7.2 \
    && mkdir -p build && cd build \
@@ -172,49 +183,57 @@ ENV CPATH=/triqs/include:/usr/include/mkl:${CPATH} \
     CTEST_PARALLEL_LEVEL=${NCORES} \
     TRIQS_ROOT=/triqs
 
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/triqs triqs.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/triqs triqs.src \
     && mkdir -p triqs.build && cd triqs.build \
     && cmake ../triqs.src -DCMAKE_INSTALL_PREFIX=/triqs -DMPIEXEC_PREFLAGS='--allow-run-as-root' -DBLA_VENDOR=Intel10_64_dyn \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # dft_tools
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/dft_tools.git dft_tools.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/dft_tools.git dft_tools.src \
     && mkdir -p dft_tools.build && cd dft_tools.build \
     && cmake ../dft_tools.src -DMPIEXEC_PREFLAGS='--allow-run-as-root' \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # cthyb
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/cthyb.git cthyb.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/cthyb.git cthyb.src \
     && mkdir -p cthyb.build && cd cthyb.build \
     && cmake ../cthyb.src -DMPIEXEC_PREFLAGS='--allow-run-as-root'  \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # maxent
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/maxent.git maxent.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/maxent.git maxent.src \
     && mkdir -p maxent.build && cd maxent.build \
     && cmake ../maxent.src \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # TPRF
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/tprf.git tprf.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/tprf.git tprf.src \
     && mkdir -p tprf.build && cd tprf.build \
     && cmake ../tprf.src -DMPIEXEC_PREFLAGS='--allow-run-as-root'  \
     && make -j$NCORES && make test && make install
 
 # hubbardI
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hubbardI.git hubbardI.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hubbardI.git hubbardI.src \
     && mkdir -p hubbardI.build && cd hubbardI.build \
     && cmake ../hubbardI.src \ 
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # hartree_fock
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hartree_fock.git hartree_fock.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/TRIQS/hartree_fock.git hartree_fock.src \
     && mkdir -p hartree_fock.build && cd hartree_fock.build \
     && cmake ../hartree_fock.src \
     && make -j$NCORES && ctest -j$NCORES && make install
 
 # download solid_dmft and install
-RUN cd /source && git clone -b unstable --depth 1 https://github.com/flatironinstitute/solid_dmft.git solid_dmft.src \
+RUN --mount=type=cache,target=/var/cache/ccache,sharing=locked \
+    cd /source && git clone -b unstable --depth 1 https://github.com/flatironinstitute/solid_dmft.git solid_dmft.src \
     && mkdir -p solid_dmft.build && cd solid_dmft.build \
     && cmake ../solid_dmft.src -DMPIEXEC_PREFLAGS='--allow-run-as-root' \
     && make test && make install