Merge pull request #466 from mkavulich/feature/macos_and_docker_build

Update docker container, add Docker auto-build and test, update documentation for building with spack-stack

.github/workflows/build_and_push_docker_latest.yml

@@ -0,0 +1,34 @@
+name: build_test_and_push_docker
+
+on:
+  push:
+    branches:
+      # Only build containers when pushing to main
+      - "main"
+
+env:
+  LATEST_TAG: dtcenter/ccpp-scm:latest
+
+jobs:
+  docker:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push latest tag
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ env.LATEST_TAG }}

.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml

@@ -1,6 +1,6 @@
 name: CI test to build the CCPP-SCM on ubuntu v22.04
 
-on: [push,pull_request,workflow_dispatch]
+on: [pull_request,workflow_dispatch]
 
 jobs:
 

.github/workflows/ci_test_docker.yml

@@ -0,0 +1,29 @@
+name: build_test_and_push_docker
+
+on: [pull_request,workflow_dispatch]
+
+env:
+  TEST_TAG: dtcenter/ccpp-scm:test
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and export test tag
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/Dockerfile
+          load: true
+          tags: ${{ env.TEST_TAG }}
+      - name: Test
+        run: |
+          mkdir $HOME/output
+          chmod a+rw $HOME/output
+          docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d

docker/Dockerfile

@@ -1,55 +1,81 @@
-FROM dtcenter/common-community-container:gnu9
+FROM debian:12
+MAINTAINER Michael Kavulich <[email protected]>
 
-MAINTAINER Michelle Harrold <[email protected]> or Grant Firl <[email protected]> or Michael Kavulich <[email protected]>
+# Set up base OS environment
 
-#
-# Dockerfile for building CCPP SCM container
-#
+RUN apt-get -y update
 
-# Note: The common community container image contains the following packages, which are used to build the SCM:
-# gfortran, gcc, cmake, netCDF, HDF5, ZLIB, SZIP, Python, and libxml2
-# To access the common community container repository: https://github.com/NCAR/Common-Community-Container
+# Get "essential" tools and libraries
+RUN apt-get -y install build-essential \
+  && apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim \
+  && apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 \
+  && apt-get -y install python3-pip python3.11-venv
 
-# Obtain CCPP SCM source code
-RUN cd /comsoftware \
-  && git clone --recursive -b release/public-v6 https://github.com/NCAR/ccpp-scm
-
-# Obtain static data that was previously stored in repository
-RUN cd /comsoftware/ccpp-scm/ \
-  && . contrib/get_all_static_data.sh
-
-# Obtain the pre-computed look-up tables for running with Thompson microphysics
-RUN cd /comsoftware/ccpp-scm/ \
-  && . contrib/get_thompson_tables.sh
-
-# Run the machine setup script to set environment variables
-ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/g++
-ENV F77=/opt/rh/devtoolset-9/root/usr/bin/gfortran
-ENV F90=/opt/rh/devtoolset-9/root/usr/bin/gfortran
-ENV FC=/opt/rh/devtoolset-9/root/usr/bin/gfortran
-
-ENV NETCDF=/comsoftware/libs/netcdf
-
-RUN cd /comsoftware/ccpp-scm/contrib \
-  && wget https://raw.githubusercontent.com/NCAR/ccpp-scm/3f501aa8af0fb00ff124d8301c932292d1d0abf3/contrib/build_nceplibs.sh \
-  && chmod +x build_nceplibs.sh \
-  && cd .. \
-  && ./contrib/build_nceplibs.sh $PWD/nceplibs
-
-ENV bacio_ROOT /comsoftware/ccpp-scm/nceplibs
-ENV sp_ROOT /comsoftware/ccpp-scm/nceplibs
-ENV w3nco_ROOT /comsoftware/ccpp-scm/nceplibs
-
-# Create your own link from python -> python3 
-# This works without setting the system PATH env var
-# since /usr/local/bin is before /usr/bin in the search path.
-USER root
-RUN ln -s /usr/bin/python3 /usr/local/bin/python
+MAINTAINER Grant Firl <[email protected]> or Michael Kavulich <[email protected]>
+
+#Compiler environment variables
+ENV CC  /usr/bin/gcc
+ENV FC  /usr/bin/gfortran
+ENV CXX /usr/bin/g++
+ENV F77 /usr/bin/gfortran
+ENV F90 /usr/bin/gfortran
+
+# Other necessary environment variables
+ENV LD_LIBRARY_PATH /usr/lib/
+
+# Set up unpriviledged user account, set up user home space and make sure user has permissions on all stuff in /comsoftware
+RUN groupadd comusers -g 9999 \
+ && useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser \
+ && mkdir /comsoftware \
+ && chown -R comuser:comusers /home \
+ && chmod 6755 /home \
+ && chown -R comuser:comusers /comsoftware \
+ && chmod -R 6755 /comsoftware
+
+# Link version-specific aliases (python3 will be created later with virtual environment)
+RUN ln -s ~comuser/.venv/bin/python3 /usr/local/bin/python
+RUN ln -s /usr/bin/gfortran-12 /usr/bin/gfortran
+
+# all root steps completed above, now continue below as regular userID comuser
 USER comuser
+WORKDIR /home
+
+# Build NCEP libraries we need for SCM
+
+ENV NCEPLIBS_DIR /comsoftware/nceplibs
 
-# Invoke cmake on the source code to build
-RUN cd /comsoftware/ccpp-scm/scm \
+RUN mkdir -p $NCEPLIBS_DIR/src && cd $NCEPLIBS_DIR/src \
+  && git clone -b v2.4.1 --recursive https://github.com/NOAA-EMC/NCEPLIBS-bacio \
+  && mkdir NCEPLIBS-bacio/build && cd NCEPLIBS-bacio/build \
+  && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \
+  && make VERBOSE=1 \
+  && make install 
+
+RUN cd $NCEPLIBS_DIR/src \
+  && git clone -b v2.3.3 --recursive https://github.com/NOAA-EMC/NCEPLIBS-sp \
+  && mkdir NCEPLIBS-sp/build && cd NCEPLIBS-sp/build \
+  && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \
+  && make VERBOSE=1 \
+  && make install
+
+RUN cd $NCEPLIBS_DIR/src \
+  && git clone -b v2.11.0 --recursive https://github.com/NOAA-EMC/NCEPLIBS-w3emc \
+  && mkdir NCEPLIBS-w3emc/build && cd NCEPLIBS-w3emc/build \
+  && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \
+  && make VERBOSE=1 \
+  && make install
+
+ENV bacio_ROOT /comsoftware/nceplibs
+ENV sp_ROOT /comsoftware/nceplibs
+ENV w3emc_ROOT /comsoftware/nceplibs
+
+# Obtain CCPP SCM source code and static data, build code
+RUN cd /comsoftware \
+  && git clone --recursive -b main https://github.com/NCAR/ccpp-scm \
+  && cd /comsoftware/ccpp-scm/ \
+  && ./contrib/get_all_static_data.sh \
+  && ./contrib/get_thompson_tables.sh \
+  && cd /comsoftware/ccpp-scm/scm \
   && mkdir bin \
   && cd bin \
   && cmake ../src \
@@ -67,3 +93,12 @@ RUN cd /comsoftware/ccpp-scm/scm \
 WORKDIR /comsoftware/ccpp-scm/scm/bin
 ENV SCM_WORK=/comsoftware
 ENV SCM_ROOT=/comsoftware/ccpp-scm/
+
+# For interactive use, vim mouse settings are infuriating
+RUN echo "set mouse=" > ~/.vimrc
+
+# Set up python virtual environment and install needed packages
+ENV VIRTUAL_ENV=~/.venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN pip3 install f90nml==1.4.4 netcdf4==1.6.5

scm/doc/TechGuide/chap_intro.rst

@@ -103,7 +103,7 @@ This release bundle has some known limitations:
 
 -  As of this release, using the SCM over a land point with an LSM is
    possible through the use of UFS initial conditions (see 
-   :numref:`Section %s <UFS ICs>`). However, advective forcing terms
+   :numref:`Section %s <UFSreplay>`). However, advective forcing terms
    are unavailable as of this release, so only short integrations using
    this configuration should be employed. Using dynamical tendencies
    (advective forcing terms) from the UFS will be part of a future