diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml new file mode 100644 index 000000000..9e998655f --- /dev/null +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -0,0 +1,264 @@ +name: CI test to build the CCPP-SCM on ubuntu v22.04 + +on: [push,pull_request,workflow_dispatch] + +jobs: + + build_scm: + # The type of runner that the job will run on + runs-on: ubuntu-22.04 + strategy: + matrix: + fortran-compiler: [nvfortran] + build-type: [Release]#, Debug] + enable-gpu-acc: [False, True] + py-version: [3.7.13, 3.9.12] + + # Environmental variables + env: + NETCDF: /home/runner/netcdf + bacio_ROOT: /home/runner/bacio + sp_ROOT: /home/runner/NCEPLIBS-sp + w3emc_ROOT: /home/runner/myw3emc + SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm + zlib_ROOT: /home/runner/zlib + HDF5_ROOT: /home/runner/hdf5 + suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 + suites_ps: SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps + + # Workflow steps + steps: + + ####################################################################################### + # Cleanup space + ####################################################################################### + - name: Check space (pre) + run: | + df -h + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: false + dotnet: false + haskell: true + large-packages: true + docker-images: false + swap-storage: false + + - name: Check space (post) + run: | + df -h + + ####################################################################################### + # Initial + ####################################################################################### + - name: Checkout SCM code (into /home/runner/work/ccpp-scm/) + uses: actions/checkout@v3 + + - name: Initialize submodules + run: git submodule update --init --recursive + + ####################################################################################### + # Python setup + ####################################################################################### + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{matrix.py-version}} + + - name: Add conda to system path + run: | + echo $CONDA/bin >> $GITHUB_PATH + + - name: Install NetCDF Python libraries + run: | + conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml + + ####################################################################################### + # Install Nvidia. + ####################################################################################### + + - name: Nvidia setup compilers. + env: + NVCOMPILERS: /home/runner/hpc_sdk + NVARCH: Linux_x86_64 + NVHPC_SILENT: true + NVHPC_INSTALL_DIR: /home/runner/hpc_sdk + NVHPC_INSTALL_TYPE: network + NVHPC_INSTALL_LOCAL_DIR: /home/runner/hpc_sdk + run: | + mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk + wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran + echo "Removing tarball" + rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + + - name: Set environment for Nvidia compiler. + run: | + echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV + echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV + echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV + echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV + + ####################################################################################### + # Install FORTRAN dependencies + ####################################################################################### + + - name: Install zlib + env: + CFLAGS: -fPIC + run: | + wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz + tar -zxvf zlib-1.2.13.tar.gz + cd zlib-1.2.13 + ./configure --prefix=${zlib_ROOT} + make + make install + echo "LD_LIBRARY_PATH=$zlib_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Install HDF5 + env: + CPPFLAGS: -I${zlib_ROOT}/include + LDFLAGS: -L${zlib_ROOT}/lib + run: | + wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz + tar -zxvf hdf5-1_14_1-2.tar.gz + cd hdf5-hdf5-1_14_1-2 + ./configure --prefix=${HDF5_ROOT} --with-zlib=${zlib_ROOT} + make -j4 + make install + echo "LD_LIBRARY_PATH=$HDF5_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=$HDF5_ROOT/lib:$PATH" >> $GITHUB_ENV + + - name: Install Curl + run: | + sudo apt-get install curl + sudo apt-get install libssl-dev libcurl4-openssl-dev + + - name: Cache NetCDF C library + id: cache-netcdf-c + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-c + key: cache-netcdf-c-${{matrix.fortran-compiler}}-key + + - name: Install NetCDF C library + if: steps.cache-netcdf-c.outputs.cache-hit != 'true' + run: | + wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz + tar -zvxf v4.7.4.tar.gz + cd netcdf-c-4.7.4 + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" ./configure --prefix=${NETCDF} + make + make install + echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=$NETCDF/lib:$PATH" >> $GITHUB_ENV + + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + + - name: Install NetCDF Fortran library + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz + tar -zvxf v4.6.1.tar.gz + cd netcdf-fortran-4.6.1 + FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" ./configure --prefix=${NETCDF} + make + make install + + - name: Cache bacio library v2.4.1 + id: cache-bacio-fortran + uses: actions/cache@v3 + with: + path: /home/runner/bacio + key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key + + - name: Install bacio library v2.4.1 + if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio + cd bacio && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${bacio_ROOT} ../ + make -j2 + make install + echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV + + - name: Cache SP-library v2.3.3 + id: cache-sp-fortran + uses: actions/cache@v3 + with: + path: /home/runner/NCEPLIBS-sp + key: cache-sp-fortran-${{matrix.fortran-compiler}}-key + + - name: Install SP-library v2.3.3 + if: steps.cache-sp-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp + cd NCEPLIBS-sp && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${sp_ROOT} ../ + make -j2 + make install + echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV + + - name: Cache w3emc library v2.9.2 + id: cache-w3emc-fortran + uses: actions/cache@v3 + with: + path: /home/runner/myw3emc + key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key + + - name: Install w3emc library v2.9.2 + if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc + cd NCEPLIBS-w3emc && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${w3emc_ROOT} ../ + make -j2 + make install + echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV + + ####################################################################################### + # Build and run SCM regression tests (ccpp-scm/test/rt_test_cases.py) + ####################################################################################### + + - name: Configure build with CMake + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites},${suites_ps} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src + + - name: Build SCM. + run: | + cd ${SCM_ROOT}/scm/bin + make -j4 + + - name: Download data for SCM + run: | + cd ${SCM_ROOT} + ./contrib/get_all_static_data.sh + ./contrib/get_thompson_tables.sh + + - name: Run SCM RTs (w/o GPU) + if: contains(matrix.enable-gpu-acc, 'False') + run: | + cd ${SCM_ROOT}/scm/bin + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..4c335023f --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,54 @@ +title: "CCPP Single Column Model (SCM)" +version: 6.0.0 +message: "Please cite this software using the metadata from this file." +type: software +identifiers: + - description: "This is the archived snapshot of all versions of CCPP SCM" + type: doi + value: 10.5281/zenodo.6896437 + - description: "This is the archived snapshot of CCPP SCM v6.0.0" + type: doi + value: 10.5281/zenodo.6896438 +repository-code: "https://github.com/NCAR/ccpp-scm" +url: "https://dtcenter.org/community-code/common-community-physics-package-ccpp" +authors: + - given-names: Grant + family-names: Firl + affiliation: >- + Cooperative Institute for Research in the Atmosphere, + Developmental Testbed Center, + National Oceanic and Atmospheric Administration Global Systems Laboratory + orcid: 0009-0001-7076-2735 + - given-names: Dustin + family-names: Swales + affiliation: >- + Developmental Testbed Center and + National Oceanic and Atmospheric Administration Global Systems Laboratory + orcid: 0000-0002-5322-4521 + - given-names: Laurie + family-names: Carson + affiliation: >- + Developmental Testbed Center and National Center for Atmospheric Research + - given-names: Ligia + family-names: Bernardet + affiliation: >- + Developmental Testbed Center and + National Oceanic and Atmospheric Administration Global Systems Laboratory + orcid: 0000-0002-4952-4038 + - given-names: Dominikus + family-names: Heinzeller + affiliation: Joint Center for Satellite Data Assimilation + orcid: 0000-0003-2962-1049 + - given-names: Michelle + family-names: Harrold + affiliation: >- + Developmental Testbed Center and National Center for Atmospheric Research + - given-names: Tracy + family-names: Hertneky + affiliation: >- + Developmental Testbed Center and National Center for Atmospheric Research + - given-names: Michael + family-names: Kavulich + affiliation: >- + Developmental Testbed Center and National Center for Atmospheric Research +cff-version: 1.2.0 diff --git a/ccpp/physics_namelists/input_GFS_v17_H3.nml b/ccpp/physics_namelists/input_GFS_v17_H3.nml new file mode 100644 index 000000000..10d60a77e --- /dev/null +++ b/ccpp/physics_namelists/input_GFS_v17_H3.nml @@ -0,0 +1,171 @@ +&gfs_physics_nml + fhzero = 6 + h2o_phys = .true. + ldiag3d = .true. + qdiag3d = .true. + print_diff_pgr = .false. + fhcyc = 24 + use_ufo = .true. + pre_rad = .false. + imp_physics = 8 + iovr = 3 + ltaerosol = .false. + lradar = .true. + ttendlim = -999 + dt_inner = 225.0 + sedi_semi = .true. + decfl = 10 + oz_phys = .false. + oz_phys_2015 = .true. + lsoil_lsm = 4 + do_mynnedmf = .false. + do_mynnsfclay = .false. + icloud_bl = 1 + bl_mynn_edmf = 1 + bl_mynn_tkeadvect = .true. + bl_mynn_edmf_mom = 1 + do_ugwp = .false. + do_tofd = .false. + gwd_opt = 2 + do_ugwp_v0 = .false. + do_ugwp_v1 = .true. + do_ugwp_v0_orog_only = .false. + do_ugwp_v0_nst_only = .false. + do_gsl_drag_ls_bl = .true. + do_gsl_drag_ss = .true. + do_gsl_drag_tofd = .true. + do_ugwp_v1_orog_only = .false. + min_lakeice = 0.15 + min_seaice = 0.15 + use_cice_alb = .false. + pdfcld = .false. + fhswr = 1200. + fhlwr = 1200. + ialb = 1 + iems = 1 + iaer = 5111 + icliq_sw = 2 + ico2 = 2 + isubc_sw = 2 + isubc_lw = 2 + isol = 2 + lwhtr = .true. + swhtr = .true. + cnvgwd = .true. + shal_cnv = .true. + cal_pre = .false. + redrag = .true. + dspheat = .true. + hybedmf = .false. + satmedmf = .true. + isatmedmf = 1 + lheatstrg = .true. + lseaspray = .true. + random_clds = .false. + trans_trac = .true. + cnvcld = .true. + imfshalcnv = 2 + imfdeepcnv = 2 + ras = .false. + cdmbgwd = 10.0,3.5,1.0,1.0 + prslrd0 = 0. + ivegsrc = 1 + isot = 1 + lsoil = 4 + lsm = 2 + iopt_dveg = 4 + iopt_crs = 2 + iopt_btr = 1 + iopt_run = 1 + iopt_sfc = 3 + iopt_trs = 2 + iopt_frz = 1 + iopt_inf = 1 + iopt_rad = 3 + iopt_alb = 1 + iopt_snf = 4 + iopt_tbot = 2 + iopt_stc = 3 + debug = .false. + nstf_name = 2,1,0,0,0 + nst_anl = .true. + psautco = 0.0008,0.0005 + prautco = 0.00015,0.00015 + lgfdlmprad = .false. + effr_in = .true. + ldiag_ugwp = .false. + fscav_aero = "'*:0.0'" + do_sppt = .false. + do_shum = .false. + do_skeb = .false. + do_RRTMGP = .false. + doGP_cldoptics_LUT = .true. + doGP_lwscat = .true. + active_gases = 'h2o_co2_o3_n2o_ch4_o2' + ngases = 6 + rrtmgp_root = '../../ccpp/physics/physics/rte-rrtmgp/' + lw_file_gas = 'rrtmgp/data/rrtmgp-data-lw-g128-210809.nc' + lw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-lw.nc' + sw_file_gas = 'rrtmgp/data/rrtmgp-data-sw-g112-210809.nc' + sw_file_clouds = 'extensions/cloud_optics/rrtmgp-cloud-optics-coeffs-sw.nc' + rrtmgp_nGptsSW = 112 + rrtmgp_nGptsLW = 128 + rrtmgp_nBandsLW = 16 + rrtmgp_nBandsSW = 14 + frac_grid = .true. + cplchm = .false. + cplflx = .false. + cplice = .false. + cplwav = .false. + cplwav2atm = .false. + do_ca = .false. + ca_global = .false. + ca_sgs = .false. + nca = 1 + ncells = 5 + nlives = 12 + nseed = 1 + nfracseed = 0.5 + nthresh = 18 + ca_trigger = .true. + nspinup = 1 + iseed_ca = 12345 + lndp_type = 0 + n_var_lndp = 0 + do_ccpp_suite_sim = .false. +/ + +&cires_ugwp_nml + knob_ugwp_solver = 2 + knob_ugwp_version = 1 + knob_ugwp_source = 1,1,0,0 + knob_ugwp_wvspec = 1,25,25,25 + knob_ugwp_azdir = 2,4,4,4 + knob_ugwp_stoch = 0,0,0,0 + knob_ugwp_effac = 1,1,1,1 + knob_ugwp_doaxyz = 1 + knob_ugwp_doheat = 1 + knob_ugwp_dokdis = 2 + knob_ugwp_ndx4lh = 4 + knob_ugwp_palaunch = 275.0e2 + knob_ugwp_nslope = 0 + knob_ugwp_lzmax = 15.750e3 + knob_ugwp_lzmin = 0.75e3 + knob_ugwp_lzstar = 2.0e3 + knob_ugwp_taumin = 0.25e-3 + knob_ugwp_tauamp = 1.5e-3 + knob_ugwp_lhmet = 200.0e3 + knob_ugwp_orosolv = 'pss-1986' +/ + +&ccpp_suite_sim_nml + suite_sim_file = '' + nprc_sim = 7 + prc_LWRAD_cfg = 0, 0, 1 + prc_SWRAD_cfg = 0, 0, 2 + prc_PBL_cfg = 1, 0, 3 + prc_GWD_cfg = 1, 0, 4 + prc_SCNV_cfg = 1, 1, 5 + prc_DCNV_cfg = 1, 1, 6 + prc_cldMP_cfg = 1, 1, 7 +/ diff --git a/ccpp/suites/suite_SCM_GFS_v17_HR3.xml b/ccpp/suites/suite_SCM_GFS_v17_HR3.xml new file mode 100644 index 000000000..92effe13c --- /dev/null +++ b/ccpp/suites/suite_SCM_GFS_v17_HR3.xml @@ -0,0 +1,88 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmg_pre + GFS_radiation_surface + rad_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + noahmpdrv + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + ugwpv1_gsldrag + ugwpv1_gsldrag_post + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + GFS_physics_post + + + + diff --git a/ccpp/suites/suite_SCM_GFS_v17_HR3_ps.xml b/ccpp/suites/suite_SCM_GFS_v17_HR3_ps.xml new file mode 100644 index 000000000..d9e833936 --- /dev/null +++ b/ccpp/suites/suite_SCM_GFS_v17_HR3_ps.xml @@ -0,0 +1,69 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmg_pre + GFS_radiation_surface + rad_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + scm_sfc_flux_spec + dcyc2t3 + GFS_suite_interstitial_2 + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + ugwpv1_gsldrag + ugwpv1_gsldrag_post + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + GFS_physics_post + + + + diff --git a/scm/etc/Cheyenne_setup_gnu.csh b/scm/etc/Cheyenne_setup_gnu.csh deleted file mode 100755 index c37acbdc2..000000000 --- a/scm/etc/Cheyenne_setup_gnu.csh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/tcsh - -echo "Setting environment variables for CCPP-SCM on Cheyenne with gcc/gfortran" - -set called=($_) - -if ( "$called" != "") then ### called by source - set MYSCRIPT=`readlink -f -n $called[2]` -else ### called by direct execution of the script - set MYSCRIPT=`readlink -f -n '$0'` -endif -set MYDIR=`dirname $MYSCRIPT` -set MYDIR=`cd $MYDIR && pwd -P` - -setenv SCM_ROOT $MYDIR/../.. - -#start with a "clean" environment; activate and deactivate ncar_pylib in order to successfully deactivate previously activated environment without errors -module load ncarenv/1.3 -conda deactivate -module purge - -#load the modules in order to compile the CCPP SCM -echo "Loading gnu and netcdf modules..." -module load ncarenv/1.3 -module load gnu/11.2.0 -module load mpt/2.25 -module load ncarcompilers/0.5.0 -module load netcdf - -echo "Setting up NCEPLIBS" -module use /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack -module load hpc/1.2.0 -module load hpc-gnu/11.2.0 -module load hpc-mpt/2.25 -setenv bacio_ROOT /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/bacio/2.4.1 -setenv sp_ROOT /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/sp/2.3.3 -setenv w3emc_ROOT /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -setenv CC gcc -setenv CXX g++ -setenv FC gfortran - -echo "Loading cmake" -module load cmake/3.16.4 -setenv CMAKE_C_COMPILER gcc -setenv CMAKE_CXX_COMPILER g++ -setenv CMAKE_Fortran_COMPILER gfortran -setenv CMAKE_Platform cheyenne.gnu - -echo "Setting up python environment for running and plotting." -module load conda/latest - -conda activate /glade/p/ral/jntp/GMTB/CCPP_SCM/conda/ccpp-scm - diff --git a/scm/etc/Cheyenne_setup_gnu.sh b/scm/etc/Cheyenne_setup_gnu.sh deleted file mode 100755 index e1a190fcc..000000000 --- a/scm/etc/Cheyenne_setup_gnu.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -echo "Setting environment variables for CCPP-SCM on Cheyenne with gcc/gfortran" - -MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) - -export SCM_ROOT=$MYDIR/../.. - -#start with a "clean" environment; activate and deactivate ncar_pylib in order to successfully deactivate previously activated environment without errors -module load ncarenv/1.3 -conda deactivate -module purge - -#load the modules in order to compile the CCPP SCM -echo "Loading gnu and netcdf modules..." -module load ncarenv/1.3 -module load gnu/11.2.0 -module load mpt/2.25 -module load ncarcompilers/0.5.0 -module load netcdf - -echo "Setting up NCEPLIBS" -module use /glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack -module load hpc/1.2.0 -module load hpc-gnu/11.2.0 -module load hpc-mpt/2.25 -export bacio_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/bacio/2.4.1 -export sp_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/sp/2.3.3 -export w3emc_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/gnu-11.2.0/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -export CC=gcc -export CXX=g++ -export FC=gfortran - -echo "Loading cmake" -module load cmake/3.22.0 -export CMAKE_C_COMPILER=gcc -export CMAKE_CXX_COMPILER=g++ -export CMAKE_Fortran_COMPILER=gfortran -export CMAKE_Platform=cheyenne.gnu - -echo "Setting up python environment for running and plotting." -module load conda/latest - -conda activate /glade/p/ral/jntp/GMTB/CCPP_SCM/conda/ccpp-scm - diff --git a/scm/etc/Cheyenne_setup_intel.csh b/scm/etc/Cheyenne_setup_intel.csh deleted file mode 100755 index d8bc17890..000000000 --- a/scm/etc/Cheyenne_setup_intel.csh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/tcsh - -echo "Setting environment variables for CCPP-SCM on Cheyenne with icc/ifort" - -set called=($_) - -if ( "$called" != "") then ### called by source - set MYSCRIPT=`readlink -f -n $called[2]` -else ### called by direct execution of the script - set MYSCRIPT=`readlink -f -n '$0'` -endif -set MYDIR=`dirname $MYSCRIPT` -set MYDIR=`cd $MYDIR && pwd -P` - -setenv SCM_ROOT $MYDIR/../.. - -#start with a "clean" environment; activate and deactivate ncar_pylib in order to successfully deactivate previously activated environment without errors -module load ncarenv/1.3 -conda deactivate -module purge - -#load the modules in order to compile the CCPP SCM -echo "Loading intel and netcdf modules..." -module load ncarenv/1.3 -module load intel/2022.1 -module load mpt/2.25 -module load ncarcompilers/0.5.0 -module load netcdf - -echo "Setting up NCEPLIBS" -module use /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack -module load hpc/1.2.0 -module load hpc-intel/2022.1 -module load hpc-mpt/2.25 -setenv bacio_ROOT /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/bacio/2.4.1 -setenv sp_ROOT /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/sp/2.3.3 -setenv w3emc_ROOT /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -setenv CC icc -setenv CXX icpc -setenv FC ifort - -echo "Loading cmake" -module load cmake/3.22.0 -setenv CMAKE_C_COMPILER icc -setenv CMAKE_CXX_COMPILER icpc -setenv CMAKE_Fortran_COMPILER ifort -setenv CMAKE_Platform cheyenne.intel - -echo "Setting up python environment for running and plotting." -module load conda/latest - -conda activate /glade/p/ral/jntp/GMTB/CCPP_SCM/conda/ccpp-scm - diff --git a/scm/etc/Cheyenne_setup_intel.sh b/scm/etc/Cheyenne_setup_intel.sh deleted file mode 100755 index 71727e034..000000000 --- a/scm/etc/Cheyenne_setup_intel.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -echo "Setting environment variables for CCPP-SCM on Cheyenne with icc/ifort" - -MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) - -export SCM_ROOT=$MYDIR/../.. - -#start with a "clean" environment; activate and deactivate ncar_pylib in order to successfully deactivate previously activated environment without errors -module load ncarenv/1.3 -conda deactivate -module purge - -#load the modules in order to compile the CCPP SCM -echo "Loading intel and netcdf modules..." -module load ncarenv/1.3 -module load intel/2022.1 -module load mpt/2.25 -module load ncarcompilers/0.5.0 -module load netcdf - -echo "Setting up NCEPLIBS" -module use /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack -module load hpc/1.2.0 -module load hpc-intel/2022.1 -module load hpc-mpt/2.25 -export bacio_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/bacio/2.4.1 -export sp_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/sp/2.3.3 -export w3emc_ROOT=/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/intel-2022.1/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -export CC=icc -export CXX=icpc -export FC=ifort - -echo "Loading cmake" -module load cmake/3.22.0 -export CMAKE_C_COMPILER=icc -export CMAKE_CXX_COMPILER=icpc -export CMAKE_Fortran_COMPILER=ifort -export CMAKE_Platform=cheyenne.intel - -echo "Setting up python environment for running and plotting." -module load conda/latest - -conda activate /glade/p/ral/jntp/GMTB/CCPP_SCM/conda/ccpp-scm - diff --git a/scm/etc/Desktop_setup_gfortran.csh b/scm/etc/Desktop_setup_gfortran.csh deleted file mode 100755 index d038405f4..000000000 --- a/scm/etc/Desktop_setup_gfortran.csh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/tcsh - -echo "Setting environment variables for CCPP-SCM on Desktop (MacOS) with gcc/gfortran" - -set called=($_) - -if ( "$called" != "") then ### called by source - set MYSCRIPT=`readlink -f -n $called[2]` -else ### called by direct execution of the script - set MYSCRIPT=`readlink -f -n '$0'` -endif -set MYDIR=`dirname $MYSCRIPT` -set MYDIR=`cd $MYDIR && pwd -P` - -setenv SCM_ROOT $MYDIR/../.. - -echo "Setting CC/CXX/FC environment variables" -setenv CC /opt/local/bin/gcc-mp-10 -setenv CXX /opt/local/bin/g++-mp-10 -setenv FC gfortran-mp-10 - -echo "Setting location of NETCDF" -setenv NETCDF /opt/local -setenv LDFLAGS "-I${NETCDF}/include -L${NETCDF}/lib -Wl,-rpath,${NETCDF}/lib" - -echo "Setting location of NCEPLIBS libraries" -setenv BACIO_LIB4 /Users/$USER/NCEPLIBS/lib/libbacio_v2.2.0_4.a -setenv SP_LIBd /Users/$USER/NCEPLIBS/lib/libsp_v2.1.0_d.a -setenv W3NCO_LIBd /Users/$USER/NCEPLIBS/lib/libw3nco_v2.1.0_d.a - -#check to see if CMake is installed locally -echo "Checking if CMake is installed" -cmake --version - -if ( $? != 0 ) then - echo "CMake not found; installing CMake" - pip install cmake -else - echo "CMake is installed" -endif - -#check to see if f90nml is installed locally -echo "Checking if f90nml python module is installed" -python -c "import f90nml" - -if ( $? != 0 ) then - echo "Not found; installing f90nml" - pip install --no-cache-dir f90nml==0.19 -else - echo "f90nml is installed" -endif diff --git a/scm/etc/Desktop_setup_gfortran.sh b/scm/etc/Desktop_setup_gfortran.sh deleted file mode 100755 index 37485a163..000000000 --- a/scm/etc/Desktop_setup_gfortran.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -echo "Setting environment variables for CCPP-SCM on Desktop (MacOS) with gcc/gfortran" - -if [[ $(uname -s) == Darwin ]]; then - MYDIR=$(cd "$(dirname "$(greadlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) -else - MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) -fi - -export SCM_ROOT=$MYDIR/../.. - -echo "Setting CC/CXX/FC environment variables" -export CC=/opt/local/bin/gcc-mp-10 -export CXX=/opt/local/bin/g++-mp-10 -export FC=gfortran-mp-10 - -echo "Setting location of NETCDF" -export NETCDF=/opt/local -export LDFLAGS="-I${NETCDF}/include -L${NETCDF}/lib -Wl,-rpath,${NETCDF}/lib" - -echo "Setting location of NCEPLIBS libraries" -export BACIO_LIB4=/Users/$USER/NCEPLIBS/lib/libbacio_v2.2.0_4.a -export SP_LIBd=/Users/$USER/NCEPLIBS/lib/libsp_v2.1.0_d.a -export W3NCO_LIBd=/Users/$USER/NCEPLIBS/lib/libw3nco_v2.1.0_d.a - -#check to see if CMake is installed locally -echo "Checking if CMake is installed" -cmake --version - -if [ $? -ne 0 ]; then - echo "CMake not found; installing CMake" - pip install cmake -else - echo "CMake is installed" -fi - -#check to see if f90nml is installed locally -echo "Checking if f90nml python module is installed" -python -c "import f90nml" - -if [ $? -ne 0 ]; then - echo "Not found; installing f90nml" - pip install --no-cache-dir f90nml==0.19 -else - echo "f90nml is installed" -fi diff --git a/scm/etc/Hera_setup_intel.csh b/scm/etc/Hera_setup_intel.csh deleted file mode 100755 index 369f6e3af..000000000 --- a/scm/etc/Hera_setup_intel.csh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/tcsh - -echo "Setting environment variables for CCPP-SCM on Hera with icc/ifort" - -set called=($_) - -if ( "$called" != "") then ### called by source - set MYSCRIPT=`readlink -f -n $called[2]` -else ### called by direct execution of the script - set MYSCRIPT=`readlink -f -n '$0'` -endif -set MYDIR=`dirname $MYSCRIPT` -set MYDIR=`cd $MYDIR && pwd -P` - -setenv SCM_ROOT $MYDIR/../.. - -#load the modules in order to compile the CCPP SCM -echo "Loading intel and netcdf modules..." -module purge -module load intel/2022.1.2 -module load impi/2022.1.2 -module use /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack -module load hpc/1.2.0 -module load hpc-intel/2022.1.2 -module load hpc-impi/2022.1.2 -module load netcdf - -echo "Setting up NCEPLIBS" -setenv bacio_ROOT /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/bacio/2.4.1 -setenv sp_ROOT /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/sp/2.3.3 -setenv w3emc_ROOT /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -setenv CC icc -setenv CXX icpc -setenv FC ifort - -echo "Loading cmake" -module load cmake/3.20.1 -setenv CMAKE_C_COMPILER icc -setenv CMAKE_CXX_COMPILER icpc -setenv CMAKE_Fortran_COMPILER ifort -setenv CMAKE_Platform hera.intel - -echo "Loading the SCM python environment" -source /scratch1/BMC/gmtb/SCM_anaconda/etc/profile.d/conda.csh -conda activate pyccpp diff --git a/scm/etc/Hera_setup_intel.sh b/scm/etc/Hera_setup_intel.sh deleted file mode 100755 index 5d760260b..000000000 --- a/scm/etc/Hera_setup_intel.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -echo "Setting environment variables for CCPP-SCM on Hera with icc/ifort" - -MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) - -export SCM_ROOT=$MYDIR/../.. - -#load the modules in order to compile the CCPP SCM -echo "Loading intel and netcdf modules..." -module purge -module load intel/2022.1.2 -module load impi/2022.1.2 -module use /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack -module load hpc/1.2.0 -module load hpc-intel/2022.1.2 -module load hpc-impi/2022.1.2 -module load netcdf - -echo "Setting up NCEPLIBS" -export bacio_ROOT=/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/bacio/2.4.1 -export sp_ROOT=/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/sp/2.3.3 -export w3emc_ROOT=/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/w3emc/2.9.2 - -echo "Setting CC/CXX/FC environment variables" -export CC=icc -export CXX=icpc -export FC=ifort - -echo "Loading cmake" -module load cmake/3.20.1 -export CMAKE_C_COMPILER=icc -export CMAKE_CXX_COMPILER=icpc -export CMAKE_Fortran_COMPILER=ifort -export CMAKE_Platform=hera.intel - -echo "Loading the SCM python environment" -. "/scratch1/BMC/gmtb/SCM_anaconda/etc/profile.d/conda.sh" -conda activate pyccpp diff --git a/scm/etc/modules/hera_gnu.lua b/scm/etc/modules/hera_gnu.lua index 23867c4ac..0e28df2d0 100644 --- a/scm/etc/modules/hera_gnu.lua +++ b/scm/etc/modules/hera_gnu.lua @@ -7,10 +7,10 @@ whatis([===[Loads libraries needed for building the CCPP SCM on Hera with GNU co prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -load("cmake/3.20.1") +load("cmake/3.28.1") load("miniconda/3.9.12") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core") load("stack-gcc/9.2.0") load("stack-openmpi/4.1.5") diff --git a/scm/etc/modules/hera_intel.lua b/scm/etc/modules/hera_intel.lua index d84db665c..6cdbf7d4c 100644 --- a/scm/etc/modules/hera_intel.lua +++ b/scm/etc/modules/hera_intel.lua @@ -6,10 +6,10 @@ the NOAA RDHPC machine Hera using Intel-2021.5.0 whatis([===[Loads libraries needed for building the CCPP SCM on Hera with Intel compilers ]===]) prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -load("cmake/3.20.1") +load("cmake/3.28.1") load("miniconda/3.9.12") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core") load("stack-intel/2021.5.0") load("stack-intel-oneapi-mpi/2021.5.1") diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index cb74197ee..7c8d7d1a9 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1355,7 +1355,7 @@ def get_UFS_oro_data(dir, tile, i, j, lam): if lam: nc_file = Dataset('{0}/{1}'.format(dir,'oro_data.nc')) else: - filename_pattern = 'oro_data.tile{0}.nc'.format(tile) + filename_pattern = 'oro*.tile{0}.nc'.format(tile) for f_name in os.listdir(dir): if fnmatch.fnmatch(f_name, filename_pattern): filename = f_name @@ -1504,8 +1504,8 @@ def get_UFS_forcing_data(nlevs, state_IC, location, use_nearest, forcing_dir, gr atm_ftag = 'atmf*.tile{0}.nc'.format(tile) sfc_ftag = 'sfcf*.tile{0}.nc'.format(tile) else: - atm_ftag = 'atmf*.nc' - sfc_ftag = 'sfcf*.nc' + atm_ftag = '*atmf*.nc' + sfc_ftag = '*sfcf*.nc' # end if # Get list of UFS history files with 3D ATMospheric state variables. @@ -2811,7 +2811,7 @@ def write_comparison_file(comp_data, case_name, date, surface): ######################################################################################## def find_date(forcing_dir): - atm_ftag = 'atmf*.nc' + atm_ftag = '*atmf*.nc' atm_filenames = [] for f_name in os.listdir(forcing_dir): diff --git a/scm/etc/scripts/extract_FV3GFS_column_ic.py b/scm/etc/scripts/extract_FV3GFS_column_ic.py deleted file mode 100755 index 6684125c1..000000000 --- a/scm/etc/scripts/extract_FV3GFS_column_ic.py +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env python -from netCDF4 import Dataset -import numpy as np - -# define date -YYYY=2014 -MM=8 -DD=1 -HH=0 -MI=0 -SC=0 - -# define path to FV3 GFS initial and boundary conditions -icpath='../../data/raw_case_input/UFS_test_ics' -fixpath='../../data/raw_case_input/UFS_test_ics' -print (icpath) -# defint i,j and tile to extract colmn -ipt=16 -jpt=41 -tilenum=2 - -ipt2=ipt*2+1 -jpt2=jpt*2+1 - -# I have surface cycle on the sfc_data initial conditions to get the proper surface fields -infile1='%s/gfs_data.tile%i.nc' % (icpath,tilenum) -infile2='%s/sfc_data.tile%i.nc' %(icpath,tilenum) -infile3='%s/C96_oro_data.tile%i.nc' %(fixpath,tilenum) -infile4='%s/C96_grid.tile%i.nc' %(fixpath,tilenum) -infile5='%s/gfs_ctrl.nc' %icpath -ncin1=Dataset(infile1) -ncin2=Dataset(infile2) -ncin3=Dataset(infile3) -ncin4=Dataset(infile4) -ncin5=Dataset(infile5) -# assume model contains one less level than the cold start spectral GFS initial conditions -nlevs=len(ncin1.dimensions['lev'])-1 -# pick off lat and lon from i and j point defined above -lon0=ncin4['x'][jpt2,ipt2] -lat0=ncin4['y'][jpt2,ipt2] - -# extract out area of grid cell -area_in=ncin4['area'][jpt2-1:jpt2+1,ipt2-1:ipt2+1] -print (lat0,lon0) - -# upper air fields from initial conditions -zh=ncin1['zh'][::-1,jpt,ipt] -uw1=ncin1['u_w'][::-1,jpt,ipt] -uw2=ncin1['u_w'][::-1,jpt,ipt+1] -us1=ncin1['u_s'][::-1,jpt,ipt] -us2=ncin1['u_s'][::-1,jpt+1,ipt] -vw1=ncin1['v_w'][::-1,jpt,ipt] -vw2=ncin1['v_w'][::-1,jpt,ipt+1] -vs1=ncin1['v_s'][::-1,jpt,ipt] -vs2=ncin1['v_s'][::-1,jpt+1,ipt] -ucomp=0.25*(uw1+uw2+us1+us2) # estimate u winds on the a grid -vcomp=0.25*(vw1+vw2+vs1+vs2) # estimate v winds on the a grid -sphum=ncin1['sphum'][::-1,jpt,ipt] -# o3 and qv are taken from ics. -o3=ncin1['o3mr'][::-1,jpt,ipt] -liqwat=ncin1['liq_wat'][:-1,jpt,ipt] - -# surface pressure and skin temperature -ps=ncin1['ps'][jpt,ipt] -ts=ncin2['tsea'][jpt,ipt] - -# land state -stc_in=ncin2['stc'][:,jpt,ipt] -smc_in=ncin2['smc'][:,jpt,ipt] -slc_in=ncin2['slc'][:,jpt,ipt] -tg3_in=ncin2['tg3'][jpt,ipt] - -# surface properties -uustar_in=ncin2['uustar'][jpt,ipt] -alvsf=ncin2['alvsf'][jpt,ipt] -alvwf=ncin2['alvwf'][jpt,ipt] -alnsf=ncin2['alnsf'][jpt,ipt] -alnwf=ncin2['alnwf'][jpt,ipt] -facsf_in=ncin2['facsf'][jpt,ipt] -facwf_in=ncin2['facwf'][jpt,ipt] -styp_in=ncin2['stype'][jpt,ipt] -slope_in=ncin2['slope'][jpt,ipt] -vtyp_in=ncin2['vtype'][jpt,ipt] -vfrac_in=ncin2['vfrac'][jpt,ipt] -shdmin_in=ncin2['shdmin'][jpt,ipt] -shdmax_in=ncin2['shdmax'][jpt,ipt] -zorl_in=ncin2['zorl'][jpt,ipt] -slmsk_in=ncin2['slmsk'][jpt,ipt] -canopy_in=ncin2['canopy'][jpt,ipt] -hice_in=ncin2['hice'][jpt,ipt] -fice_in=ncin2['fice'][jpt,ipt] -tisfc_in=ncin2['tisfc'][jpt,ipt] -snwdph_in=ncin2['snwdph'][jpt,ipt] -snoalb_in=ncin2['snoalb'][jpt,ipt] - -# orographyic properties -stddev_in=ncin3['stddev'][jpt,ipt] -convexity_in=ncin3['convexity'][jpt,ipt] -oa1_in=ncin3['oa1'][jpt,ipt] -oa2_in=ncin3['oa2'][jpt,ipt] -oa3_in=ncin3['oa3'][jpt,ipt] -oa4_in=ncin3['oa4'][jpt,ipt] -ol1_in=ncin3['ol1'][jpt,ipt] -ol2_in=ncin3['ol2'][jpt,ipt] -ol3_in=ncin3['ol3'][jpt,ipt] -ol4_in=ncin3['ol4'][jpt,ipt] -theta_in=ncin3['theta'][jpt,ipt] -gamma_in=ncin3['gamma'][jpt,ipt] -sigma_in=ncin3['sigma'][jpt,ipt] -elvmax_in=ncin3['elvmax'][jpt,ipt] - -# vertical coordinate definition -ak=ncin5['vcoord'][0,::-1] -bk=ncin5['vcoord'][1,::-1] - -#calculate temperature -rdgas = 287.05 -rvgas = 461.50 -zvir = rvgas/rdgas - 1. -grav=9.80665 -gz=zh*grav -pn1=np.zeros([nlevs+1]) -temp=np.zeros([nlevs]) -for k in range(nlevs+1): - pn1[k]=np.log(ak[k]+ps*bk[k]) -for k in range(nlevs): - temp[k] = (gz[k]-gz[k+1])/( rdgas*(pn1[k+1]-pn1[k])*(1.+zvir*sphum[k]) ) - -# open output file -nc = Dataset('../../data/processed_case_input/fv3_model_point.nc', mode='w') -nc.description = "FV3GFS model profile input (no forcing)" - -time = nc.createDimension('time',None) -levels = nc.createDimension('levels',None) -nsoil = nc.createDimension('nsoil',None) -t = nc.createVariable('time',np.float64,('time',)) -t.units = "s" -t.description = "elapsed time since the beginning of the simulation" -z = nc.createVariable('levels',np.float64,('levels',)) -z.units = "Pa" -z.description = "pressure levels" -#scalars -iyr = nc.createVariable('scalars/init_year',np.int32) -imo = nc.createVariable('scalars/init_month',np.int32) -idy = nc.createVariable('scalars/init_day',np.int32) -ihr = nc.createVariable('scalars/init_hour',np.int32) -imi = nc.createVariable('scalars/init_minute',np.int32) -isc = nc.createVariable('scalars/init_second',np.int32) -ivegsrc = nc.createVariable('scalars/vegsrc',np.int32) -ivegtyp = nc.createVariable('scalars/vegtyp',np.int32) -isoiltyp = nc.createVariable('scalars/soiltyp',np.int32) -islopetyp = nc.createVariable('scalars/slopetyp',np.int32) -ivegfrac = nc.createVariable('scalars/vegfrac',np.float) -ishdmin = nc.createVariable('scalars/shdmin',np.float) -ishdmax = nc.createVariable('scalars/shdmax',np.float) -izorl = nc.createVariable('scalars/zorl',np.float) -islmsk = nc.createVariable('scalars/slmsk',np.float) -icanopy = nc.createVariable('scalars/canopy',np.float) -ihice = nc.createVariable('scalars/hice',np.float) -ifice = nc.createVariable('scalars/fice',np.float) -itisfc = nc.createVariable('scalars/tisfc',np.float) -isnwdph = nc.createVariable('scalars/snwdph',np.float) -isnoalb = nc.createVariable('scalars/snoalb',np.float) -isncovr = nc.createVariable('scalars/sncovr',np.float) -itg3 = nc.createVariable('scalars/tg3',np.float) -iuustar = nc.createVariable('scalars/uustar',np.float) - -iyr.units = "years" -iyr.description = "year at time of initial values" -imo.units = "months" -imo.description = "month at time of initial values" -idy.units = "days" -idy.description = "day at time of initial values" -ihr.units = "hours" -ihr.description = "hour at time of initial values" -imi.units = "minutes" -imi.description = "minute at time of initial values" -isc.units = "seconds" -isc.description = "second at time of initial values" -ivegsrc.description = "vegetation soure (1-2)" -ivegtyp.description = "vegetation type (1-12)" -isoiltyp.description = "soil type (1-12)" -islopetyp.description = "slope type (1-9)" -ivegfrac.description = "vegetation fraction" -ishdmin.description = "minimum vegetation fraction" -ishdmax.description = "maximum vegetation fraction" -izorl.description = "surface roughness length" -islmsk.description = "land-sea-ice mask" -icanopy.description = "canopy moisture" -ihice.description = "ice thickness" -ifice.description = "ice fraction" -itisfc.description = "ice temperature" -isnwdph.description = "snow depth" -isnoalb.description = "snow albedo" -isncovr.description = "snow cover" -itg3.description = "deep soil temperature" -itg3.units = "K" -iuustar.description = "frication velocity" -iuustar.units = "m2s-2?" - - -#initial -ic_t = nc.createVariable('initial/temp',np.float64,('levels',)) -ic_qt = nc.createVariable('initial/qt',np.float64,('levels',)) -ic_ql = nc.createVariable('initial/ql',np.float64,('levels',)) -ic_qi = nc.createVariable('initial/qi',np.float64,('levels',)) -ic_u = nc.createVariable('initial/u',np.float64,('levels',)) -ic_v = nc.createVariable('initial/v',np.float64,('levels',)) -ic_tke = nc.createVariable('initial/tke',np.float64,('levels',)) -ic_o3 = nc.createVariable('initial/ozone',np.float64,('levels',)) -ic_stc = nc.createVariable('initial/stc',np.float64,('nsoil',)) -ic_smc = nc.createVariable('initial/smc',np.float64,('nsoil',)) -ic_slc = nc.createVariable('initial/slc',np.float64,('nsoil',)) -ic_t.units = "K" -#ic_t.description = "initial profile of ice-liquid water potential temperature" -ic_t.description = "initial profile of temperature" -ic_qt.units = "kg kg^-1" -ic_qt.description = "initial profile of total water specific humidity" -ic_ql.units = "kg kg^-1" -ic_ql.description = "initial profile of liquid water specific humidity" -ic_qi.units = "kg kg^-1" -ic_qi.description = "initial profile of ice water specific humidity" -ic_u.units = "m s^-1" -ic_u.description = "initial profile of E-W horizontal wind" -ic_v.units = "m s^-1" -ic_v.description = "initial profile of N-S horizontal wind" -ic_tke.units = "m^2 s^-2" -ic_tke.description = "initial profile of turbulence kinetic energy" -ic_o3.units = "kg kg^-1" -ic_o3.description = "initial profile of ozone mass mixing ratio" -ic_stc.units = "K" -ic_stc.description = "initial profile of soil temperature" -ic_smc.units = "kg" -ic_smc.description = "initial profile of soil moisture" -ic_slc.units = "kg" -ic_slc.description = "initial profile of soil liquid moisture" - -lat=nc.createVariable('forcing/lat',np.float64,('time',)) -lat.units = "degrees N" -lat.description = "latitude of column" -lon=nc.createVariable('forcing/lon',np.float64,('time',)) -lon.units = "degrees E" -lon.description = "longitude of column" -p_surf=nc.createVariable('forcing/p_surf',np.float64,('time',)) -p_surf.units = "Pa" -p_surf.description = "surface pressure" -T_surf=nc.createVariable('forcing/T_surf',np.float64,('time',)) -T_surf.units = "K" -T_surf.description = "surface absolute temperature" -area1=nc.createVariable('scalars/area',np.float64,('time',)) -alb1=nc.createVariable('scalars/alvsf',np.float64,('time',)) -alb2=nc.createVariable('scalars/alnsf',np.float64,('time',)) -alb3=nc.createVariable('scalars/alvwf',np.float64,('time',)) -alb4=nc.createVariable('scalars/alnwf',np.float64,('time',)) -stddev=nc.createVariable('scalars/stddev',np.float64,('time',)) -convexity=nc.createVariable('scalars/convexity',np.float64,('time',)) -oa1=nc.createVariable('scalars/oa1',np.float64,('time',)) -oa2=nc.createVariable('scalars/oa2',np.float64,('time',)) -oa3=nc.createVariable('scalars/oa3',np.float64,('time',)) -oa4=nc.createVariable('scalars/oa4',np.float64,('time',)) -ol1=nc.createVariable('scalars/ol1',np.float64,('time',)) -ol2=nc.createVariable('scalars/ol2',np.float64,('time',)) -ol3=nc.createVariable('scalars/ol3',np.float64,('time',)) -ol4=nc.createVariable('scalars/ol4',np.float64,('time',)) -theta=nc.createVariable('scalars/theta',np.float64,('time',)) -gamma=nc.createVariable('scalars/gamma',np.float64,('time',)) -sigma=nc.createVariable('scalars/sigma',np.float64,('time',)) -elvmax=nc.createVariable('scalars/elvmax',np.float64,('time',)) -facsf=nc.createVariable('scalars/facsf',np.float64,('time',)) -facwf=nc.createVariable('scalars/facwf',np.float64,('time',)) -area1.units = "m^2" -alb1.units = "None" -alb2.units = "None" -alb3.units = "None" -alb4.units = "None" -facsf.units = "None" -facwf.units = "None" -area1.description = "grid cell area" -alb1.description = "uv+visible black sky albedo (z=60 degree)" -alb2.description = "near IR black sky albedo (z=60 degree)" -alb3.description = "uv+visible white sky albedo" -alb4.description = "near IR white sky albedo" -stddev.description = "surface orography standard deviation" -facsf.description = "fraction of grid cell with strong sun angle albedo dependence" -facwf.description = "fraction of grid cell with weak sun angle albedo dependence" -w_ls=nc.createVariable('forcing/w_ls',np.float64,('levels','time',)) -w_ls.units = "m s^-1" -w_ls.description = "large scale vertical velocity" -omega=nc.createVariable('forcing/omega',np.float64,('levels','time',)) -omega.units = "Pa s^-1" -omega.description = "large scale pressure vertical velocity" -u_g=nc.createVariable('forcing/u_g',np.float64,('levels','time',)) -u_g.units = "m s^-1" -u_g.description = "large scale geostrophic E-W wind" -v_g=nc.createVariable('forcing/v_g',np.float64,('levels','time',)) -v_g.units = "m s^-1" -v_g.description = "large scale geostrophic N-S wind" -u_nudge=nc.createVariable('forcing/u_nudge',np.float64,('levels','time',)) -u_nudge.units = "m s^-1" -u_nudge.description = "E-W wind to nudge toward" -v_nudge=nc.createVariable('forcing/v_nudge',np.float64,('levels','time',)) -v_nudge.units = "m s^-1" -v_nudge.description = "N-S wind to nudge toward" -T_nudge=nc.createVariable('forcing/T_nudge',np.float64,('levels','time',)) -T_nudge.units = "K" -T_nudge.description = "absolute temperature to nudge toward" -thil_nudge=nc.createVariable('forcing/thil_nudge',np.float64,('levels','time',)) -thil_nudge.units = "K" -thil_nudge.description = "potential temperature to nudge toward" -qt_nudge=nc.createVariable('forcing/qt_nudge',np.float64,('levels','time',)) -qt_nudge.units = "kg kg^-1" -qt_nudge.description = "q_t to nudge toward" -dT_dt_rad=nc.createVariable('forcing/dT_dt_rad',np.float64,('levels','time',)) -dT_dt_rad.units = "K s^-1" -dT_dt_rad.description = "prescribed radiative heating rate" -h_advec_thetail=nc.createVariable('forcing/h_advec_thetail',np.float64,('levels','time',)) -h_advec_thetail.units = "K s^-1" -h_advec_thetail.description = "prescribed theta_il tendency due to horizontal advection" -v_advec_thetail=nc.createVariable('forcing/v_advec_thetail',np.float64,('levels','time',)) -v_advec_thetail.units = "K s^-1" -v_advec_thetail.description = "prescribed theta_il tendency due to vertical advection" -h_advec_qt=nc.createVariable('forcing/h_advec_qt',np.float64,('levels','time',)) -h_advec_qt.units = "kg kg^-1 s^-1" -h_advec_qt.description = "prescribed q_t tendency due to horizontal advection" -v_advec_qt=nc.createVariable('forcing/v_advec_qt',np.float64,('levels','time',)) -v_advec_qt.units = "kg kg^-1 s^-1" -v_advec_qt.description = "prescribed q_t tendency due to vertical advection" - - -# date -iyr[:]=YYYY -imo[:]=MM -idy[:]=DD -ihr[:]=HH -imi[:]=MI -isc[:]=SC -# axes -t[0]=0 - -#ics -ic_t[:] = temp[0:nlevs] -ic_qt[:] = sphum[0:nlevs] -ic_ql[:] = liqwat[0:nlevs] -ic_qi[:] = 0.0 -ic_u[:] = ucomp[0:nlevs] -ic_v[:] = vcomp[0:nlevs] -ic_tke[:] = 0.0 -ic_o3[:] = o3[0:nlevs] -ic_stc[:] = stc_in -ic_smc[:] = smc_in -ic_slc[:] = slc_in - -lat[:]=lat0 -lon[:]=lon0 - -p_surf[:]=ps -T_surf[:]=ts -area1[:]=area_in.sum() -alb1[:]=alvsf -alb2[:]=alnsf -alb3[:]=alvwf -alb4[:]=alnwf -stddev[:]=stddev_in -convexity[:]=convexity_in -oa1[:]=oa1_in -oa2[:]=oa2_in -oa3[:]=oa3_in -oa4[:]=oa4_in -ol1[:]=ol1_in -ol2[:]=ol2_in -ol3[:]=ol3_in -ol4[:]=ol4_in -theta[:]=theta_in -gamma[:]=gamma_in -sigma[:]=sigma_in -elvmax[:]=elvmax_in -facsf[:]=facsf_in -facwf[:]=facwf_in -z[:]=np.exp(pn1[0:nlevs]) -w_ls[:]=0.0 -omega[:]=0.0 -u_g[:]=0.0 -v_g[:]=0.0 -u_nudge[:]=0.0 -v_nudge[:]=0.0 -T_nudge[:]=0.0 -thil_nudge[:]=0.0 -qt_nudge[:]=0.0 -dT_dt_rad[:]=0.0 -h_advec_thetail[:]=0.0 -v_advec_thetail[:]=0.0 -h_advec_qt[:]=0.0 -v_advec_qt[:]=0.0 - -ivegsrc[:] = 1 -ivegtyp[:] = vtyp_in -isoiltyp[:] = styp_in -islopetyp[:] = slope_in -ishdmin[:] = shdmin_in -ishdmax[:] = shdmax_in -izorl[:] = zorl_in -islmsk[:] = slmsk_in -icanopy[:] = canopy_in -ihice[:] = hice_in -ifice[:] = fice_in -itisfc[:] = tisfc_in -isnwdph[:] = snwdph_in -isnoalb[:] = snoalb_in -isncovr[:] = 0.0 -itg3[:] = tg3_in -iuustar[:] = uustar_in -ivegfrac[:]=vfrac_in - -nc.close() - diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index b952680b6..2ef20bb84 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -186,6 +186,37 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + +elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") + if(ENABLE_NVIDIA_OPENACC MATCHES "true") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc -Minfo=accel") + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + endif() + + if(NOT 32BIT) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8") + endif() + + if (${CMAKE_BUILD_TYPE} MATCHES "Debug") + set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -O0 -g") + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g") + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") + endif() + + set(MPI_C_COMPILER mpicc) + set(MPI_CXX_COMPILER mpicxx) + set(MPI_Fortran_COMPILER mpif90) + + set(CMAKE_C_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + else (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") message (FATAL_ERROR "This program has only been compiled with gfortran and ifort. If another compiler is needed, the appropriate flags must be added in ${CMAKE_SOURCE_DIR}/CMakeLists.txt") endif (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 20cfbce6a..926fb68ba 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -488,12 +488,12 @@ def setup_rundir(self): try: input_type = case_nml['case_config']['input_type'] if input_type == 1: + surface_flux_spec = False #open the case data file and read the surfaceForcing global attribute - case_data_dir = case_nml['case_config']['case_data_dir'] - nc_fid = Dataset(os.path.join(SCM_ROOT, case_data_dir) + '/' + self._case + '_SCM_driver.nc' , 'r') - surfaceForcing = nc_fid.getncattr('surfaceForcing') + nc_fid = Dataset(os.path.join(SCM_ROOT, self._case_data_dir) + '/' + self._case + '_SCM_driver.nc' , 'r') + surfaceForcing = nc_fid.getncattr('surface_forcing_temp') nc_fid.close() - if (surfaceForcing.lower() == 'flux' or surfaceForcing.lower() == 'surfaceflux'): + if (surfaceForcing.lower() == 'flux' or surfaceForcing.lower() == 'surface_flux'): surface_flux_spec = True except KeyError: # if not using DEPHY format, check to see if surface fluxes are specified in the case configuration file (default is False) diff --git a/scm/src/scm_input.F90 b/scm/src/scm_input.F90 index 3269c16ac..cc3effcd5 100644 --- a/scm/src/scm_input.F90 +++ b/scm/src/scm_input.F90 @@ -950,7 +950,7 @@ subroutine get_case_init_DEPHY(scm_state, scm_input) real(kind=sp) :: z_nudging_temp, z_nudging_theta, z_nudging_thetal, z_nudging_qv, z_nudging_qt, z_nudging_rv, z_nudging_rt, z_nudging_u, z_nudging_v real(kind=sp) :: p_nudging_temp, p_nudging_theta, p_nudging_thetal, p_nudging_qv, p_nudging_qt, p_nudging_rv, p_nudging_rt, p_nudging_u, p_nudging_v character(len=5) :: input_surfaceType - character(len=11) :: input_surfaceForcingWind='',input_surfaceForcingMoist='',input_surfaceForcingLSM='',input_surfaceForcingTemp='' + character(len=12) :: input_surfaceForcingWind='',input_surfaceForcingMoist='',input_surfaceForcingLSM='',input_surfaceForcingTemp='' ! initial variables (IC = Initial Condition) real(kind=dp), allocatable :: input_lat(:) !< column latitude (deg) @@ -1630,7 +1630,7 @@ subroutine get_case_init_DEPHY(scm_state, scm_input) call NetCDF_read_var(ncid, "wprvp_s", .False., input_force_wprvp) call NetCDF_read_var(ncid, "wprtp_s", .False., input_force_wprtp) else if (trim(input_surfaceForcingMoist) == 'surface_flux') then - call NetCDF_read_var(ncid, "hfls", .False., input_force_sfc_sens_flx) + call NetCDF_read_var(ncid, "hfls", .False., input_force_sfc_lat_flx) endif ! @@ -1969,7 +1969,6 @@ subroutine get_case_init_DEPHY(scm_state, scm_input) if (trim(input_surfaceForcingLSM) == "lsm") then scm_input%input_ozone = input_ozone(:,active_init_time) scm_input%input_area = input_area(active_init_time) - scm_state%area = input_area(active_init_time) scm_input%input_stddev = input_stddev(active_init_time) scm_input%input_convexity= input_convexity(active_init_time) diff --git a/scm/src/suite_info.py b/scm/src/suite_info.py index 63d35c97c..f3420cdf5 100755 --- a/scm/src/suite_info.py +++ b/scm/src/suite_info.py @@ -44,6 +44,7 @@ def timestep(self, value): suite_list = [] suite_list.append(suite('SCM_GFS_v16', 'tracers_GFS_v16.txt', 'input_GFS_v16.nml', 600.0, 1800.0, True )) suite_list.append(suite('SCM_GFS_v17_p8', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_p8.nml', 600.0, 600.0, True )) +suite_list.append(suite('SCM_GFS_v17_HR3', 'tracers_GFS_v17_p8.txt', 'input_GFS_v17_HR3.nml', 600.0, 600.0, True )) suite_list.append(suite('SCM_RAP', 'tracers_RAP.txt', 'input_RAP.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_RRFS_v1beta', 'tracers_RRFS_v1beta.txt', 'input_RRFS_v1beta.nml', 600.0, 600.0 , True )) suite_list.append(suite('SCM_WoFS_v0', 'tracers_WoFS_v0.txt', 'input_WoFS_v0.nml', 600.0, 600.0 , True )) diff --git a/test/rt_test_cases_nvidia.py b/test/rt_test_cases_nvidia.py new file mode 100644 index 000000000..a19c81ba9 --- /dev/null +++ b/test/rt_test_cases_nvidia.py @@ -0,0 +1,9 @@ +run_list = [\ + #---------------------------------------------------------------------------------------------------------------------------------------------- + # CCPP-SCM v6 supported suites + #---------------------------------------------------------------------------------------------------------------------------------------------- + {"case": "arm_sgp_summer_1997_A", "suite": "SCM_RAP"}, \ + {"case": "twpice", "suite": "SCM_RAP"}, \ + {"case": "bomex", "suite": "SCM_RAP"}, \ + {"case": "astex", "suite": "SCM_RAP"}, \ + {"case": "LASSO_2016051812", "suite": "SCM_RAP"}]