From 33d7b748be08b5605a2af97d591f9d92411a5f7c Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Wed, 24 Jan 2024 17:47:53 -0500 Subject: [PATCH 001/172] update ccpp/physics and .gitmodules after testing --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5edd6912e..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR98 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 8cf9374da..378517c81 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 8cf9374da99d7932752173b40c6932e1f52caaac +Subproject commit 378517c81dac9cbe4f0bee41284b46ae070b4323 From baff0ec5926c57fc22813aedc15242dcfea56c87 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Tue, 30 Jan 2024 15:15:04 -0500 Subject: [PATCH 002/172] update ccpp/physics and .gitmodules --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index cdb76751f..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR102 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 8a64c5965..fb0000c3f 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 8a64c5965ecb99d95558f71c2435d5bbbb77b88b +Subproject commit fb0000c3f255c6f271817f0f6d88e43a29e179a3 From 63c9450f2c4c87d49d4f380c780e06d24295dcaa Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Fri, 9 Feb 2024 17:50:44 -0500 Subject: [PATCH 003/172] update for ufs/dev PR#94 --- .gitmodules | 2 +- ccpp/config/ccpp_prebuild_config.py | 1 + ccpp/physics | 2 +- scm/src/GFS_typedefs.F90 | 111 ++++++++++++++---- scm/src/GFS_typedefs.meta | 170 +++++++++++++++++++++++++++- 5 files changed, 256 insertions(+), 30 deletions(-) diff --git a/.gitmodules b/.gitmodules index db0fb33f3..a8006a363 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ #url = https://github.com/NCAR/ccpp-physics #branch = main url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR153 + branch = ufs-dev-PR94 [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/config/ccpp_prebuild_config.py b/ccpp/config/ccpp_prebuild_config.py index d86710b97..41736151a 100755 --- a/ccpp/config/ccpp_prebuild_config.py +++ b/ccpp/config/ccpp_prebuild_config.py @@ -204,6 +204,7 @@ 'ccpp/physics/physics/SFC_Layer/UFS/sfc_nst_post.f90', 'ccpp/physics/physics/SFC_Models/Land/RUC/lsm_ruc.F90', 'ccpp/physics/physics/SFC_Models/SeaIce/CICE/sfc_cice.f', + 'ccpp/physics/physics/SFC_Models/Land/sfc_land.F90', 'ccpp/physics/physics/SFC_Models/Land/Noah/lsm_noah.f', 'ccpp/physics/physics/SFC_Models/Land/Noahmp/noahmpdrv.F90', 'ccpp/physics/physics/SFC_Models/Lake/Flake/flake_driver.F90', diff --git a/ccpp/physics b/ccpp/physics index d8cb45cbf..57763127c 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit d8cb45cbfd252088743a55b2bc2932ad75ea806d +Subproject commit 57763127c01907361fccc5a212082db04cc30f7c diff --git a/scm/src/GFS_typedefs.F90 b/scm/src/GFS_typedefs.F90 index a20d73be3..3922c31ef 100644 --- a/scm/src/GFS_typedefs.F90 +++ b/scm/src/GFS_typedefs.F90 @@ -451,15 +451,18 @@ module GFS_typedefs real (kind=kind_phys), pointer :: emanoc (:) => null() !< instantaneous anthro. oc emission !--- Smoke. These 2 arrays are input smoke emission and frp - real (kind=kind_phys), pointer :: ebb_smoke_in(:) => null() !< input smoke emission - real (kind=kind_phys), pointer :: frp_input (:) => null() !< input FRP - + real (kind=kind_phys), pointer :: ebb_smoke_in(:) => null() !< input smoke emission + real (kind=kind_phys), pointer :: frp_output (:) => null() !< output FRP !--- For fire diurnal cycle real (kind=kind_phys), pointer :: fhist (:) => null() !< instantaneous fire coef_bb real (kind=kind_phys), pointer :: coef_bb_dc (:) => null() !< instantaneous fire coef_bb + integer, pointer :: fire_type (:) => null() !< fire type + real (kind=kind_phys), pointer :: peak_hr (:) => null() !< peak hour of fire emissions + real (kind=kind_phys), pointer :: lu_nofire (:) => null() ! null() ! null() !< heat flux from wildfire - real (kind=kind_phys), pointer :: frac_grid_burned_out (:) => null() !< fraction of grid cell burning + real (kind=kind_phys), pointer :: fire_heat_flux (:) => null() !< heat flux from wildfire + real (kind=kind_phys), pointer :: frac_grid_burned (:) => null() !< fraction of grid cell burning !--- For smoke and dust auxiliary inputs real (kind=kind_phys), pointer :: fire_in (:,:) => null() !< fire auxiliary inputs @@ -529,6 +532,20 @@ module GFS_typedefs real (kind=kind_phys), pointer :: dtsfcin_med(:) => null() !< sfc latent heat flux over ocean real (kind=kind_phys), pointer :: dqsfcin_med(:) => null() !< sfc sensible heat flux over ocean real (kind=kind_phys), pointer :: ulwsfcin_med(:) => null() !< sfc upward lw flux over ocean + !--- variables needed for cpllnd = .TRUE. and cpllnd2atm=.TRUE. + real (kind=kind_phys), pointer :: sncovr1_lnd(:) => null() !< sfc snow area fraction over land + real (kind=kind_phys), pointer :: qsurf_lnd(:) => null() !< sfc specific humidity + real (kind=kind_phys), pointer :: evap_lnd(:) => null() !< sfc latent heat flux over land, converted to evaporative flux + real (kind=kind_phys), pointer :: hflx_lnd(:) => null() !< sfc sensible heat flux over land + real (kind=kind_phys), pointer :: ep_lnd(:) => null() !< sfc up pot latent heat flux over land + real (kind=kind_phys), pointer :: t2mmp_lnd(:) => null() !< 2 meter temperature over land + real (kind=kind_phys), pointer :: q2mp_lnd(:) => null() !< 2 meter spec humidity over land + real (kind=kind_phys), pointer :: gflux_lnd(:) => null() !< soil heat flux over land + real (kind=kind_phys), pointer :: runoff_lnd(:) => null() !< surface runoff over land + real (kind=kind_phys), pointer :: drain_lnd(:) => null() !< subsurface runoff over land + real (kind=kind_phys), pointer :: cmm_lnd(:) => null() !< surface drag wind speed for momentum + real (kind=kind_phys), pointer :: chh_lnd(:) => null() !< surface drag mass flux for heat and moisture + real (kind=kind_phys), pointer :: zvfun_lnd(:) => null() !< function of surface roughness length and green vegetation fraction !--- outgoing accumulated quantities real (kind=kind_phys), pointer :: rain_cpl (:) => null() !< total rain precipitation @@ -609,10 +626,6 @@ module GFS_typedefs !--- For fire diurnal cycle real (kind=kind_phys), pointer :: ebu_smoke (:,:) => null() !< 3D ebu array - !--- For smoke and dust optical extinction - real (kind=kind_phys), pointer :: smoke_ext (:,:) => null() !< 3D aod array - real (kind=kind_phys), pointer :: dust_ext (:,:) => null() !< 3D aod array - !--- For MYNN PBL transport of smoke and dust real (kind=kind_phys), pointer :: chem3d (:,:,:) => null() !< 3D aod array real (kind=kind_phys), pointer :: ddvel (:,: ) => null() !< 2D dry deposition velocity @@ -626,6 +639,8 @@ module GFS_typedefs !--- Fire plume rise diagnostics real (kind=kind_phys), pointer :: min_fplume (:) => null() !< minimum plume rise level real (kind=kind_phys), pointer :: max_fplume (:) => null() !< maximum plume rise level + real (kind=kind_phys), pointer :: uspdavg (:) => null() !< BL average wind speed + real (kind=kind_phys), pointer :: hpbl_thetav (:) => null() !< BL depth parcel method !--- hourly fire potential index real (kind=kind_phys), pointer :: rrfs_hwp (:) => null() !< hourly fire potential index real (kind=kind_phys), pointer :: rrfs_hwp_ave (:) => null() !< *Average* hourly fire potential index @@ -734,6 +749,7 @@ module GFS_typedefs logical :: cplaqm !< default no cplaqm collection logical :: cplchm !< default no cplchm collection logical :: cpllnd !< default no cpllnd collection + logical :: cpllnd2atm !< default no lnd->atm coupling logical :: rrfs_sd !< default no rrfs_sd collection logical :: use_cice_alb !< default .false. - i.e. don't use albedo imported from the ice model logical :: cpl_imp_mrg !< default no merge import with internal forcings @@ -1522,6 +1538,7 @@ module GFS_typedefs logical :: do_plumerise integer :: addsmoke_flag integer :: plumerisefire_frq + integer :: n_dbg_lines integer :: smoke_forecast logical :: aero_ind_fdb ! WFA/IFA indirect logical :: aero_dir_fdb ! smoke/dust direct @@ -2678,8 +2695,8 @@ subroutine sfcprop_create (Sfcprop, IM, Model) allocate (Sfcprop%acsnow_land (IM)) allocate (Sfcprop%acsnow_ice (IM)) allocate (Sfcprop%xlaixy (IM)) - allocate (Sfcprop%fire_heat_flux_out (IM)) - allocate (Sfcprop%frac_grid_burned_out (IM)) + allocate (Sfcprop%fire_heat_flux (IM)) + allocate (Sfcprop%frac_grid_burned(IM)) ! Sfcprop%wetness = clear_val @@ -2700,8 +2717,8 @@ subroutine sfcprop_create (Sfcprop, IM, Model) Sfcprop%acsnow_land = clear_val Sfcprop%acsnow_ice = clear_val Sfcprop%xlaixy = clear_val - Sfcprop%fire_heat_flux_out = clear_val - Sfcprop%frac_grid_burned_out = clear_val + Sfcprop%fire_heat_flux = clear_val + Sfcprop%frac_grid_burned= clear_val ! end if @@ -2795,9 +2812,13 @@ subroutine sfcprop_create (Sfcprop, IM, Model) allocate (Sfcprop%emseas (IM)) allocate (Sfcprop%emanoc (IM)) allocate (Sfcprop%ebb_smoke_in (IM)) - allocate (Sfcprop%frp_input (IM)) + allocate (Sfcprop%frp_output (IM)) allocate (Sfcprop%fhist (IM)) allocate (Sfcprop%coef_bb_dc(IM)) + allocate (Sfcprop%fire_type (IM)) + allocate (Sfcprop%peak_hr (IM)) + allocate (Sfcprop%lu_nofire (IM)) + allocate (Sfcprop%lu_qfire (IM)) allocate (Sfcprop%fire_in (IM,Model%fire_aux_data_levels)) ! IMPORTANT: This initialization must match rrfs_sd_fill_data @@ -2805,10 +2826,14 @@ subroutine sfcprop_create (Sfcprop, IM, Model) Sfcprop%emseas = clear_val Sfcprop%emanoc = clear_val Sfcprop%ebb_smoke_in = clear_val - Sfcprop%frp_input = clear_val + Sfcprop%frp_output = clear_val Sfcprop%fhist = 1. Sfcprop%coef_bb_dc = clear_val + Sfcprop%fire_type = 0 Sfcprop%fire_in = clear_val + Sfcprop%peak_hr = clear_val + Sfcprop%lu_nofire = clear_val + Sfcprop%lu_qfire = clear_val endif end subroutine sfcprop_create @@ -3031,7 +3056,38 @@ subroutine coupling_create (Coupling, IM, Model) Coupling%slmsk_cpl = clear_val !< pointer to sfcprop%slmsk endif - !-- cellular automata + ! -- Coupling options to retrive land fluxes from external land component + if (Model%cpllnd .and. Model%cpllnd2atm) then + allocate (Coupling%sncovr1_lnd (IM)) + allocate (Coupling%qsurf_lnd (IM)) + allocate (Coupling%evap_lnd (IM)) + allocate (Coupling%hflx_lnd (IM)) + allocate (Coupling%ep_lnd (IM)) + allocate (Coupling%t2mmp_lnd (IM)) + allocate (Coupling%q2mp_lnd (IM)) + allocate (Coupling%gflux_lnd (IM)) + allocate (Coupling%runoff_lnd (IM)) + allocate (Coupling%drain_lnd (IM)) + allocate (Coupling%cmm_lnd (IM)) + allocate (Coupling%chh_lnd (IM)) + allocate (Coupling%zvfun_lnd (IM)) + + Coupling%sncovr1_lnd = clear_val + Coupling%qsurf_lnd = clear_val + Coupling%evap_lnd = clear_val + Coupling%hflx_lnd = clear_val + Coupling%ep_lnd = clear_val + Coupling%t2mmp_lnd = clear_val + Coupling%q2mp_lnd = clear_val + Coupling%gflux_lnd = clear_val + Coupling%runoff_lnd = clear_val + Coupling%drain_lnd = clear_val + Coupling%cmm_lnd = clear_val + Coupling%chh_lnd = clear_val + Coupling%zvfun_lnd = clear_val + end if + + !-- cellular automata allocate (Coupling%condition(IM)) if (Model%do_ca) then allocate (Coupling%ca1 (IM)) @@ -3147,8 +3203,6 @@ subroutine coupling_create (Coupling, IM, Model) if(Model%rrfs_sd) then !--- needed for smoke aerosol option allocate (Coupling%ebu_smoke (IM,Model%levs)) - allocate (Coupling%smoke_ext (IM,Model%levs)) - allocate (Coupling%dust_ext (IM,Model%levs)) allocate (Coupling%chem3d (IM,Model%levs,Model%nchem)) allocate (Coupling%ddvel (IM,Model%ndvel)) allocate (Coupling%wetdpc_flux(IM,Model%nchem)) @@ -3156,11 +3210,11 @@ subroutine coupling_create (Coupling, IM, Model) allocate (Coupling%drydep_flux(IM,Model%ndvel)) allocate (Coupling%min_fplume(IM)) allocate (Coupling%max_fplume(IM)) + allocate (Coupling%uspdavg(IM)) + allocate (Coupling%hpbl_thetav(IM)) allocate (Coupling%rrfs_hwp (IM)) allocate (Coupling%rrfs_hwp_ave (IM)) Coupling%ebu_smoke = clear_val - Coupling%smoke_ext = clear_val - Coupling%dust_ext = clear_val Coupling%chem3d = clear_val Coupling%ddvel = clear_val Coupling%wetdpc_flux = clear_val @@ -3168,6 +3222,8 @@ subroutine coupling_create (Coupling, IM, Model) Coupling%drydep_flux = clear_val Coupling%min_fplume = clear_val Coupling%max_fplume = clear_val + Coupling%uspdavg = clear_val + Coupling%hpbl_thetav = clear_val Coupling%rrfs_hwp = clear_val Coupling%rrfs_hwp_ave = clear_val endif @@ -3272,6 +3328,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & logical :: cplaqm = .false. !< default no cplaqm collection logical :: cplchm = .false. !< default no cplchm collection logical :: cpllnd = .false. !< default no cpllnd collection + logical :: cpllnd2atm = .false. !< default no cpllnd2atm coupling logical :: rrfs_sd = .false. !< default no rrfs_sd collection logical :: use_cice_alb = .false. !< default no cice albedo logical :: cpl_imp_mrg = .false. !< default no merge import with internal forcings @@ -3857,11 +3914,11 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys) :: dust_moist_correction = 1.0 real(kind=kind_phys) :: dust_alpha = 0. real(kind=kind_phys) :: dust_gamma = 0. - real(kind=kind_phys) :: wetdep_ls_alpha = 0. + real(kind=kind_phys) :: wetdep_ls_alpha = 0.5 integer :: dust_moist_opt = 1 ! fecan :1 else shao integer :: ebb_dcycle = 1 ! 1:retro; 2:forecast integer :: seas_opt = 2 - integer :: dust_opt = 5 + integer :: dust_opt = 1 integer :: drydep_opt = 1 integer :: coarsepm_settling = 1 integer :: plume_wind_eff = 1 @@ -3870,6 +3927,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & logical :: do_plumerise = .false. integer :: addsmoke_flag = 1 integer :: plumerisefire_frq = 60 + integer :: n_dbg_lines = 3 integer :: smoke_forecast = 0 ! RRFS-sd read in ebb_smoke logical :: aero_ind_fdb = .false. ! RRFS-sd wfa/ifa emission logical :: aero_dir_fdb = .false. ! RRFS-sd smoke/dust radiation feedback @@ -3919,8 +3977,8 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & thermodyn_id, sfcpress_id, & !--- coupling parameters cplflx, cplice, cplocn2atm, cplwav, cplwav2atm, cplaqm, & - cplchm, cpllnd, cpl_imp_mrg, cpl_imp_dbg, rrfs_sd, & - use_cice_alb, & + cplchm, cpllnd, cpllnd2atm, cpl_imp_mrg, cpl_imp_dbg, & + rrfs_sd, use_cice_alb, & #ifdef IDEA_PHYS lsidea, weimer_model, f107_kp_size, f107_kp_interval, & f107_kp_skip_size, f107_kp_data_size, f107_kp_read_in_start, & @@ -4054,7 +4112,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & wetdep_ls_opt, smoke_forecast, aero_ind_fdb, aero_dir_fdb, & rrfs_smoke_debug, do_plumerise, plumerisefire_frq, & addsmoke_flag, enh_mix, mix_chem, smoke_dir_fdb_coef, & - do_smoke_transport,smoke_conv_wet_coef, & + do_smoke_transport,smoke_conv_wet_coef,n_dbg_lines, & !--- C3/GF closures ichoice,ichoicem,ichoice_s, & !--- (DFI) time ranges with radar-prescribed microphysics tendencies @@ -4314,6 +4372,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%cplaqm = cplaqm Model%cplchm = cplchm .or. cplaqm Model%cpllnd = cpllnd + Model%cpllnd2atm = cpllnd2atm Model%use_cice_alb = use_cice_alb Model%cpl_imp_mrg = cpl_imp_mrg Model%cpl_imp_dbg = cpl_imp_dbg @@ -4336,6 +4395,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%extended_sd_diags = extended_sd_diags Model%wetdep_ls_opt = wetdep_ls_opt Model%do_plumerise = do_plumerise + Model%n_dbg_lines = n_dbg_lines Model%plumerisefire_frq = plumerisefire_frq Model%addsmoke_flag = addsmoke_flag Model%smoke_forecast = smoke_forecast @@ -6443,6 +6503,7 @@ subroutine control_print(Model) print *, ' cplaqm : ', Model%cplaqm print *, ' cplchm : ', Model%cplchm print *, ' cpllnd : ', Model%cpllnd + print *, ' cpllnd2atm : ', Model%cpllnd2atm print *, ' rrfs_sd : ', Model%rrfs_sd print *, ' use_cice_alb : ', Model%use_cice_alb print *, ' cpl_imp_mrg : ', Model%cpl_imp_mrg diff --git a/scm/src/GFS_typedefs.meta b/scm/src/GFS_typedefs.meta index 654332198..caf348a5e 100644 --- a/scm/src/GFS_typedefs.meta +++ b/scm/src/GFS_typedefs.meta @@ -979,7 +979,7 @@ type = real kind = kind_phys active = (control_for_land_surface_scheme == identifier_for_ruc_land_surface_scheme) -[fire_heat_flux_out] +[fire_heat_flux] standard_name = surface_fire_heat_flux long_name = heat flux of fire at the surface units = W m-2 @@ -987,7 +987,7 @@ type = real kind = kind_phys active = (control_for_land_surface_scheme == identifier_for_ruc_land_surface_scheme) -[frac_grid_burned_out] +[frac_grid_burned] standard_name = fraction_of_grid_cell_burning long_name = ration of the burnt area to the grid cell area units = frac @@ -2302,7 +2302,7 @@ type = real kind = kind_phys active = (do_smoke_coupling) -[frp_input] +[frp_output] standard_name = frp_hourly long_name = hourly fire radiative power units = MW @@ -2326,6 +2326,37 @@ type = real kind = kind_phys active = (do_smoke_coupling) +[fire_type] + standard_name = fire_type + long_name = type of fire + units = 1 + dimensions = (horizontal_loop_extent) + type = integer + active = (do_smoke_coupling) +[peak_hr] + standard_name = peak_hr_fire + long_name = time_of_peak_fire_emissions + units = s + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (do_smoke_coupling) +[lu_nofire] + standard_name = sum_of_land_use_fractions_for_no_fire_pixels + long_name = land use of no fire pixels for type + units = 1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (do_smoke_coupling) +[lu_qfire] + standard_name = sum_of_land_use_fractions_for_cropland_fire_pixels + long_name = land use of fire pixels for type + units = 1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (do_smoke_coupling) [fire_in] standard_name = smoke_fire_auxiliary_input long_name = smoke fire auxiliary input variables @@ -2836,6 +2867,110 @@ type = real kind = kind_phys active = (flag_for_surface_flux_coupling .and. do_mediator_atmosphere_ocean_fluxes) +[sncovr1_lnd] + standard_name = surface_snow_area_fraction_over_land_from_land + long_name = surface snow area fraction over land for coupling + units = frac + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[qsurf_lnd] + standard_name = surface_specific_humidity_over_land_from_land + long_name = surface air saturation specific humidity over land + units = kg kg-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[hflx_lnd] + standard_name = surface_upward_sensible_heat_flux_over_land_from_land + long_name = sfc sensible heat flux input over land for coupling + units = K m s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[evap_lnd] + standard_name = surface_upward_latent_heat_flux_over_land_from_land + long_name = sfc latent heat flux input over land for coupling + units = kg kg-1 m s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[ep_lnd] + standard_name = surface_upward_potential_latent_heat_flux_over_land_from_land + long_name = surface upward potential latent heat flux over land for coupling + units = W m-2 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[t2mmp_lnd] + standard_name = temperature_at_2m_over_land_from_land + long_name = 2 meter temperature over land for coupling + units = K + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[q2mp_lnd] + standard_name = specific_humidity_at_2m_over_land_from_land + long_name = 2 meter specific humidity over land for coupling + units = kg kg-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[gflux_lnd] + standard_name = upward_heat_flux_in_soil_over_land_from_land + long_name = soil heat flux over land for coupling + units = W m-2 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[runoff_lnd] + standard_name = surface_runoff_flux_from_land + long_name = surface runoff flux over land for coupling + units = kg m-2 s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[drain_lnd] + standard_name = subsurface_runoff_flux_from_land + long_name = subsurface runoff flux over land for coupling + units = kg m-2 s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[cmm_lnd] + standard_name = surface_drag_wind_speed_for_momentum_in_air_over_land_from_land + long_name = momentum exchange coefficient over land for coupling + units = m s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[chh_lnd] + standard_name = surface_drag_mass_flux_for_heat_and_moisture_in_air_over_land_from_land + long_name = thermal exchange coefficient over land for coupling + units = kg m-2 s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) +[zvfun_lnd] + standard_name = function_of_surface_roughness_length_and_green_vegetation_fraction_from_land + long_name = function of surface roughness length and green vegetation fraction + units = none + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (flag_for_land_coupling .and. flag_for_one_way_land_coupling_to_atmosphere) [hsnoin_cpl] standard_name = lwe_surface_snow_from_coupled_process long_name = sfc snow depth in meters over sea ice for coupling @@ -3066,6 +3201,22 @@ type = real kind = kind_phys active = (do_smoke_coupling) +[uspdavg] + standard_name = mean_wind_speed_in_boundary_layer + long_name = average wind speed within the boundary layer + units = m s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (do_smoke_coupling) +[hpbl_thetav] + standard_name = atmosphere_boundary_layer_thickness_from_modified_parcel + long_name = pbl height based on modified parcel method + units = m + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys + active = (do_smoke_coupling) [rrfs_hwp] standard_name = hourly_wildfire_potential long_name = rrfs hourly fire weather potential @@ -3444,6 +3595,12 @@ units = flag dimensions = () type = logical +[cpllnd2atm] + standard_name = flag_for_one_way_land_coupling_to_atmosphere + long_name = flag controlling land coupling to the atmosphere (default off) + units = flag + dimensions = () + type = logical [rrfs_sd] standard_name = do_smoke_coupling long_name = flag controlling rrfs_sd collection (default off) @@ -6692,6 +6849,13 @@ dimensions = () type = integer active = (do_smoke_coupling) +[n_dbg_lines] + standard_name = smoke_debug_lines + long_name = rrfs smoke add smoke option + units = index + dimensions = () + type = integer + active = (do_smoke_coupling) [addsmoke_flag] standard_name = control_for_smoke_biomass_burning_emissions long_name = rrfs smoke add smoke option From 467fa613fc8b46efa9e3f524269a1bbd9c45b924 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Fri, 9 Feb 2024 20:45:52 -0500 Subject: [PATCH 004/172] changes to work with ufs/dev PR#155 --- .gitmodules | 2 +- ccpp/physics | 2 +- scm/src/GFS_typedefs.F90 | 17 ++++++++++++++++- scm/src/GFS_typedefs.meta | 20 ++++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index a8006a363..4ded4eb57 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ #url = https://github.com/NCAR/ccpp-physics #branch = main url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR94 + branch = ufs-dev-PR155 [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 57763127c..5356ef23a 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 57763127c01907361fccc5a212082db04cc30f7c +Subproject commit 5356ef23a826ad5235d5616ba616312e56e5303b diff --git a/scm/src/GFS_typedefs.F90 b/scm/src/GFS_typedefs.F90 index 3922c31ef..2e822487a 100644 --- a/scm/src/GFS_typedefs.F90 +++ b/scm/src/GFS_typedefs.F90 @@ -227,6 +227,8 @@ module GFS_typedefs real (kind=kind_phys), pointer :: soiltype_frac(:,:) => null() !< fractions [0:1] of soil categories !< [tsea in gbphys.f] real (kind=kind_phys), pointer :: tsfco (:) => null() !< sst in K + real (kind=kind_phys), pointer :: usfco (:) => null() !< surface zonal current in m s-1 + real (kind=kind_phys), pointer :: vsfco (:) => null() !< surface meridional current in m s-1 real (kind=kind_phys), pointer :: tsfcl (:) => null() !< surface land temperature in K real (kind=kind_phys), pointer :: tisfc (:) => null() !< surface temperature over ice fraction real (kind=kind_phys), pointer :: tiice(:,:) => null() !< internal ice temperature @@ -1309,6 +1311,10 @@ module GFS_typedefs !< 0=no change !< 6=areodynamical roughness over water with input 10-m wind !< 7=slightly decrease Cd for higher wind speed compare to 6 +!--- air_sea_flux scheme + integer :: icplocn2atm !< air_sea flux options over ocean: + !< 0=no change + !< l=including ocean current in the computation of air_sea fluxes !--- potential temperature definition in surface layer physics logical :: thsfc_loc !< flag for local vs. standard potential temperature @@ -2310,6 +2316,8 @@ subroutine sfcprop_create (Sfcprop, IM, Model) endif allocate (Sfcprop%tsfc (IM)) + allocate (Sfcprop%usfco (IM)) + allocate (Sfcprop%vsfco (IM)) allocate (Sfcprop%tsfco (IM)) allocate (Sfcprop%tsfcl (IM)) allocate (Sfcprop%tisfc (IM)) @@ -2367,6 +2375,8 @@ subroutine sfcprop_create (Sfcprop, IM, Model) endif Sfcprop%tsfc = clear_val + Sfcprop%usfco = clear_val + Sfcprop%vsfco = clear_val Sfcprop%tsfco = clear_val Sfcprop%tsfcl = clear_val Sfcprop%tisfc = clear_val @@ -3807,6 +3817,9 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & !< 6=areodynamical roughness over water with input 10-m wind !< 7=slightly decrease Cd for higher wind speed compare to 6 !< negative when cplwav2atm=.true. - i.e. two way wave coupling + integer :: icplocn2atm = 0 !< air_sea_flux options over ocean + !< 0=ocean current is not used in the computation of air_sea fluxes + !< 1=including ocean current in the computation of air_sea fluxes !--- potential temperature definition in surface layer physics logical :: thsfc_loc = .true. !< flag for local vs. standard potential temperature @@ -4079,7 +4092,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & frac_grid, min_lakeice, min_seaice, min_lake_height, & ignore_lake, frac_ice, & !--- surface layer - sfc_z0_type, & + sfc_z0_type, icplocn2atm, & !--- switch beteeen local and standard potential temperature thsfc_loc, & !--- switches in 2-m diagnostics @@ -5055,6 +5068,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & !--- surface layer Model%sfc_z0_type = sfc_z0_type if (Model%cplwav2atm) Model%sfc_z0_type = -1 + Model%icplocn2atm = icplocn2atm !--- potential temperature reference in sfc layer Model%thsfc_loc = thsfc_loc @@ -6880,6 +6894,7 @@ subroutine control_print(Model) print *, ' ' print *, 'surface layer options' print *, ' sfc_z0_type : ', Model%sfc_z0_type + print *, ' icplocn2atm : ', Model%icplocn2atm print *, ' ' print *, 'vertical diffusion coefficients' print *, ' xkzm_m : ', Model%xkzm_m diff --git a/scm/src/GFS_typedefs.meta b/scm/src/GFS_typedefs.meta index caf348a5e..02f58b37a 100644 --- a/scm/src/GFS_typedefs.meta +++ b/scm/src/GFS_typedefs.meta @@ -805,6 +805,20 @@ dimensions = (horizontal_loop_extent) type = real kind = kind_phys +[usfco] + standard_name = x_ocean_current + long_name = zonal current at ocean surface + units = m s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys +[vsfco] + standard_name = y_ocean_current + long_name = meridional current at ocean surface + units = m s-1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys [tsfcl] standard_name = surface_skin_temperature_over_land long_name = surface skin temperature over land @@ -5869,6 +5883,12 @@ units = flag dimensions = () type = integer +[icplocn2atm] + standard_name = control_for_air_sea_flux_computation_over_water + long_name = air-sea flux option + units = 1 + dimensions = () + type = integer [xkzminv] standard_name = max_atmosphere_heat_diffusivity_due_to_background long_name = maximum background value of heat diffusivity From 0b4c7b0de35d79e7754037e2df5597848e56cedd Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Wed, 14 Feb 2024 16:03:40 -0500 Subject: [PATCH 005/172] update ccpp/physics submodule and revert .gitmodules --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 647fdaba6..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR122 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index cc2a974fe..23f120a54 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit cc2a974fe7fb17020262d1c4cb7de9b9786c062a +Subproject commit 23f120a54ac1de039a496f249b07532f07cfb38b From 7d99f783c6e638ef990a48300920c4a5a24d4a70 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Fri, 16 Feb 2024 16:56:40 -0500 Subject: [PATCH 006/172] update ccpp/physics and revert .gitmodules --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0e73a7201..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR120 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 5c349caa9..c552057f5 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 5c349caa9e148a8dc65aec5bc2168f32d9d69ea6 +Subproject commit c552057f566d4a3b75c6ea2689450b3a084d8338 From c3e01ea322c45cfb927f01420faed0e3563059df Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 22 Feb 2024 20:30:31 +0000 Subject: [PATCH 007/172] If using model initial conditions and forcing from the UFS (e.g. Replay mode) with an active LSM, the SCM performs a cold-start initialization for the LSM. That is, UFS replay mode currently support cases generated from UFS Restart files (e.g spun up (warm) LSM parameters) --- scm/etc/scripts/UFS_IC_generator.py | 2 + scm/src/scm_type_defs.F90 | 386 ++++++++-------------------- 2 files changed, 116 insertions(+), 272 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index 6fdbd13fd..6004cb39e 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1187,6 +1187,7 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): "facsf": facsf_in, "facwf": facwf_in, "soiltyp": styp_in, + "scolor": 0., "slopetyp": slope_in, "vegtyp": vtyp_in, "vegfrac": vfrac_in, @@ -2450,6 +2451,7 @@ def write_SCM_case_file(state, surface, oro, forcing, case, date, stateREGRID): {"name": "q2m", "type":wp, "dimd": ('t0'), "units": "kg kg-1", "desc": "2-meter specific humidity"}, \ {"name": "vegtyp", "type":wi, "dimd": ('t0'), "units": "none", "desc": "vegetation type (1-12)"}, \ {"name": "soiltyp", "type":wi, "dimd": ('t0'), "units": "none", "desc": "soil type (1-12)"}, \ + {"name": "scolor", "type":wp, "dimd": ('t0'), "units": "none", "desc": "soil color"}, \ {"name": "ffmm", "type":wp, "dimd": ('t0'), "units": "none", "desc": "Monin-Obukhov similarity function for momentum"}, \ {"name": "ffhh", "type":wp, "dimd": ('t0'), "units": "none", "desc": "Monin-Obukhov similarity function for heat"}, \ {"name": "hice", "type":wp, "dimd": ('t0'), "units": "m", "desc": "sea ice thickness"}, \ diff --git a/scm/src/scm_type_defs.F90 b/scm/src/scm_type_defs.F90 index df026e067..550f7d951 100644 --- a/scm/src/scm_type_defs.F90 +++ b/scm/src/scm_type_defs.F90 @@ -1044,27 +1044,6 @@ subroutine physics_set(physics, scm_input, scm_state) logical :: missing_var(100) real, parameter:: min_lake_orog = 200.0_dp - !check whether input has NoahMP or RUC LSM input data - if (scm_state%model_ics .or. scm_state%lsm_ics) then - if (physics%Model%lsm == physics%Model%lsm_noahmp) then - !FV3GFS_io.F90 uses the presence of the snowxy variable in the ICs to indicate presence of NoahMP warm start - call check_missing(scm_input%input_snowxy, missing_var(1)) - if (missing_var(1)) then - physics%Model%lsm_cold_start = .true. - else - physics%Model%lsm_cold_start = .false. - end if - elseif (physics%Model%lsm == physics%Model%lsm_ruc) then - !RUC LSM uses the tslb variable as soil temperature; if it is missing, assume a cold start using Noah LSM ICs - call check_missing(scm_input%input_tslb(:), missing_var(1)) - if (missing_var(1)) then - physics%Model%lsm_cold_start = .true. - else - physics%Model%lsm_cold_start = .false. - end if - end if - end if - !double check under what circumstances these should actually be set from input!!! (these overwrite the initialzation in GFS_typedefs) missing_var = .false. do i = 1, physics%Model%ncols @@ -1074,6 +1053,9 @@ subroutine physics_set(physics, scm_input, scm_state) physics%Sfcprop%landfrac(i) = missing_value physics%Sfcprop%lakefrac(i) = missing_value end if + ! + ! Orographical data (2D) + ! if (scm_state%model_ics) then write(0,'(a)') "Setting internal physics variables from the orographic section of the case input file (scalars)..." call conditionally_set_var(scm_input%input_stddev, physics%Sfcprop%hprime(i,1), "stddev", .true., missing_var(1)) @@ -1096,7 +1078,6 @@ subroutine physics_set(physics, scm_input, scm_state) call conditionally_set_var(scm_input%input_lakefrac, physics%Sfcprop%lakefrac(i), "lakefrac", (physics%Model%lkm == 1), missing_var(18)) call conditionally_set_var(scm_input%input_lakedepth, physics%Sfcprop%lakedepth(i), "lakedepth", (physics%Model%lkm == 1), missing_var(19)) - !write out warning if missing data for non-required variables n = 19 if ( i==1 .and. ANY( missing_var(1:n) ) ) then write(0,'(a)') "INPUT CHECK: Some missing input data was found related to (potentially non-required) orography and gravity wave drag parameters. This may lead to crashes or other strange behavior." @@ -1108,55 +1089,58 @@ subroutine physics_set(physics, scm_input, scm_state) missing_var = .false. end if - if (scm_state%model_ics .or. scm_state%lsm_ics) then + ! + ! Surface data (2D) + ! + if (scm_state%model_ics .or. scm_state%lsm_ics) then write(0,'(a)') "Setting internal physics variables from the surface section of the case input file (scalars)..." - call conditionally_set_var(scm_input%input_slmsk, physics%Sfcprop%slmsk(i), "slmsk", (.not. physics%Model%frac_grid), missing_var(1)) - call conditionally_set_var(scm_input%input_tsfco, physics%Sfcprop%tsfco(i), "tsfco", .true., missing_var(2)) - call conditionally_set_var(scm_input%input_weasd, physics%Sfcprop%weasd(i), "weasd", .true., missing_var(3)) - call conditionally_set_var(scm_input%input_tg3, physics%Sfcprop%tg3(i), "tg3", .true., missing_var(4)) - call conditionally_set_var(scm_input%input_zorl, physics%Sfcprop%zorl(i), "zorl", .true., missing_var(5)) - call conditionally_set_var(scm_input%input_alvsf, physics%Sfcprop%alvsf(i), "alvsf", .true., missing_var(6)) - call conditionally_set_var(scm_input%input_alnsf, physics%Sfcprop%alnsf(i), "alnsf", .true., missing_var(7)) - call conditionally_set_var(scm_input%input_alvwf, physics%Sfcprop%alvwf(i), "alvwf", .true., missing_var(8)) - call conditionally_set_var(scm_input%input_alnwf, physics%Sfcprop%alnwf(i), "alnwf", .true., missing_var(9)) - call conditionally_set_var(scm_input%input_facsf, physics%Sfcprop%facsf(i), "facsf", .true., missing_var(10)) - call conditionally_set_var(scm_input%input_facwf, physics%Sfcprop%facwf(i), "facwf", .true., missing_var(11)) - call conditionally_set_var(scm_input%input_vegfrac, physics%Sfcprop%vfrac(i), "vegfrac", .true., missing_var(12)) + call conditionally_set_var(scm_input%input_slmsk, physics%Sfcprop%slmsk(i), "slmsk", (.not. physics%Model%frac_grid), missing_var(1)) + call conditionally_set_var(scm_input%input_tsfco, physics%Sfcprop%tsfco(i), "tsfco", .true., missing_var(2)) + call conditionally_set_var(scm_input%input_weasd, physics%Sfcprop%weasd(i), "weasd", .true., missing_var(3)) + call conditionally_set_var(scm_input%input_tg3, physics%Sfcprop%tg3(i), "tg3", .true., missing_var(4)) + call conditionally_set_var(scm_input%input_zorl, physics%Sfcprop%zorl(i), "zorl", .true., missing_var(5)) + call conditionally_set_var(scm_input%input_alvsf, physics%Sfcprop%alvsf(i), "alvsf", .true., missing_var(6)) + call conditionally_set_var(scm_input%input_alnsf, physics%Sfcprop%alnsf(i), "alnsf", .true., missing_var(7)) + call conditionally_set_var(scm_input%input_alvwf, physics%Sfcprop%alvwf(i), "alvwf", .true., missing_var(8)) + call conditionally_set_var(scm_input%input_alnwf, physics%Sfcprop%alnwf(i), "alnwf", .true., missing_var(9)) + call conditionally_set_var(scm_input%input_facsf, physics%Sfcprop%facsf(i), "facsf", .true., missing_var(10)) + call conditionally_set_var(scm_input%input_facwf, physics%Sfcprop%facwf(i), "facwf", .true., missing_var(11)) + call conditionally_set_var(scm_input%input_vegfrac, physics%Sfcprop%vfrac(i), "vegfrac", .true., missing_var(12)) !GJF: is this needed anymore (not in FV3GFS_io)? physics%Interstitial%sigmaf(i) = min(physics%Sfcprop%vfrac(i),0.01) - call conditionally_set_var(scm_input%input_canopy, physics%Sfcprop%canopy(i), "canopy", .true., missing_var(13)) - call conditionally_set_var(scm_input%input_f10m, physics%Sfcprop%f10m(i), "f10m", .false., missing_var(14)) - call conditionally_set_var(scm_input%input_t2m, physics%Sfcprop%t2m(i), "t2m", physics%Model%cplflx, missing_var(15)) - call conditionally_set_var(scm_input%input_q2m, physics%Sfcprop%q2m(i), "q2m", physics%Model%cplflx, missing_var(16)) - call conditionally_set_var(scm_input%input_vegtyp, physics%Sfcprop%vtype(i), "vegtyp", .true., missing_var(17)) - call conditionally_set_var(scm_input%input_soiltyp, physics%Sfcprop%stype(i), "soiltyp", .true., missing_var(18)) - call conditionally_set_var(scm_input%input_uustar, physics%Sfcprop%uustar(i), "uustar", .true., missing_var(19)) - call conditionally_set_var(scm_input%input_ffmm, physics%Sfcprop%ffmm(i), "ffmm", .false., missing_var(20)) - call conditionally_set_var(scm_input%input_ffhh, physics%Sfcprop%ffhh(i), "ffhh", .false., missing_var(21)) - call conditionally_set_var(scm_input%input_hice, physics%Sfcprop%hice(i), "hice", .true., missing_var(22)) - call conditionally_set_var(scm_input%input_fice, physics%Sfcprop%fice(i), "fice", .true., missing_var(23)) - call conditionally_set_var(scm_input%input_tisfc, physics%Sfcprop%tisfc(i), "tisfc", .true., missing_var(24)) - call conditionally_set_var(scm_input%input_tprcp, physics%Sfcprop%tprcp(i), "tprcp", .false., missing_var(25)) - call conditionally_set_var(scm_input%input_srflag, physics%Sfcprop%srflag(i), "srflag", .false., missing_var(26)) - call conditionally_set_var(scm_input%input_snwdph, physics%Sfcprop%snowd(i), "snwdph", .true., missing_var(27)) - call conditionally_set_var(scm_input%input_shdmin, physics%Sfcprop%shdmin(i), "shdmin", .true., missing_var(28)) - call conditionally_set_var(scm_input%input_shdmax, physics%Sfcprop%shdmax(i), "shdmax", .true., missing_var(29)) - call conditionally_set_var(scm_input%input_slopetype, physics%Sfcprop%slope(i), "slopetyp", .true., missing_var(30)) - call conditionally_set_var(scm_input%input_snoalb, physics%Sfcprop%snoalb(i), "snoalb", .true., missing_var(31)) - call conditionally_set_var(scm_input%input_sncovr, physics%Sfcprop%sncovr(i), "sncovr", .false., missing_var(32)) - call conditionally_set_var(scm_input%input_snodl, physics%Sfcprop%snodl(i), "snodl", .false., missing_var(33)) - call conditionally_set_var(scm_input%input_weasdl, physics%Sfcprop%weasdl(i), "weasdl", .false., missing_var(34)) - call conditionally_set_var(scm_input%input_tsfc, physics%Sfcprop%tsfc(i), "tsfc", .false., missing_var(35)) - call conditionally_set_var(scm_input%input_tsfcl, physics%Sfcprop%tsfcl(i), "tsfcl", .false., missing_var(36)) - call conditionally_set_var(scm_input%input_zorlw, physics%Sfcprop%zorlw(i), "zorlw", .false., missing_var(37)) - call conditionally_set_var(scm_input%input_zorll, physics%Sfcprop%zorll(i), "zorll", .false., missing_var(38)) - call conditionally_set_var(scm_input%input_zorli, physics%Sfcprop%zorli(i), "zorli", .false., missing_var(39)) + call conditionally_set_var(scm_input%input_canopy, physics%Sfcprop%canopy(i), "canopy", .true., missing_var(13)) + call conditionally_set_var(scm_input%input_f10m, physics%Sfcprop%f10m(i), "f10m", .false., missing_var(14)) + call conditionally_set_var(scm_input%input_t2m, physics%Sfcprop%t2m(i), "t2m", physics%Model%cplflx, missing_var(15)) + call conditionally_set_var(scm_input%input_q2m, physics%Sfcprop%q2m(i), "q2m", physics%Model%cplflx, missing_var(16)) + call conditionally_set_var(scm_input%input_vegtyp, physics%Sfcprop%vtype(i), "vegtyp", .true., missing_var(17)) + call conditionally_set_var(scm_input%input_soiltyp, physics%Sfcprop%stype(i), "soiltyp", .true., missing_var(18)) + call conditionally_set_var(scm_input%input_uustar, physics%Sfcprop%uustar(i), "uustar", .true., missing_var(19)) + call conditionally_set_var(scm_input%input_ffmm, physics%Sfcprop%ffmm(i), "ffmm", .false., missing_var(20)) + call conditionally_set_var(scm_input%input_ffhh, physics%Sfcprop%ffhh(i), "ffhh", .false., missing_var(21)) + call conditionally_set_var(scm_input%input_hice, physics%Sfcprop%hice(i), "hice", .true., missing_var(22)) + call conditionally_set_var(scm_input%input_fice, physics%Sfcprop%fice(i), "fice", .true., missing_var(23)) + call conditionally_set_var(scm_input%input_tisfc, physics%Sfcprop%tisfc(i), "tisfc", .true., missing_var(24)) + call conditionally_set_var(scm_input%input_tprcp, physics%Sfcprop%tprcp(i), "tprcp", .false., missing_var(25)) + call conditionally_set_var(scm_input%input_srflag, physics%Sfcprop%srflag(i), "srflag", .false., missing_var(26)) + call conditionally_set_var(scm_input%input_snwdph, physics%Sfcprop%snowd(i), "snwdph", .true., missing_var(27)) + call conditionally_set_var(scm_input%input_shdmin, physics%Sfcprop%shdmin(i), "shdmin", .true., missing_var(28)) + call conditionally_set_var(scm_input%input_shdmax, physics%Sfcprop%shdmax(i), "shdmax", .true., missing_var(29)) + call conditionally_set_var(scm_input%input_slopetype, physics%Sfcprop%slope(i), "slopetyp", .true., missing_var(30)) + call conditionally_set_var(scm_input%input_snoalb, physics%Sfcprop%snoalb(i), "snoalb", .true., missing_var(31)) + call conditionally_set_var(scm_input%input_sncovr, physics%Sfcprop%sncovr(i), "sncovr", .false., missing_var(32)) + call conditionally_set_var(scm_input%input_snodl, physics%Sfcprop%snodl(i), "snodl", .false., missing_var(33)) + call conditionally_set_var(scm_input%input_weasdl, physics%Sfcprop%weasdl(i), "weasdl", .false., missing_var(34)) + call conditionally_set_var(scm_input%input_tsfc, physics%Sfcprop%tsfc(i), "tsfc", .false., missing_var(35)) + call conditionally_set_var(scm_input%input_tsfcl, physics%Sfcprop%tsfcl(i), "tsfcl", .false., missing_var(36)) + call conditionally_set_var(scm_input%input_zorlw, physics%Sfcprop%zorlw(i), "zorlw", .false., missing_var(37)) + call conditionally_set_var(scm_input%input_zorll, physics%Sfcprop%zorll(i), "zorll", .false., missing_var(38)) + call conditionally_set_var(scm_input%input_zorli, physics%Sfcprop%zorli(i), "zorli", .false., missing_var(39)) call conditionally_set_var(scm_input%input_albdirvis_lnd, physics%Sfcprop%albdirvis_lnd(i), "albdirvis_lnd", .false., missing_var(40)) call conditionally_set_var(scm_input%input_albdirnir_lnd, physics%Sfcprop%albdirnir_lnd(i), "albdirnir_lnd", .false., missing_var(41)) call conditionally_set_var(scm_input%input_albdifvis_lnd, physics%Sfcprop%albdifvis_lnd(i), "albdifvis_lnd", .false., missing_var(42)) call conditionally_set_var(scm_input%input_albdifnir_lnd, physics%Sfcprop%albdifnir_lnd(i), "albdifnir_lnd", .false., missing_var(43)) - call conditionally_set_var(scm_input%input_emis_lnd, physics%Sfcprop%emis_lnd(i), "emis_lnd", .false., missing_var(44)) - if (physics%Model%use_cice_alb .or. physics%Model%lsm == physics%Model%lsm_ruc) then + call conditionally_set_var(scm_input%input_emis_lnd, physics%Sfcprop%emis_lnd(i), "emis_lnd", .false., missing_var(44)) + if (physics%Model%use_cice_alb) then call conditionally_set_var(scm_input%input_albdirvis_ice, physics%Sfcprop%albdirvis_ice(i), "albdirvis_ice", .false., missing_var(45)) call conditionally_set_var(scm_input%input_albdirnir_ice, physics%Sfcprop%albdirnir_ice(i), "albdirnir_ice", .false., missing_var(46)) call conditionally_set_var(scm_input%input_albdifvis_ice, physics%Sfcprop%albdifvis_ice(i), "albdifvis_ice", .false., missing_var(47)) @@ -1206,12 +1190,10 @@ subroutine physics_set(physics, scm_input, scm_state) if (physics%Sfcprop%slmsk(i) > 1.9_dp) physics%Sfcprop%fice(i) = 1.0 !needed to calculate tsfc and zorl below when model_ics == .false. if (physics%Sfcprop%slmsk(i) < 0.1_dp) physics%Sfcprop%oceanfrac(i) = 1.0 end if - - !this overwrites what is in the suite namelist file -- is that desirable? - !if (scm_state%model_ics) then - ! physics%Model%ivegsrc = scm_input%input_vegsrc - !end if - + + ! + ! Derive physics quantities using surface model ICs. + ! if(scm_state%model_ics .or. scm_state%lsm_ics) then if (physics%Sfcprop%stype(i) == 14 .or. physics%Sfcprop%stype(i)+0.5 <= 0) then physics%Sfcprop%landfrac(i) = real_zero @@ -1311,10 +1293,11 @@ subroutine physics_set(physics, scm_input, scm_state) endif end if - !--- NSSTM variables + ! + ! NSSTM variables + ! if (physics%Model%nstf_name(1) > 0) then - if (physics%Model%nstf_name(2) == 1 .or. .not. (scm_state%model_ics .or. scm_state%lsm_ics)) then ! nsst spinup - !--- nsstm tref + if (physics%Model%nstf_name(2) == 1 .or. .not. (scm_state%model_ics .or. scm_state%lsm_ics)) then physics%Sfcprop%tref(i) = physics%Sfcprop%tsfco(i) physics%Sfcprop%z_c(i) = real_zero physics%Sfcprop%c_0(i) = real_zero @@ -1335,183 +1318,50 @@ subroutine physics_set(physics, scm_input, scm_state) physics%Sfcprop%qrain(i) = real_zero elseif (physics%Model%nstf_name(2) == 0) then ! nsst restart write(0,'(a)') "Setting internal physics variables from the NSST section of the case input file (scalars)..." - call conditionally_set_var(scm_input%input_tref, physics%Sfcprop%tref(i), "tref", .true., missing_var(1)) - call conditionally_set_var(scm_input%input_z_c, physics%Sfcprop%z_c(i), "z_c", .true., missing_var(2)) - call conditionally_set_var(scm_input%input_c_0, physics%Sfcprop%c_0(i), "c_0", .true., missing_var(3)) - call conditionally_set_var(scm_input%input_c_d, physics%Sfcprop%c_d(i), "c_d", .true., missing_var(4)) - call conditionally_set_var(scm_input%input_w_0, physics%Sfcprop%w_0(i), "w_0", .true., missing_var(5)) - call conditionally_set_var(scm_input%input_w_d, physics%Sfcprop%w_d(i), "w_d", .true., missing_var(6)) - call conditionally_set_var(scm_input%input_xt, physics%Sfcprop%xt(i), "xt", .true., missing_var(7)) - call conditionally_set_var(scm_input%input_xs, physics%Sfcprop%xs(i), "xs", .true., missing_var(8)) - call conditionally_set_var(scm_input%input_xu, physics%Sfcprop%xu(i), "xu", .true., missing_var(9)) - call conditionally_set_var(scm_input%input_xv, physics%Sfcprop%xv(i), "xv", .true., missing_var(10)) - call conditionally_set_var(scm_input%input_xz, physics%Sfcprop%xz(i), "xz", .true., missing_var(11)) - call conditionally_set_var(scm_input%input_zm, physics%Sfcprop%zm(i), "zm", .true., missing_var(12)) - call conditionally_set_var(scm_input%input_xtts, physics%Sfcprop%xtts(i), "xtts", .true., missing_var(13)) - call conditionally_set_var(scm_input%input_xzts, physics%Sfcprop%xzts(i), "xzts", .true., missing_var(14)) - call conditionally_set_var(scm_input%input_d_conv, physics%Sfcprop%d_conv(i), "d_conv", .true., missing_var(15)) - call conditionally_set_var(scm_input%input_ifd, physics%Sfcprop%ifd(i), "ifd", .true., missing_var(16)) + call conditionally_set_var(scm_input%input_tref, physics%Sfcprop%tref(i), "tref", .true., missing_var(1)) + call conditionally_set_var(scm_input%input_z_c, physics%Sfcprop%z_c(i), "z_c", .true., missing_var(2)) + call conditionally_set_var(scm_input%input_c_0, physics%Sfcprop%c_0(i), "c_0", .true., missing_var(3)) + call conditionally_set_var(scm_input%input_c_d, physics%Sfcprop%c_d(i), "c_d", .true., missing_var(4)) + call conditionally_set_var(scm_input%input_w_0, physics%Sfcprop%w_0(i), "w_0", .true., missing_var(5)) + call conditionally_set_var(scm_input%input_w_d, physics%Sfcprop%w_d(i), "w_d", .true., missing_var(6)) + call conditionally_set_var(scm_input%input_xt, physics%Sfcprop%xt(i), "xt", .true., missing_var(7)) + call conditionally_set_var(scm_input%input_xs, physics%Sfcprop%xs(i), "xs", .true., missing_var(8)) + call conditionally_set_var(scm_input%input_xu, physics%Sfcprop%xu(i), "xu", .true., missing_var(9)) + call conditionally_set_var(scm_input%input_xv, physics%Sfcprop%xv(i), "xv", .true., missing_var(10)) + call conditionally_set_var(scm_input%input_xz, physics%Sfcprop%xz(i), "xz", .true., missing_var(11)) + call conditionally_set_var(scm_input%input_zm, physics%Sfcprop%zm(i), "zm", .true., missing_var(12)) + call conditionally_set_var(scm_input%input_xtts, physics%Sfcprop%xtts(i), "xtts", .true., missing_var(13)) + call conditionally_set_var(scm_input%input_xzts, physics%Sfcprop%xzts(i), "xzts", .true., missing_var(14)) + call conditionally_set_var(scm_input%input_d_conv, physics%Sfcprop%d_conv(i), "d_conv", .true., missing_var(15)) + call conditionally_set_var(scm_input%input_ifd, physics%Sfcprop%ifd(i), "ifd", .true., missing_var(16)) call conditionally_set_var(scm_input%input_dt_cool, physics%Sfcprop%dt_cool(i), "dt_cool", .true., missing_var(17)) - call conditionally_set_var(scm_input%input_qrain, physics%Sfcprop%qrain(i), "qrain", .true., missing_var(18)) + call conditionally_set_var(scm_input%input_qrain, physics%Sfcprop%qrain(i), "qrain", .true., missing_var(18)) ! all NNST variables are required when NNST spin-up is off, so no need to write out warning for missing data (the model would have already stopped) missing_var = .false. endif endif - - if ((scm_state%model_ics .or. scm_state%lsm_ics) .and. physics%Model%lsm == physics%Model%lsm_ruc .and. .not. physics%Model%lsm_cold_start) then - !--- Extra RUC LSM variables - write(0,'(a)') "Setting internal physics variables from the RUC LSM section of the case input file (scalars)..." - call conditionally_set_var(scm_input%input_wetness, physics%Sfcprop%wetness(i), "wetness", .true., missing_var(1)) - call conditionally_set_var(scm_input%input_clw_surf_land, physics%Sfcprop%clw_surf_land(i), "clw_surf_land", .true., missing_var(2)) - call conditionally_set_var(scm_input%input_clw_surf_ice, physics%Sfcprop%clw_surf_ice(i), "clw_surf_ice", .true., missing_var(3)) - call conditionally_set_var(scm_input%input_qwv_surf_land, physics%Sfcprop%qwv_surf_land(i), "qwv_surf_land", .true., missing_var(4)) - call conditionally_set_var(scm_input%input_qwv_surf_ice, physics%Sfcprop%qwv_surf_ice(i), "qwv_surf_ice", .true., missing_var(5)) - call conditionally_set_var(scm_input%input_tsnow_land, physics%Sfcprop%tsnow_land(i), "tsnow_land", .true., missing_var(6)) - call conditionally_set_var(scm_input%input_tsnow_ice, physics%Sfcprop%tsnow_ice(i), "tsnow_ice", .true., missing_var(7)) - call conditionally_set_var(scm_input%input_snowfallac_land, physics%Sfcprop%snowfallac_land(i), "snowfallac_land", .true., missing_var(8)) - call conditionally_set_var(scm_input%input_snowfallac_ice, physics%Sfcprop%snowfallac_ice(i), "snowfallac_ice", .true., missing_var(9)) - call conditionally_set_var(scm_input%input_sncovr_ice, physics%Sfcprop%sncovr_ice(i), "sncovr_ice", .false., missing_var(10)) - call conditionally_set_var(scm_input%input_sfalb_lnd, physics%Sfcprop%sfalb_lnd(i), "sfalb_lnd", .true., missing_var(11)) - call conditionally_set_var(scm_input%input_sfalb_lnd_bck, physics%Sfcprop%sfalb_lnd_bck(i), "sfalb_lnd_bck", .true., missing_var(12)) - call conditionally_set_var(scm_input%input_sfalb_ice, physics%Sfcprop%sfalb_ice(i), "sfalb_ice", .true., missing_var(13)) - call conditionally_set_var(scm_input%input_emis_ice, physics%Sfcprop%emis_ice(i), "emis_ice", .true., missing_var(14)) - if (physics%Model%lsm == physics%Model%lsm_ruc .and. physics%Model%rdlai) then - !when rdlai = T, RUC LSM expects the LAI to be read in, hence the required variable attribute below - call conditionally_set_var(scm_input%input_lai, physics%Sfcprop%xlaixy(i), "lai", .true., missing_var(15)) - end if - - !if sncovr_ice is missing, set to the land value - if(missing_var(10)) then - call conditionally_set_var(scm_input%input_sncovr, physics%Sfcprop%sncovr_ice(i), "sncovr_ice", .true., missing_var(10)) - end if - - !write out warning if missing data for non-required variables - n = 15 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to (potentially non-required) surface variables for RUC LSM. This may lead to crashes or other strange behavior." - write(0,'(a)') "Check gmtb_scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n - if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - elseif ((scm_state%model_ics .or. scm_state%lsm_ics) .and. physics%Model%lsm == physics%Model%lsm_ruc .and. physics%Model%lsm_cold_start) then - call conditionally_set_var(scm_input%input_sncovr, physics%Sfcprop%sncovr_ice(i), "sncovr_ice", .true., missing_var(1)) - if (physics%Model%rdlai) then - !when rdlai = T, RUC LSM expects the LAI to be read in, hence the required variable attribute below - call conditionally_set_var(scm_input%input_lai, physics%Sfcprop%xlaixy(i), "lai", .true., missing_var(2)) - end if - - !write out warning if missing data for non-required variables - n = 2 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to (potentially non-required) surface variables for RUC LSM. This may lead to crashes or other strange behavior." - write(0,'(a)') "Check gmtb_scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n - if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - elseif ((scm_state%model_ics .or. scm_state%lsm_ics) .and. physics%Model%lsm == physics%Model%lsm_noahmp) then - write(0,'(a)') "Setting internal physics variables from the NoahMP section of the case input file (scalars)..." - !all of these can be missing, since a method exists to "cold start" these variables - call conditionally_set_var(scm_input%input_snowxy, physics%Sfcprop%snowxy(i), "snowxy", .false., missing_var(1)) - call conditionally_set_var(scm_input%input_tvxy, physics%Sfcprop%tvxy(i), "tvxy", .false., missing_var(2)) - call conditionally_set_var(scm_input%input_tgxy, physics%Sfcprop%tgxy(i), "tgxy", .false., missing_var(3)) - call conditionally_set_var(scm_input%input_canicexy, physics%Sfcprop%canicexy(i), "canicexy", .false., missing_var(4)) - call conditionally_set_var(scm_input%input_canliqxy, physics%Sfcprop%canliqxy(i), "canliqxy", .false., missing_var(5)) - call conditionally_set_var(scm_input%input_eahxy, physics%Sfcprop%eahxy(i), "eahxy", .false., missing_var(6)) - call conditionally_set_var(scm_input%input_tahxy, physics%Sfcprop%tahxy(i), "tahxy", .false., missing_var(7)) - call conditionally_set_var(scm_input%input_cmxy, physics%Sfcprop%cmxy(i), "cmxy", .false., missing_var(8)) - call conditionally_set_var(scm_input%input_chxy, physics%Sfcprop%chxy(i), "chxy", .false., missing_var(9)) - call conditionally_set_var(scm_input%input_fwetxy, physics%Sfcprop%fwetxy(i), "fwetxy", .false., missing_var(10)) - call conditionally_set_var(scm_input%input_sneqvoxy, physics%Sfcprop%sneqvoxy(i), "sneqvoxy", .false., missing_var(11)) - call conditionally_set_var(scm_input%input_alboldxy, physics%Sfcprop%alboldxy(i), "alboldxy", .false., missing_var(12)) - call conditionally_set_var(scm_input%input_qsnowxy, physics%Sfcprop%qsnowxy(i), "qsnowxy", .false., missing_var(13)) - call conditionally_set_var(scm_input%input_wslakexy, physics%Sfcprop%wslakexy(i), "wslakexy", .false., missing_var(14)) - call conditionally_set_var(scm_input%input_zwtxy, physics%Sfcprop%zwtxy(i), "zwtxy", .false., missing_var(15)) - call conditionally_set_var(scm_input%input_waxy, physics%Sfcprop%waxy(i), "waxy", .false., missing_var(16)) - call conditionally_set_var(scm_input%input_wtxy, physics%Sfcprop%wtxy(i), "wtxy", .false., missing_var(17)) - call conditionally_set_var(scm_input%input_lfmassxy, physics%Sfcprop%lfmassxy(i), "lfmassxy", .false., missing_var(18)) - call conditionally_set_var(scm_input%input_rtmassxy, physics%Sfcprop%rtmassxy(i), "rtmassxy", .false., missing_var(19)) - call conditionally_set_var(scm_input%input_stmassxy, physics%Sfcprop%stmassxy(i), "stmassxy", .false., missing_var(20)) - call conditionally_set_var(scm_input%input_woodxy, physics%Sfcprop%woodxy(i), "woodxy", .false., missing_var(21)) - call conditionally_set_var(scm_input%input_stblcpxy, physics%Sfcprop%stblcpxy(i), "stblcpxy", .false., missing_var(22)) - call conditionally_set_var(scm_input%input_fastcpxy, physics%Sfcprop%fastcpxy(i), "fastcpxy", .false., missing_var(23)) - call conditionally_set_var(scm_input%input_xsaixy, physics%Sfcprop%xsaixy(i), "xsaixy", .false., missing_var(24)) - call conditionally_set_var(scm_input%input_xlaixy, physics%Sfcprop%xlaixy(i), "xlaixy", .false., missing_var(25)) - call conditionally_set_var(scm_input%input_taussxy, physics%Sfcprop%taussxy(i), "taussxy", .false., missing_var(26)) - call conditionally_set_var(scm_input%input_smcwtdxy, physics%Sfcprop%smcwtdxy(i), "smcwtdxy", .false., missing_var(27)) - call conditionally_set_var(scm_input%input_deeprechxy, physics%Sfcprop%deeprechxy(i), "deeprechxy", .false., missing_var(28)) - call conditionally_set_var(scm_input%input_rechxy, physics%Sfcprop%rechxy(i), "rechxy", .false., missing_var(29)) - - !write out warning if missing data for non-required variables - n = 29 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to surface variables for NoahMP LSM. Due to this, a cold-start algorithm to initialize variables will be used." - write(0,'(a)') "Check scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n - if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - end if - - if ((scm_state%model_ics .or. scm_state%lsm_ics) .and. (physics%Model%lsm == physics%Model%lsm_noah .or. & - physics%Model%lsm == physics%Model%lsm_noahmp .or. physics%Model%lsm_cold_start)) then - + + ! + ! LSM model ICs (3D) + ! + if (scm_state%model_ics .or. scm_state%lsm_ics) then call conditionally_set_var(scm_input%input_stc(:), physics%Sfcprop%stc(i,:), "stc", .true., missing_var(1)) call conditionally_set_var(scm_input%input_smc(:), physics%Sfcprop%smc(i,:), "smc", .true., missing_var(2)) call conditionally_set_var(scm_input%input_slc(:), physics%Sfcprop%slc(i,:), "slc", .true., missing_var(3)) - if (physics%Model%lsm == physics%Model%lsm_noahmp) then - call conditionally_set_var(scm_input%input_snicexy(:), physics%Sfcprop%snicexy(i,:), "snicexy", .false., missing_var(4)) - call conditionally_set_var(scm_input%input_snliqxy(:), physics%Sfcprop%snliqxy(i,:), "sliqexy", .false., missing_var(5)) - call conditionally_set_var(scm_input%input_tsnoxy(:), physics%Sfcprop%tsnoxy(i,:), "tsnoxy", .false., missing_var(6)) - - call conditionally_set_var(scm_input%input_smoiseq(:), physics%Sfcprop%smoiseq(i,:), "smoiseq", .false., missing_var(7)) - - call conditionally_set_var(scm_input%input_zsnsoxy(:), physics%Sfcprop%zsnsoxy(i,:), "zsnzoxy", .false., missing_var(8)) - - !write out warning if missing data for non-required variables - n = 8 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to surface variables for NoahMP LSM. Due to this, a cold-start algorithm to initialize variables will be used." - write(0,'(a)') "Check scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n + n = 3 + if ( i==1 .and. ANY( missing_var(1:n) ) ) then + write(0,'(a)') "INPUT CHECK: Some missing input data was found related to surface variables needed by the LSM. Due to this, a cold-start algorithm to initialize variables will be used." + write(0,'(a)') "Check scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" + do j=1, n if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - endif - - else if ((scm_state%model_ics .or. scm_state%lsm_ics) .and. physics%Model%lsm == physics%Model%lsm_ruc) then - call conditionally_set_var(scm_input%input_tslb(:), physics%Sfcprop%tslb(i,:), "tslb", .false., missing_var(1)) - call conditionally_set_var(scm_input%input_smois(:), physics%Sfcprop%smois(i,:), "smois", .false., missing_var(2)) - call conditionally_set_var(scm_input%input_sh2o(:), physics%Sfcprop%sh2o(i,:), "sh2o", .false., missing_var(3)) - call conditionally_set_var(scm_input%input_smfr(:), physics%Sfcprop%keepsmfr(i,:), "smfr", .false., missing_var(4)) - call conditionally_set_var(scm_input%input_flfr(:), physics%Sfcprop%flag_frsoil(i,:), "flfr", .false., missing_var(5)) - - !write out warning if missing data for non-required variables - n = 5 - if ( i==1 .and. ANY( missing_var(1:n) ) ) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to (potentially non-required) surface variables for RUC LSM. This may lead to crashes or other strange behavior." - write(0,'(a)') "Check scm_type_defs.F90/physics_set to see the names of variables that are missing, corresponding to the following indices:" - do j=1, n - if (missing_var(j)) write(0,'(a,i0)') "variable index ",j - end do - end if - missing_var = .false. - end if - - if (scm_state%model_ics .or. scm_state%lsm_ics) then - !check for nonmissing values - call conditionally_set_var(scm_input%input_tiice(:), physics%Sfcprop%tiice(i,:), "tiice", .false., missing_var(1)) - if (missing_var(1)) then - write(0,'(a)') "INPUT CHECK: Some missing input data was found related to the internal sea ice temperature. These will be set from the internal soil temperature variable (stc)." + end do end if - end if + end if + ! + ! Compute surface fields that may/maynot present in model IC files. + ! if (scm_state%model_ics .or. scm_state%lsm_ics) then if (scm_input%input_snodl <= real_zero) then if (physics%Sfcprop%landfrac(i) > real_zero) then @@ -1563,41 +1413,33 @@ subroutine physics_set(physics, scm_input, scm_state) if (scm_input%input_zorlwav <= real_zero) then physics%Sfcprop%zorlwav(i) = physics%Sfcprop%zorlw(i) !--- compute zorlwav from existing variables end if - - if (physics%Model%lsm_cold_start) then - if(physics%Model%frac_grid .and. (scm_state%model_ics .or. scm_state%lsm_ics)) then ! 3-way composite - if( physics%Model%phour < 1.e-7) physics%Sfcprop%tsfco(i) = max(con_tice, physics%Sfcprop%tsfco(i)) - tem1 = real_one - physics%Sfcprop%landfrac(i) - tem = tem1 * physics%Sfcprop%fice(i) ! tem = ice fraction wrt whole cell - physics%Sfcprop%zorl(i) = physics%Sfcprop%zorll(i) * physics%Sfcprop%landfrac(i) & - + physics%Sfcprop%zorli(i) * tem & - + physics%Sfcprop%zorlw(i) * (tem1-tem) - - physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfcl(i) * physics%Sfcprop%landfrac(i) & - + physics%Sfcprop%tisfc(i) * tem & - + physics%Sfcprop%tsfco(i) * (tem1-tem) - else - !--- specify tsfcl/zorll/zorli from existing variable tsfco/zorlw - ! physics%Sfcprop%tsfcl(i) = physics%Sfcprop%tsfco(i) - ! physics%Sfcprop%zorll(i) = physics%Sfcprop%zorlw(i) - ! physics%Sfcprop%zorli(i) = physics%Sfcprop%zorlw(i) - ! physics%Sfcprop%zorl(i) = physics%Sfcprop%zorlw(i) - ! physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfco(i) - if (physics%Sfcprop%slmsk(i) == 1) then - physics%Sfcprop%zorl(i) = physics%Sfcprop%zorll(i) - physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfcl(i) - else - tem = real_one - physics%Sfcprop%fice(i) - physics%Sfcprop%zorl(i) = physics%Sfcprop%zorli(i) * physics%Sfcprop%fice(i) & - + physics%Sfcprop%zorlw(i) * tem - physics%Sfcprop%tsfc(i) = physics%Sfcprop%tisfc(i) * physics%Sfcprop%fice(i) & - + physics%Sfcprop%tsfco(i) * tem - endif - endif ! if (Model%frac_grid) - endif !if (physics%Model%lsm_cold_start) - - if ((scm_state%model_ics .or. scm_state%lsm_ics) .and. MAXVAL(scm_input%input_tiice) < real_zero) then + if(physics%Model%frac_grid .and. (scm_state%model_ics .or. scm_state%lsm_ics) ) then ! 3-way composite + if( physics%Model%phour < 1.e-7) physics%Sfcprop%tsfco(i) = max(con_tice, physics%Sfcprop%tsfco(i)) + tem1 = real_one - physics%Sfcprop%landfrac(i) + tem = tem1 * physics%Sfcprop%fice(i) ! tem = ice fraction wrt whole cell + physics%Sfcprop%zorl(i) = physics%Sfcprop%zorll(i) * physics%Sfcprop%landfrac(i) & + + physics%Sfcprop%zorli(i) * tem & + + physics%Sfcprop%zorlw(i) * (tem1-tem) + + physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfcl(i) * physics%Sfcprop%landfrac(i) & + + physics%Sfcprop%tisfc(i) * tem & + + physics%Sfcprop%tsfco(i) * (tem1-tem) + else + if (physics%Sfcprop%slmsk(i) == 1) then + physics%Sfcprop%zorl(i) = physics%Sfcprop%zorll(i) + physics%Sfcprop%tsfc(i) = physics%Sfcprop%tsfcl(i) + else + tem = real_one - physics%Sfcprop%fice(i) + physics%Sfcprop%zorl(i) = physics%Sfcprop%zorli(i) * physics%Sfcprop%fice(i) & + + physics%Sfcprop%zorlw(i) * tem + + physics%Sfcprop%tsfc(i) = physics%Sfcprop%tisfc(i) * physics%Sfcprop%fice(i) & + + physics%Sfcprop%tsfco(i) * tem + endif + endif ! if (Model%frac_grid) + + if (scm_state%model_ics .and. MAXVAL(scm_input%input_tiice) < real_zero) then physics%Sfcprop%tiice(i,1) = physics%Sfcprop%stc(i,1) !--- initialize internal ice temp from soil temp at layer 1 physics%Sfcprop%tiice(i,2) = physics%Sfcprop%stc(i,2) !--- initialize internal ice temp from soil temp at layer 2 end if From 53548eaedbc49d75468edeed0eef5553ad6d9d49 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 28 Feb 2024 22:01:05 +0000 Subject: [PATCH 008/172] Some changes for Man to test --- scm/etc/scripts/UFS_IC_generator.py | 14 +++++++++----- scm/src/scm_input.F90 | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index 6004cb39e..2e3df2b68 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1066,10 +1066,14 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): tprcp_in = read_NetCDF_surface_var(nc_file, 'tprcp', i, j, old_chgres, 0) srflag_in = read_NetCDF_surface_var(nc_file, 'srflag', i, j, old_chgres, 0) sncovr_in = read_NetCDF_surface_var(nc_file, 'sncovr', i, j, old_chgres, 0) - tsfcl_in = read_NetCDF_surface_var(nc_file, 'tsfcl', i, j, old_chgres, 0) - zorll_in = read_NetCDF_surface_var(nc_file, 'zorll', i, j, old_chgres, 0) - zorli_in = read_NetCDF_surface_var(nc_file, 'zorli', i, j, old_chgres, 0) - + tsfcl_in = read_NetCDF_surface_var(nc_file, 'tsea', i, j, old_chgres, 0) + zorll_in = read_NetCDF_surface_var(nc_file, 'zorl', i, j, old_chgres, 0) + zorli_in = read_NetCDF_surface_var(nc_file, 'zorl', i, j, old_chgres, 0) + if (snwdph_in > 0): + sncovr_in = 1.0 + else: + sncovr_in = 0.0 + # present when cplwav = T zorlw_in = read_NetCDF_surface_var(nc_file, 'zorlw', i, j, old_chgres, 0) @@ -2393,7 +2397,7 @@ def write_SCM_case_file(state, surface, oro, forcing, case, date, stateREGRID): {"name": "mrsos_forc", "type":wp, "dimd": ('time' ), "units": "kg m-2", "desc": "forcing_mass_content_of_water_in_soil_layer"}] # - var_oro = [{"name": "area", "type":wp, "dimd": ('t0'), "units": "m 2-1", "desc": "grid_cell_area"},\ + var_oro = [{"name": "area", "type":wp, "dimd": ('t0'), "units": "m2", "desc": "grid_cell_area"},\ {"name": "stddev", "type":wp, "dimd": ('t0'), "units": "m", "desc": "standard deviation of subgrid orography"}, \ {"name": "convexity", "type":wp, "dimd": ('t0'), "units": "none", "desc": "convexity of subgrid orography"}, \ {"name": "oa1", "type":wp, "dimd": ('t0'), "units": "none", "desc": "assymetry of subgrid orography 1"}, \ diff --git a/scm/src/scm_input.F90 b/scm/src/scm_input.F90 index a39404366..3269c16ac 100644 --- a/scm/src/scm_input.F90 +++ b/scm/src/scm_input.F90 @@ -1969,6 +1969,7 @@ subroutine get_case_init_DEPHY(scm_state, scm_input) if (trim(input_surfaceForcingLSM) == "lsm") then scm_input%input_ozone = input_ozone(:,active_init_time) scm_input%input_area = input_area(active_init_time) + scm_state%area = input_area(active_init_time) scm_input%input_stddev = input_stddev(active_init_time) scm_input%input_convexity= input_convexity(active_init_time) From f22b07e1fe16449ae341750e948b91982a606744 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 28 Feb 2024 22:42:50 +0000 Subject: [PATCH 009/172] Replay with RUC and NoahMP LSM working --- scm/etc/scripts/UFS_IC_generator.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index 2e3df2b68..af3f45d29 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1175,6 +1175,12 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): # fractional grid tiice_in = read_NetCDF_surface_var(nc_file, 'tiice', i, j, old_chgres, missing_variable_ice_layers) + # soil color + if (slmsk_in == 1): + scolor_in = 4 + else: + scolor_in = 1 + # nc_file.close() @@ -1191,7 +1197,7 @@ def get_UFS_surface_data(dir, tile, i, j, old_chgres, lam): "facsf": facsf_in, "facwf": facwf_in, "soiltyp": styp_in, - "scolor": 0., + "scolor": scolor_in, "slopetyp": slope_in, "vegtyp": vtyp_in, "vegfrac": vfrac_in, From 8e21d36301c6c9b583c397d6fabb8ffe960e303c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 15:17:14 +0000 Subject: [PATCH 010/172] Small cahnge from physics needed for replay --- .gitmodules | 4 ++-- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index dc0798c32..b1421ef8f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - url = https://github.com/NCAR/ccpp-physics - branch = main + url = https://github.com/dustinswales/ccpp-physics + branch = replay_fixLSMinit [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 23f120a54..832dcced3 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 23f120a54ac1de039a496f249b07532f07cfb38b +Subproject commit 832dcced3692bd9c0d3526ebb622f79a02413d1a From 3f06191655cfb0785f49a1e05c7154d51cd19087 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 15:19:28 +0000 Subject: [PATCH 011/172] Synced physics --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index 832dcced3..aabe63844 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 832dcced3692bd9c0d3526ebb622f79a02413d1a +Subproject commit aabe63844ae4a81cc20d1937d737c078380a0392 From 8107c705a0e2f32af773e6b82a9ec8f1dd2eb3e0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 16:54:37 +0000 Subject: [PATCH 012/172] Foundatio for Nvidia CI script --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml new file mode 100644 index 000000000..c976020b6 --- /dev/null +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -0,0 +1,182 @@ +name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers + +on: [push,pull_request,workflow_dispatch] + +jobs: + build_scm: + + # The type of runner that the job will run on + runs-on: ubuntu-22.04 + strategy: + matrix: + fortran-compiler: [nvfortran] + build-type: [Release, Debug] + py-version: [3.7.13, 3.9.12] + + # Environmental variables + env: + NFHOME: /home/runner/netcdf-fortran + NFVERSION: v4.5.3 + bacio_ROOT: /home/runner/bacio + sp_ROOT: /home/runner/NCEPLIBS-sp + w3emc_ROOT: /home/runner/myw3emc + SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm + intel_ROOT: /home/runner/intel + suites: SCM_RAP + + # Workflow steps + steps: + ####################################################################################### + # Initial + ####################################################################################### + - name: Checkout SCM code (into /home/runner/work/ccpp-scm/) + uses: actions/checkout@v3 + + - name: Initialize submodules + run: git submodule update --init --recursive + + ####################################################################################### + # Python setup + ####################################################################################### + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{matrix.py-version}} + + - name: Add conda to system path + run: | + echo $CONDA/bin >> $GITHUB_PATH + + - name: Install NetCDF Python libraries + run: | + conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml + + - name: Update system packages + run: sudo apt-get update + + ####################################################################################### + # Install Nvidia. + ####################################################################################### + - name: cache-nvidia-compilers + id: cache-nvidia-compilers + if: contains(matrix.fortran-compiler, 'nvfortran') + uses: actions/cache@v3 + with: + path: /opt/nvidia/hpc_sdk/ + key: nvhpc-${{ runner.os }}-2024-24.1 + + - name: Nvidia setup compilers + if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' + env: + NVCOMPILERS: /opt/nvidia/hpc_sdk + run: | + wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + + ####################################################################################### + # Install FORTRAN dependencies + ####################################################################################### + - name: Environment for nvidia compiler. + if: contains(matrix.fortran-compiler, 'nvfortran') + run: | + echo "CC=nvc" >> $GITHUB_ENV + echo "FC=nvfortran" >> $GITHUB_ENV + echo "F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk" > $GITHUB_ENV + echo "NFVERSION=v4.4.4" >> $GITHUB_ENV + + - name: Cache bacio library v2.4.1 + id: cache-bacio-fortran + uses: actions/cache@v3 + with: + path: /home/runner/bacio + key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key + + - name: Install bacio library v2.4.1 + if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio + cd bacio && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${bacio_ROOT} ../ + make -j2 + make install + echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV + + - name: Cache SP-library v2.3.3 + id: cache-sp-fortran + uses: actions/cache@v3 + with: + path: /home/runner/NCEPLIBS-sp + key: cache-sp-fortran-${{matrix.fortran-compiler}}-key + + - name: Install SP-library v2.3.3 + if: steps.cache-sp-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp + cd NCEPLIBS-sp && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${sp_ROOT} ../ + make -j2 + make install + echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV + + - name: Cache w3emc library v2.9.2 + id: cache-w3emc-fortran + uses: actions/cache@v3 + with: + path: /home/runner/myw3emc + key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key + + - name: Install w3emc library v2.9.2 + if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc + cd NCEPLIBS-w3emc && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=${w3emc_ROOT} ../ + make -j2 + make install + echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV + + - name: Install NetCDF C library + run: | + sudo apt-get update + sudo apt-get install libnetcdf-dev + + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + + - name: Install NetCDF Fortran library + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git + cd netcdf-fortran + ./configure + make -j + sudo make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + + ####################################################################################### + # Build SCM. + ####################################################################################### + + - name: Configure build with CMake (Release) + if: contains(matrix.build-type, 'Release') + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites} ../src + + - name: Configure build with CMake (Debug) + if: contains(matrix.build-type, 'Debug') + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug ../src + + - name: Build SCM + run: | + cd ${SCM_ROOT}/scm/bin + make -j4 From 271710bf7692d4f558bd17c0d0a431f081a19ae7 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:05:47 +0000 Subject: [PATCH 013/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c976020b6..e9445216c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -10,19 +10,19 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release, Debug] - py-version: [3.7.13, 3.9.12] + build-type: [Release]#, Debug] + py-version: [3.7.13]#, 3.9.12] # Environmental variables env: - NFHOME: /home/runner/netcdf-fortran - NFVERSION: v4.5.3 - bacio_ROOT: /home/runner/bacio - sp_ROOT: /home/runner/NCEPLIBS-sp - w3emc_ROOT: /home/runner/myw3emc - SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - intel_ROOT: /home/runner/intel - suites: SCM_RAP + NFHOME: /home/runner/netcdf-fortran + NFVERSION: v4.5.3 + bacio_ROOT: /home/runner/bacio + sp_ROOT: /home/runner/NCEPLIBS-sp + w3emc_ROOT: /home/runner/myw3emc + SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm + nvidia_ROOT: /home/runner/intel + suites: SCM_RAP # Workflow steps steps: @@ -62,7 +62,7 @@ jobs: if: contains(matrix.fortran-compiler, 'nvfortran') uses: actions/cache@v3 with: - path: /opt/nvidia/hpc_sdk/ + path: /home/runner/hpc_sdk/ key: nvhpc-${{ runner.os }}-2024-24.1 - name: Nvidia setup compilers @@ -70,9 +70,13 @@ jobs: env: NVCOMPILERS: /opt/nvidia/hpc_sdk run: | - wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg + echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list + sudo apt-get update -y + sudo apt-get install -y nvhpc-24-1 +# wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz +# tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz +# nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ####################################################################################### # Install FORTRAN dependencies From 2321196bbe439b8780ff098ba96839cdbe95e7ae Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:14:32 +0000 Subject: [PATCH 014/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index e9445216c..a5adf5f67 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -74,9 +74,6 @@ jobs: echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y sudo apt-get install -y nvhpc-24-1 -# wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz -# tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz -# nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ####################################################################################### # Install FORTRAN dependencies @@ -139,48 +136,3 @@ jobs: make -j2 make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV - - - name: Install NetCDF C library - run: | - sudo apt-get update - sudo apt-get install libnetcdf-dev - - - name: Cache NetCDF Fortran library - id: cache-netcdf-fortran - uses: actions/cache@v3 - with: - path: /home/runner/netcdf-fortran - key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - - - name: Install NetCDF Fortran library - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' - run: | - git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git - cd netcdf-fortran - ./configure - make -j - sudo make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib - - ####################################################################################### - # Build SCM. - ####################################################################################### - - - name: Configure build with CMake (Release) - if: contains(matrix.build-type, 'Release') - run: | - cd ${SCM_ROOT}/scm - mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src - - - name: Configure build with CMake (Debug) - if: contains(matrix.build-type, 'Debug') - run: | - cd ${SCM_ROOT}/scm - mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug ../src - - - name: Build SCM - run: | - cd ${SCM_ROOT}/scm/bin - make -j4 From 0bfbe997b333be71bd4386e07d0e65caa970ffb3 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:28:41 +0000 Subject: [PATCH 015/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a5adf5f67..ab9f76274 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -21,7 +21,7 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - nvidia_ROOT: /home/runner/intel + nvidia_ROOT: /home/runner/nvidia suites: SCM_RAP # Workflow steps @@ -68,17 +68,19 @@ jobs: - name: Nvidia setup compilers if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' env: - NVCOMPILERS: /opt/nvidia/hpc_sdk + NVCOMPILERS: /home/runner/hpc_sdk run: | curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y - sudo apt-get install -y nvhpc-24-1 + sudo apt-get install -d -o=dir::cache=/home/runner/hpc_sdk/ -y nvhpc-24-1 ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - name: Environment for nvidia compiler. + env: + NVCOMPILERS: /home/runner/hpc_sdk if: contains(matrix.fortran-compiler, 'nvfortran') run: | echo "CC=nvc" >> $GITHUB_ENV From c6511633dc6946b35409a668a0976011eff392a2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:41:23 +0000 Subject: [PATCH 016/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ab9f76274..5efbb425a 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -21,7 +21,9 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - nvidia_ROOT: /home/runner/nvidia + NVHPC_SILENT: true + NVHPC_INSTALL_TYPE: single + NVHPC_INSTALL_DIR: /home/runner/nvidia suites: SCM_RAP # Workflow steps @@ -73,7 +75,7 @@ jobs: curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y - sudo apt-get install -d -o=dir::cache=/home/runner/hpc_sdk/ -y nvhpc-24-1 + sudo apt-get install -d -o=dir::cache=/home/runner/hpc_sdk -y nvhpc-24-1 ####################################################################################### # Install FORTRAN dependencies From 915bd81d535fb5b30cd5782f99560b0bc2c98ed4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 17:48:04 +0000 Subject: [PATCH 017/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5efbb425a..260f36c89 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -75,7 +75,7 @@ jobs: curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y - sudo apt-get install -d -o=dir::cache=/home/runner/hpc_sdk -y nvhpc-24-1 + sudo apt-get install -y nvhpc-24-1 ####################################################################################### # Install FORTRAN dependencies From 6aba5e8f9dc6680055e020d56b7aa975c9f6c34c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 18:02:26 +0000 Subject: [PATCH 018/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 260f36c89..45d06cfac 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -28,6 +28,12 @@ jobs: # Workflow steps steps: + + - name: Clear caches + uses: easimon/wipe-cache@main + with: + dry-run: 'true' + ####################################################################################### # Initial ####################################################################################### From 4ec0f3dc9982da948c5c9bf066d6700ff99df79a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 18:13:27 +0000 Subject: [PATCH 019/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 45d06cfac..64e5b017e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -28,12 +28,10 @@ jobs: # Workflow steps steps: - - - name: Clear caches - uses: easimon/wipe-cache@main - with: - dry-run: 'true' - +# - name: Clear caches +# uses: easimon/wipe-cache@main +# with: +# dry-run: 'true' ####################################################################################### # Initial ####################################################################################### @@ -71,7 +69,7 @@ jobs: uses: actions/cache@v3 with: path: /home/runner/hpc_sdk/ - key: nvhpc-${{ runner.os }}-2024-24.1 + key: cache-nvidia-compilers-${{matrix.fortran-compiler}}-key - name: Nvidia setup compilers if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' @@ -96,15 +94,15 @@ jobs: echo "F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk" > $GITHUB_ENV echo "NFVERSION=v4.4.4" >> $GITHUB_ENV - - name: Cache bacio library v2.4.1 - id: cache-bacio-fortran - uses: actions/cache@v3 - with: - path: /home/runner/bacio - key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key +# - name: Cache bacio library v2.4.1 +# id: cache-bacio-fortran +# uses: actions/cache@v3 +# with: +# path: /home/runner/bacio +# key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key - name: Install bacio library v2.4.1 - if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' +# if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio cd bacio && mkdir build && cd build @@ -113,15 +111,15 @@ jobs: make install echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV - - name: Cache SP-library v2.3.3 - id: cache-sp-fortran - uses: actions/cache@v3 - with: - path: /home/runner/NCEPLIBS-sp - key: cache-sp-fortran-${{matrix.fortran-compiler}}-key +# - name: Cache SP-library v2.3.3 +# id: cache-sp-fortran +# uses: actions/cache@v3 +# with: +# path: /home/runner/NCEPLIBS-sp +# key: cache-sp-fortran-${{matrix.fortran-compiler}}-key - name: Install SP-library v2.3.3 - if: steps.cache-sp-fortran.outputs.cache-hit != 'true' +# if: steps.cache-sp-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp cd NCEPLIBS-sp && mkdir build && cd build @@ -130,15 +128,15 @@ jobs: make install echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV - - name: Cache w3emc library v2.9.2 - id: cache-w3emc-fortran - uses: actions/cache@v3 - with: - path: /home/runner/myw3emc - key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key +# - name: Cache w3emc library v2.9.2 +# id: cache-w3emc-fortran +# uses: actions/cache@v3 +# with: +# path: /home/runner/myw3emc +# key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key - name: Install w3emc library v2.9.2 - if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' +# if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc cd NCEPLIBS-w3emc && mkdir build && cd build From 3474475fb799dac0e4a78150c6fe4131411db38e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 18:22:19 +0000 Subject: [PATCH 020/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 64e5b017e..b1db8370c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -85,9 +85,6 @@ jobs: # Install FORTRAN dependencies ####################################################################################### - name: Environment for nvidia compiler. - env: - NVCOMPILERS: /home/runner/hpc_sdk - if: contains(matrix.fortran-compiler, 'nvfortran') run: | echo "CC=nvc" >> $GITHUB_ENV echo "FC=nvfortran" >> $GITHUB_ENV From 2065f86d5bb15fbb91a814c47331239628c2d6a2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 18:34:13 +0000 Subject: [PATCH 021/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b1db8370c..a8ff85974 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -80,6 +80,10 @@ jobs: echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt-get update -y sudo apt-get install -y nvhpc-24-1 + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran ####################################################################################### # Install FORTRAN dependencies From abf075597aa4a2603f4d038bb5e7f3ae226f8c3a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 20:21:28 +0000 Subject: [PATCH 022/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a8ff85974..608fb3df7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -76,10 +76,14 @@ jobs: env: NVCOMPILERS: /home/runner/hpc_sdk run: | - curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg - echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list - sudo apt-get update -y - sudo apt-get install -y nvhpc-24-1 + mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk + wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + nvhpc_2024_241_Linux_x86_64_cuda_12.3/install +# curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg +# echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list +# sudo apt-get update -y +# sudo apt-get install -y nvhpc-24-1 echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From 53260ae124fc0b397673bb9993a7f55ac25d031d Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 20:24:16 +0000 Subject: [PATCH 023/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 608fb3df7..29e345f27 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -63,31 +63,31 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### - - name: cache-nvidia-compilers - id: cache-nvidia-compilers - if: contains(matrix.fortran-compiler, 'nvfortran') - uses: actions/cache@v3 - with: - path: /home/runner/hpc_sdk/ - key: cache-nvidia-compilers-${{matrix.fortran-compiler}}-key - +# - name: cache-nvidia-compilers +# id: cache-nvidia-compilers +# if: contains(matrix.fortran-compiler, 'nvfortran') +# uses: actions/cache@v3 +# with: +# path: /home/runner/hpc_sdk/ +# key: cache-nvidia-compilers-${{matrix.fortran-compiler}}-key +# - name: Nvidia setup compilers if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk run: | - mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk - wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk + wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + nvhpc_2024_241_Linux_x86_64_cuda_12.3/install # curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg # echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list # sudo apt-get update -y # sudo apt-get install -y nvhpc-24-1 - echo "The nvfortran installed is:" - nvfortran --version - echo "The path to nvfortran is:" - command -v nvfortran + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran ####################################################################################### # Install FORTRAN dependencies From 09bad92b9e6abc18ce976843e992a9b225f69c70 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 20:26:28 +0000 Subject: [PATCH 024/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 29e345f27..4de98f44d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -63,14 +63,6 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### -# - name: cache-nvidia-compilers -# id: cache-nvidia-compilers -# if: contains(matrix.fortran-compiler, 'nvfortran') -# uses: actions/cache@v3 -# with: -# path: /home/runner/hpc_sdk/ -# key: cache-nvidia-compilers-${{matrix.fortran-compiler}}-key -# - name: Nvidia setup compilers if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' env: @@ -80,10 +72,6 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install -# curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg -# echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list -# sudo apt-get update -y -# sudo apt-get install -y nvhpc-24-1 echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From 95fac483665c381a8c66352108ef37cbd88937bc Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:15:58 +0000 Subject: [PATCH 025/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4de98f44d..189e35f0c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -21,8 +21,8 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - NVHPC_SILENT: true - NVHPC_INSTALL_TYPE: single +# NVHPC_SILENT: true +# NVHPC_INSTALL_TYPE: single NVHPC_INSTALL_DIR: /home/runner/nvidia suites: SCM_RAP From 79a3e845e93c724f911eb919fd7459f7e3fab537 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:23:22 +0000 Subject: [PATCH 026/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 189e35f0c..6baa3e840 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -23,7 +23,7 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm # NVHPC_SILENT: true # NVHPC_INSTALL_TYPE: single - NVHPC_INSTALL_DIR: /home/runner/nvidia +# NVHPC_INSTALL_DIR: /home/runner/nvidia suites: SCM_RAP # Workflow steps @@ -68,6 +68,9 @@ jobs: env: NVCOMPILERS: /home/runner/hpc_sdk run: | + echo "NVHPC_SILENT=true" >> $GITHUB_ENV + echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz From 4e242d63c1b1b43c235e543b4a6626f66e1b5c22 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:24:16 +0000 Subject: [PATCH 027/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 6baa3e840..c371584eb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -69,8 +69,8 @@ jobs: NVCOMPILERS: /home/runner/hpc_sdk run: | echo "NVHPC_SILENT=true" >> $GITHUB_ENV - echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV + echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz From 0621092d50dfacc8233fec167ee5895bc5dc4631 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:48:01 +0000 Subject: [PATCH 028/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c371584eb..6e3deade7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -52,13 +52,16 @@ jobs: - name: Add conda to system path run: | echo $CONDA/bin >> $GITHUB_PATH + df -h - name: Install NetCDF Python libraries run: | conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml - + df -h - name: Update system packages - run: sudo apt-get update + run: | + sudo apt-get update + df -h ####################################################################################### # Install Nvidia. @@ -68,6 +71,7 @@ jobs: env: NVCOMPILERS: /home/runner/hpc_sdk run: | + df -h echo "NVHPC_SILENT=true" >> $GITHUB_ENV echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV From 091348804d6c70a5758c2793c65f51cf41eb3d43 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 21:59:35 +0000 Subject: [PATCH 029/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 6e3deade7..d26f5bce6 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -3,8 +3,27 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers on: [push,pull_request,workflow_dispatch] jobs: - build_scm: + free-disk-space: + runs-on: ubuntu-latest + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + build_scm: # The type of runner that the job will run on runs-on: ubuntu-22.04 strategy: From e3ef9301c882dcacfbde9543d8f2eea118056f96 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 22:04:58 +0000 Subject: [PATCH 030/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 73 ++++++++++++------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index d26f5bce6..a00f0dca8 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -3,25 +3,25 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers on: [push,pull_request,workflow_dispatch] jobs: - free-disk-space: - runs-on: ubuntu-latest - steps: - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true +# free-disk-space: +# runs-on: ubuntu-latest +# steps: +# +# - name: Free Disk Space (Ubuntu) +# uses: jlumbroso/free-disk-space@main +# with: +# # this might remove tools that are actually needed, +# # if set to "true" but frees about 6 GB +# tool-cache: false +# +# # all of these default to true, but feel free to set to +# # "false" if necessary for your workflow +# android: true +# dotnet: true +# haskell: true +# large-packages: true +# docker-images: true +# swap-storage: true build_scm: # The type of runner that the job will run on @@ -40,17 +40,38 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm -# NVHPC_SILENT: true -# NVHPC_INSTALL_TYPE: single -# NVHPC_INSTALL_DIR: /home/runner/nvidia suites: SCM_RAP # Workflow steps steps: -# - name: Clear caches -# uses: easimon/wipe-cache@main -# with: -# dry-run: 'true' + + ####################################################################################### + # Cleanup space + ####################################################################################### + - name: Check space (pre) + run: | + df -h + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Check space (post) + run: | + df -h + ####################################################################################### # Initial ####################################################################################### From 27c6225db1e3e9042e52c14f1fc246402d6e2497 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 22:08:33 +0000 Subject: [PATCH 031/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a00f0dca8..b5b77eda9 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -70,7 +70,7 @@ jobs: - name: Check space (post) run: | - df -h + df -h ####################################################################################### # Initial From 9eeb2cc43f2fd188836934c161451f4cb696bc31 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 22:42:29 +0000 Subject: [PATCH 032/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b5b77eda9..5e121aa43 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -110,6 +110,9 @@ jobs: if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk + NVHPC_SILENT: true + NVHPC_INSTALL_DIR: /home/runner/hpc_sdk + NVHPC_INSTALL_TYPE: single run: | df -h echo "NVHPC_SILENT=true" >> $GITHUB_ENV From ab09969af55d9d7574f3051a172e31aa6be1d092 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 29 Feb 2024 23:13:55 +0000 Subject: [PATCH 033/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5e121aa43..36dadf9e7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -122,10 +122,10 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - echo "The nvfortran installed is:" - nvfortran --version - echo "The path to nvfortran is:" - command -v nvfortran + #echo "The nvfortran installed is:" + #nvfortran --version + #echo "The path to nvfortran is:" + #command -v nvfortran ####################################################################################### # Install FORTRAN dependencies From 29d6e4e3de3c4052c1b247100d2933e92e44ffb8 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 02:30:55 +0000 Subject: [PATCH 034/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 36dadf9e7..101d2a652 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -3,25 +3,6 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers on: [push,pull_request,workflow_dispatch] jobs: -# free-disk-space: -# runs-on: ubuntu-latest -# steps: -# -# - name: Free Disk Space (Ubuntu) -# uses: jlumbroso/free-disk-space@main -# with: -# # this might remove tools that are actually needed, -# # if set to "true" but frees about 6 GB -# tool-cache: false -# -# # all of these default to true, but feel free to set to -# # "false" if necessary for your workflow -# android: true -# dotnet: true -# haskell: true -# large-packages: true -# docker-images: true -# swap-storage: true build_scm: # The type of runner that the job will run on @@ -106,8 +87,15 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### +# - name: Cache Nvidia HPC +# id: cache-nvidia +# uses: actions/cache@v3 +# with: +# path: /home/runner/hpc_sdk +# key: cache-nvidia-key + - name: Nvidia setup compilers - if: contains(matrix.fortran-compiler, 'nvfortran') && steps.cache-nvidia-compilers.outputs.cache-hit != 'true' +# if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true From 17f0e73182da65af1910adb33baa840ae9619197 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 02:45:19 +0000 Subject: [PATCH 035/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 101d2a652..20bfed11a 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -110,6 +110,7 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install + ls /home/runner/hpc_sdk #echo "The nvfortran installed is:" #nvfortran --version #echo "The path to nvfortran is:" From 4ca35b27c1177c4880b97d016e75839bdccbf584 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 02:53:40 +0000 Subject: [PATCH 036/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 20bfed11a..000f02b15 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -111,6 +111,16 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ls /home/runner/hpc_sdk + echo "NVARCH=`uname -s`_`uname -m`; export NVARCH" >> $GITHUB_ENV + echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV + echo "MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH" >> $GITHUB_ENV + echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV + +$ NVARCH=`uname -s`_`uname -m`; export NVARCH +$ NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS +$ MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH +$ PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH + #echo "The nvfortran installed is:" #nvfortran --version #echo "The path to nvfortran is:" From 69d0945d02a64bd070207e3ec29a60874145d622 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 03:04:58 +0000 Subject: [PATCH 037/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 000f02b15..cfae973fd 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -111,10 +111,11 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ls /home/runner/hpc_sdk - echo "NVARCH=`uname -s`_`uname -m`; export NVARCH" >> $GITHUB_ENV - echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV - echo "MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH" >> $GITHUB_ENV - echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV + NVCOMPILERS=/home/runner/hpc_sdk && export NVCOMPILERS + PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH + LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$LD_LIBRARY_PATH +# echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV +# echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV $ NVARCH=`uname -s`_`uname -m`; export NVARCH $ NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS From 66c067fb52610dc8a5ecbe51566815b095f3b051 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 03:06:36 +0000 Subject: [PATCH 038/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cfae973fd..b75b379f3 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -117,10 +117,10 @@ jobs: # echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV # echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV -$ NVARCH=`uname -s`_`uname -m`; export NVARCH -$ NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS -$ MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH -$ PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH +# NVARCH=`uname -s`_`uname -m`; export NVARCH +# NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS +# MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH +# PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH #echo "The nvfortran installed is:" #nvfortran --version From 584ae012bddb06cdcad306052e2123a795ae7116 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 16:54:26 +0000 Subject: [PATCH 039/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b75b379f3..ba80d82bd 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -111,11 +111,11 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ls /home/runner/hpc_sdk - NVCOMPILERS=/home/runner/hpc_sdk && export NVCOMPILERS - PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH - LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$LD_LIBRARY_PATH -# echo "NVCOMPILERS=/home/runner/hpc_sdk; export NVCOMPILERS" >> $GITHUB_ENV -# echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH" >> $GITHUB_ENV +# NVCOMPILERS=/home/runner/hpc_sdk && export NVCOMPILERS +# PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH +# LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$LD_LIBRARY_PATH + echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV # NVARCH=`uname -s`_`uname -m`; export NVARCH # NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS @@ -136,6 +136,7 @@ jobs: echo "FC=nvfortran" >> $GITHUB_ENV echo "F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk" > $GITHUB_ENV echo "NFVERSION=v4.4.4" >> $GITHUB_ENV + ls /home/runner/hpc_sdk # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 98eaa8a54ed691cd5acdec4f1d7aad7fefac9fea Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 16:56:52 +0000 Subject: [PATCH 040/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ba80d82bd..648330f09 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -95,7 +95,7 @@ jobs: # key: cache-nvidia-key - name: Nvidia setup compilers -# if: steps.cache-nvidia.outputs.cache-hit != 'true' + if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true @@ -111,17 +111,8 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install ls /home/runner/hpc_sdk -# NVCOMPILERS=/home/runner/hpc_sdk && export NVCOMPILERS -# PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH -# LD_LIBRARY_PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$LD_LIBRARY_PATH echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV - -# NVARCH=`uname -s`_`uname -m`; export NVARCH -# NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS -# MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/24.1/compilers/man; export MANPATH -# PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH; export PATH - #echo "The nvfortran installed is:" #nvfortran --version #echo "The path to nvfortran is:" From e95f7045e4fc7016cf834b39c3d9590bb5e3e258 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 17:29:38 +0000 Subject: [PATCH 041/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 648330f09..3aef9c8da 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -102,7 +102,6 @@ jobs: NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single run: | - df -h echo "NVHPC_SILENT=true" >> $GITHUB_ENV echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV @@ -110,13 +109,14 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - ls /home/runner/hpc_sdk - echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV - #echo "The nvfortran installed is:" - #nvfortran --version - #echo "The path to nvfortran is:" - #command -v nvfortran + echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV + echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "MANPATH=$MANPATH":$NVCOMPILERS/$NVARCH/24.1/compilers/man" >> $GITHUB_ENV + echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran ####################################################################################### # Install FORTRAN dependencies From 62a4ca6fda4fbcd9588a9d8edbf467b62ba4ad4c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 17:46:53 +0000 Subject: [PATCH 042/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 3aef9c8da..ea274cf9b 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -97,22 +97,27 @@ jobs: - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' env: - NVCOMPILERS: /home/runner/hpc_sdk +# NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single run: | - echo "NVHPC_SILENT=true" >> $GITHUB_ENV + echo "NVHPC_SILENT=true" >> $GITHUB_ENV echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV + echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV - echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "SWALES here1" + echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV + echo "SWALES here2" + echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV + echo "SWALES here3" echo "MANPATH=$MANPATH":$NVCOMPILERS/$NVARCH/24.1/compilers/man" >> $GITHUB_ENV - echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + echo "SWALES here4" + echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From 78365adb5121a57c76c6eb2f88962d8d40e7549c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 17:48:28 +0000 Subject: [PATCH 043/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ea274cf9b..f981ee7fb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -97,7 +97,7 @@ jobs: - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' env: -# NVCOMPILERS: /home/runner/hpc_sdk + NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single From 7cf9e400908597ff6d9dd016e9cad54c510a61b0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 17:50:59 +0000 Subject: [PATCH 044/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f981ee7fb..38e30ba2f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -97,7 +97,7 @@ jobs: - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' env: - NVCOMPILERS: /home/runner/hpc_sdk + #NVCOMPILERS: /home/runner/hpc_sdk NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single @@ -111,13 +111,13 @@ jobs: nvhpc_2024_241_Linux_x86_64_cuda_12.3/install echo "SWALES here1" echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV - echo "SWALES here2" + echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "SWALES here3" + echo "SWALES here3" echo "MANPATH=$MANPATH":$NVCOMPILERS/$NVARCH/24.1/compilers/man" >> $GITHUB_ENV - echo "SWALES here4" + echo "SWALES here4" echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV - echo "SWALES here5" + echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From eb53bde61cd802762835ffdaa3e1f98c3557fb07 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:06:15 +0000 Subject: [PATCH 045/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 38e30ba2f..f1451131e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -114,7 +114,9 @@ jobs: echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" - echo "MANPATH=$MANPATH":$NVCOMPILERS/$NVARCH/24.1/compilers/man" >> $GITHUB_ENV + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + echo "SWALES here3.5" + echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV echo "SWALES here5" From 55921f7b16b8d9552a0d42c1013e00331ef592b1 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:17:30 +0000 Subject: [PATCH 046/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f1451131e..8bafac11e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -115,10 +115,11 @@ jobs: echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man - echo "SWALES here3.5" - echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV +# echo "SWALES here3.5" +# echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" - echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + export PATH=${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin:${PATH} +# echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version From cf076f905b00befb83b423cb1c1f892c5d2537c0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:22:53 +0000 Subject: [PATCH 047/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 8bafac11e..8eed37a99 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -115,10 +115,10 @@ jobs: echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man -# echo "SWALES here3.5" +# echo "SWALES here3.5" # echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" - export PATH=${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin:${PATH} + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin # echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" From 1ef18b3f1c15025621d29aeab3e5b8923e0c8bb4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:24:10 +0000 Subject: [PATCH 048/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 8eed37a99..978d0dc46 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -118,7 +118,7 @@ jobs: # echo "SWALES here3.5" # echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" - export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin +# export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin # echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" From 20e4948df7e66aec51d1762797ec8265b9e7e336 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:26:54 +0000 Subject: [PATCH 049/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 978d0dc46..e352db3c1 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -114,12 +114,12 @@ jobs: echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" - export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man +# export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man # echo "SWALES here3.5" -# echo "MANPATH=${MANPATH}":${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV + echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" # export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin -# echo "PATH=$NVCOMPILERS/$NVARCH/24.1/compilers/bin:$PATH" >> $GITHUB_ENV + echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version From 629908ec0eeb4260c14cfc82f74db0b946686199 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:27:35 +0000 Subject: [PATCH 050/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index e352db3c1..7bca21b51 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -114,12 +114,12 @@ jobs: echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV echo "SWALES here3" -# export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man # echo "SWALES here3.5" - echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV +# echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV echo "SWALES here4" -# export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin +# echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version From 31cae533caa0b60fbd8aa9b7dfecf1f25755eaa0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:28:38 +0000 Subject: [PATCH 051/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 7bca21b51..5f3f2861b 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -113,12 +113,12 @@ jobs: echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV echo "SWALES here2" echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "SWALES here3" - export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man +# echo "SWALES here3" +# export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man # echo "SWALES here3.5" # echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV - echo "SWALES here4" - export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin +# echo "SWALES here4" +# export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin # echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" From 880165dec370d8ca200f4a20ff544bb2ef1f0b38 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:29:59 +0000 Subject: [PATCH 052/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5f3f2861b..fd1f42d0e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -97,7 +97,8 @@ jobs: - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' env: - #NVCOMPILERS: /home/runner/hpc_sdk + NVCOMPILERS: /home/runner/hpc_sdk + NVARCH: Linux_x86_64 NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single @@ -109,16 +110,16 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - echo "SWALES here1" - echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV - echo "SWALES here2" - echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV +# echo "SWALES here1" +# echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV +# echo "SWALES here2" +# echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV # echo "SWALES here3" # export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man # echo "SWALES here3.5" # echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV # echo "SWALES here4" -# export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin # echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" From 3fc5f81dd0db6f11d45c3868ee389f56608ea64c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:31:44 +0000 Subject: [PATCH 053/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index fd1f42d0e..bd97c2761 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -110,17 +110,8 @@ jobs: wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install -# echo "SWALES here1" -# echo "NVARCH=Linux_x86_64" >> $GITHUB_ENV -# echo "SWALES here2" -# echo "NVCOMPILERS=/home/runner/hpc_sdk" >> $GITHUB_ENV -# echo "SWALES here3" -# export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man -# echo "SWALES here3.5" -# echo "MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man" >> $GITHUB_ENV -# echo "SWALES here4" + echo "SWALES here4" export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin -# echo "PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin" >> $GITHUB_ENV echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version From 2194f2e71cee00ed402a2f5060d1d63d125afa49 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:42:03 +0000 Subject: [PATCH 054/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index bd97c2761..edb7863c5 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -87,12 +87,12 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### -# - name: Cache Nvidia HPC -# id: cache-nvidia -# uses: actions/cache@v3 -# with: -# path: /home/runner/hpc_sdk -# key: cache-nvidia-key + - name: Cache Nvidia HPC + id: cache-nvidia + uses: actions/cache@v3 + with: + path: /home/runner/hpc_sdk + key: cache-nvidia-key - name: Nvidia setup compilers if: steps.cache-nvidia.outputs.cache-hit != 'true' @@ -103,9 +103,9 @@ jobs: NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single run: | - echo "NVHPC_SILENT=true" >> $GITHUB_ENV - echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV - echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV + #echo "NVHPC_SILENT=true" >> $GITHUB_ENV + #echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV + #echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz From 3bd379650a803ea7313017ab2c8ff1aea928d936 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 18:52:44 +0000 Subject: [PATCH 055/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index edb7863c5..3f7c7b1ed 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -103,31 +103,27 @@ jobs: NVHPC_INSTALL_DIR: /home/runner/hpc_sdk NVHPC_INSTALL_TYPE: single run: | - #echo "NVHPC_SILENT=true" >> $GITHUB_ENV - #echo "NVHPC_INSTALL_DIR=/home/runner/hpc_sdk" >> $GITHUB_ENV - #echo "NVHPC_INSTALL_TYPE=single" >> $GITHUB_ENV mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk - wget https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install - echo "SWALES here4" export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - echo "SWALES here5" echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" command -v nvfortran + echo "Removing tarball" + rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - name: Environment for nvidia compiler. run: | - echo "CC=nvc" >> $GITHUB_ENV - echo "FC=nvfortran" >> $GITHUB_ENV - echo "F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk" > $GITHUB_ENV - echo "NFVERSION=v4.4.4" >> $GITHUB_ENV - ls /home/runner/hpc_sdk + export CC=nvc + export FC=nvfortran + export F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk + export NFVERSION=v4.4.4 # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 7d9c4af70bdca79da9159b00ee52e4f33dc1af50 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:11:24 +0000 Subject: [PATCH 056/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 3f7c7b1ed..20a4b17e4 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -22,6 +22,9 @@ jobs: w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm suites: SCM_RAP + F90FLAGS: -O3 + CC: nvc + FC: nvfortran # Workflow steps steps: @@ -122,8 +125,6 @@ jobs: run: | export CC=nvc export FC=nvfortran - export F90FLAGS=-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk - export NFVERSION=v4.4.4 # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 19081c6e04c8b52b593e4633d59ab985e7787cc2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:25:26 +0000 Subject: [PATCH 057/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 20a4b17e4..a4af6c635 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -23,8 +23,8 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm suites: SCM_RAP F90FLAGS: -O3 - CC: nvc - FC: nvfortran + #CC: nvc + #FC: nvfortran # Workflow steps steps: @@ -125,6 +125,8 @@ jobs: run: | export CC=nvc export FC=nvfortran + export CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc + export CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 45b76c4ec2308148693919d442e8db80fb924c6e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:37:34 +0000 Subject: [PATCH 058/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a4af6c635..b355a92a0 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -23,8 +23,8 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm suites: SCM_RAP F90FLAGS: -O3 - #CC: nvc - #FC: nvfortran + CC: nvc + FC: nvfortran # Workflow steps steps: From 0d8c9e47e477c52795a008235d01a56dd9e932db Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:46:18 +0000 Subject: [PATCH 059/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b355a92a0..247610c33 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -25,6 +25,8 @@ jobs: F90FLAGS: -O3 CC: nvc FC: nvfortran + CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc + CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran # Workflow steps steps: From 3365312b29ad3120d5d1c4c46c7c5a5b579f78c9 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 19:56:53 +0000 Subject: [PATCH 060/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 247610c33..a0f857c21 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -125,10 +125,10 @@ jobs: ####################################################################################### - name: Environment for nvidia compiler. run: | - export CC=nvc - export FC=nvfortran - export CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc - export CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran + echo "CC=nvc" >> $GITHUB_ENV + echo "FC=nvfortran" >> $GITHUB_ENV + echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV + echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV # - name: Cache bacio library v2.4.1 # id: cache-bacio-fortran From 8ded5a0779f718c0c1aa191e7fdbd8790f480d82 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:06:37 +0000 Subject: [PATCH 061/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index a0f857c21..7921e29b8 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -125,8 +125,8 @@ jobs: ####################################################################################### - name: Environment for nvidia compiler. run: | - echo "CC=nvc" >> $GITHUB_ENV - echo "FC=nvfortran" >> $GITHUB_ENV + echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV + echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV From d06dd31ad1a2b5bddab2f82162594f1ca80e6093 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:15:20 +0000 Subject: [PATCH 062/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 7921e29b8..515d2d08c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -130,15 +130,15 @@ jobs: echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV -# - name: Cache bacio library v2.4.1 -# id: cache-bacio-fortran -# uses: actions/cache@v3 -# with: -# path: /home/runner/bacio -# key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key + - name: Cache bacio library v2.4.1 + id: cache-bacio-fortran + uses: actions/cache@v3 + with: + path: /home/runner/bacio + key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key - name: Install bacio library v2.4.1 -# if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' + if: steps.cache-bacio-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio cd bacio && mkdir build && cd build @@ -147,15 +147,15 @@ jobs: make install echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV -# - name: Cache SP-library v2.3.3 -# id: cache-sp-fortran -# uses: actions/cache@v3 -# with: -# path: /home/runner/NCEPLIBS-sp -# key: cache-sp-fortran-${{matrix.fortran-compiler}}-key + - name: Cache SP-library v2.3.3 + id: cache-sp-fortran + uses: actions/cache@v3 + with: + path: /home/runner/NCEPLIBS-sp + key: cache-sp-fortran-${{matrix.fortran-compiler}}-key - name: Install SP-library v2.3.3 -# if: steps.cache-sp-fortran.outputs.cache-hit != 'true' + if: steps.cache-sp-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp cd NCEPLIBS-sp && mkdir build && cd build @@ -164,15 +164,15 @@ jobs: make install echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV -# - name: Cache w3emc library v2.9.2 -# id: cache-w3emc-fortran -# uses: actions/cache@v3 -# with: -# path: /home/runner/myw3emc -# key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key + - name: Cache w3emc library v2.9.2 + id: cache-w3emc-fortran + uses: actions/cache@v3 + with: + path: /home/runner/myw3emc + key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key - name: Install w3emc library v2.9.2 -# if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' + if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true' run: | git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc cd NCEPLIBS-w3emc && mkdir build && cd build From 52a22f2f408cafb2ae26d93f7c0b629843723153 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:25:05 +0000 Subject: [PATCH 063/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 515d2d08c..c5fed1110 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -78,16 +78,14 @@ jobs: - name: Add conda to system path run: | echo $CONDA/bin >> $GITHUB_PATH - df -h - name: Install NetCDF Python libraries run: | conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml - df -h + - name: Update system packages run: | sudo apt-get update - df -h ####################################################################################### # Install Nvidia. @@ -123,7 +121,7 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - - name: Environment for nvidia compiler. + - name: Set environment for nvidia compiler. run: | echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV @@ -180,3 +178,25 @@ jobs: make -j2 make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV + + - name: Install NetCDF C library + run: | + sudo apt-get update + sudo apt-get install libnetcdf-dev + + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + + - name: Install NetCDF Fortran library + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git + cd netcdf-fortran + ./configure + make -j + sudo make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib \ No newline at end of file From fe42708a2dcafce60e0523145b2fd91d78749781 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:39:20 +0000 Subject: [PATCH 064/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c5fed1110..8493f8d25 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -192,6 +192,8 @@ jobs: key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library + env: + FCFLAGS: -fPIC if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git @@ -199,4 +201,27 @@ jobs: ./configure make -j sudo make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib \ No newline at end of file + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + + ####################################################################################### + # Build SCM. + ####################################################################################### + + - name: Configure build with CMake (Release) + if: contains(matrix.build-type, 'Release') + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites} ../src + + - name: Configure build with CMake (Debug) + if: contains(matrix.build-type, 'Debug') + run: | + cd ${SCM_ROOT}/scm + mkdir bin && cd bin + cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug ../src + + - name: Build SCM + run: | + cd ${SCM_ROOT}/scm/bin + make -j4 \ No newline at end of file From a6955aba104af1d864e103d234d22ef6b4b66254 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:40:50 +0000 Subject: [PATCH 065/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 8493f8d25..c26b6c884 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -212,7 +212,7 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src + cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From c94ba5a011de6836d22a020f283041af02a48378 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 20:50:10 +0000 Subject: [PATCH 066/172] Update CMakeLists for Nvidia support --- scm/src/CMakeLists.txt | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index b952680b6..42942492d 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,9 +30,6 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) -if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") - message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) -endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose @@ -129,7 +126,8 @@ set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets") #------------------------------------------------------------------------------ # Set compile options if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -foffload=nvptx-none -fopt-info -fno-stack-protector -lm -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") @@ -186,6 +184,37 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + +elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") + if(DEFINED ENABLE_NVIDIA_OPENACC) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc -Minfo=accel") + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + endif() + + if(NOT 32BIT) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8") + endif() + + if (${CMAKE_BUILD_TYPE} MATCHES "Debug") + set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -O0 -g") + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g") + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") + endif() + + set(MPI_C_COMPILER mpicc) + set(MPI_CXX_COMPILER mpicxx) + set(MPI_Fortran_COMPILER mpif90) + + set(CMAKE_C_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + else (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") message (FATAL_ERROR "This program has only been compiled with gfortran and ifort. If another compiler is needed, the appropriate flags must be added in ${CMAKE_SOURCE_DIR}/CMakeLists.txt") endif (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") From 7ff6081afa39c3ffff0866a140769e474be62f77 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 21:22:57 +0000 Subject: [PATCH 067/172] Update CMakeLists for Nvidia support --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c26b6c884..fc5250517 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -11,6 +11,7 @@ jobs: matrix: fortran-compiler: [nvfortran] build-type: [Release]#, Debug] + enable-gpu-acc: [False]#, True] py-version: [3.7.13]#, 3.9.12] # Environmental variables @@ -111,6 +112,7 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" @@ -123,6 +125,14 @@ jobs: ####################################################################################### - name: Set environment for nvidia compiler. run: | + export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + echo "The nvfortran installed is:" + nvfortran --version + echo "The path to nvfortran is:" + command -v nvfortran + echo "Removing tarball" + rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV @@ -212,7 +222,10 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src + if: contains(matrix.enable-gpu-acc, 'True') + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True + else: + cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From b448eb40aeacd0487e86046cd453d3493e3f9e47 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 21:24:20 +0000 Subject: [PATCH 068/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index fc5250517..f19be132a 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -128,7 +128,7 @@ jobs: export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man echo "The nvfortran installed is:" - nvfortran --version + nvfortran --version echo "The path to nvfortran is:" command -v nvfortran echo "Removing tarball" From b268880a562401183616791c5b727178d73205ce Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 21:26:16 +0000 Subject: [PATCH 069/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f19be132a..3b25cf502 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -112,7 +112,7 @@ jobs: tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz nvhpc_2024_241_Linux_x86_64_cuda_12.3/install export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" @@ -126,7 +126,7 @@ jobs: - name: Set environment for nvidia compiler. run: | export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - export MANPATH=${MANPAT}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man echo "The nvfortran installed is:" nvfortran --version echo "The path to nvfortran is:" From f18a65a6bb6761eb984acc6adf5853b74f05ae86 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 21:48:45 +0000 Subject: [PATCH 070/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 3b25cf502..3de8f3438 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -91,15 +91,15 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### - - name: Cache Nvidia HPC - id: cache-nvidia - uses: actions/cache@v3 - with: - path: /home/runner/hpc_sdk - key: cache-nvidia-key +# - name: Cache Nvidia HPC +# id: cache-nvidia +# uses: actions/cache@v3 +# with: +# path: /home/runner/hpc_sdk +# key: cache-nvidia-key - name: Nvidia setup compilers - if: steps.cache-nvidia.outputs.cache-hit != 'true' +# if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVARCH: Linux_x86_64 @@ -125,14 +125,14 @@ jobs: ####################################################################################### - name: Set environment for nvidia compiler. run: | - export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man - echo "The nvfortran installed is:" - nvfortran --version - echo "The path to nvfortran is:" - command -v nvfortran - echo "Removing tarball" - rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + #export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin + #export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man + #echo "The nvfortran installed is:" + #nvfortran --version + #echo "The path to nvfortran is:" + #command -v nvfortran + #echo "Removing tarball" + #rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV From af05846982d3f8819f354c9d7798cc474bd373c6 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 22:02:54 +0000 Subject: [PATCH 071/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 3de8f3438..b5799cb71 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -222,10 +222,11 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - if: contains(matrix.enable-gpu-acc, 'True') - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True - else: - cmake -DCCPP_SUITES=${suites} ../src + cmake -DCCPP_SUITES=${suites} ../src +# if: contains(matrix.enable-gpu-acc, 'True') +# cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True +# else: +# cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From 47f0a6cbe6e70a38885cd36065c264e95a2e0339 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 1 Mar 2024 22:50:47 +0000 Subject: [PATCH 072/172] Updated CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index b5799cb71..1adfa85d1 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -189,6 +189,11 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV + - name: Install Curl + run: | + sudo apt-get update + sudo apt-get install libcurl4-openssl-dev + - name: Install NetCDF C library run: | sudo apt-get update From d2c04ed938763697ebff9f5f158f96aaaf58243e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 4 Mar 2024 22:33:46 +0000 Subject: [PATCH 073/172] Update cmakelists --- scm/src/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 42942492d..4ac9f87c9 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -68,6 +68,9 @@ find_package(NetCDF REQUIRED COMPONENTS C Fortran) find_package(bacio REQUIRED) find_package(sp REQUIRED) find_package(w3emc REQUIRED) +if (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") + find_package(curl REQUIRED) +endif() SET(CCPP_FRAMEWORK_SRC ${CMAKE_SOURCE_DIR}/../../ccpp/framework) SET(CCPP_PHYSICS_SRC ${CMAKE_SOURCE_DIR}/../../ccpp/physics) @@ -280,6 +283,9 @@ TARGET_LINK_LIBRARIES(scm sp::sp_d) TARGET_LINK_LIBRARIES(scm w3emc::w3emc_d) TARGET_LINK_LIBRARIES(scm ccpp_framework) TARGET_LINK_LIBRARIES(scm ccpp_physics) +if (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") + TARGET_LINK_LIBRARIES(scm curl) +endif() SET_TARGET_PROPERTIES(scm PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS}" From 90b7fea11dd7ea67a8bfafecb55d0e157d29bcf0 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 4 Mar 2024 22:44:36 +0000 Subject: [PATCH 074/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 +++--- scm/src/CMakeLists.txt | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 1adfa85d1..c671b50b6 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -84,9 +84,9 @@ jobs: run: | conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml - - name: Update system packages - run: | - sudo apt-get update +# - name: Update system packages +# run: | +# sudo apt-get update ####################################################################################### # Install Nvidia. diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 4ac9f87c9..42942492d 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -68,9 +68,6 @@ find_package(NetCDF REQUIRED COMPONENTS C Fortran) find_package(bacio REQUIRED) find_package(sp REQUIRED) find_package(w3emc REQUIRED) -if (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") - find_package(curl REQUIRED) -endif() SET(CCPP_FRAMEWORK_SRC ${CMAKE_SOURCE_DIR}/../../ccpp/framework) SET(CCPP_PHYSICS_SRC ${CMAKE_SOURCE_DIR}/../../ccpp/physics) @@ -283,9 +280,6 @@ TARGET_LINK_LIBRARIES(scm sp::sp_d) TARGET_LINK_LIBRARIES(scm w3emc::w3emc_d) TARGET_LINK_LIBRARIES(scm ccpp_framework) TARGET_LINK_LIBRARIES(scm ccpp_physics) -if (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") - TARGET_LINK_LIBRARIES(scm curl) -endif() SET_TARGET_PROPERTIES(scm PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS}" From ddb35914f55932f0f28729da18f71b4b1cf75596 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 4 Mar 2024 23:00:16 +0000 Subject: [PATCH 075/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c671b50b6..577c39224 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -51,7 +51,7 @@ jobs: android: true dotnet: true haskell: true - large-packages: true + large-packages: false docker-images: true swap-storage: true From 7ef27e803c94af3cd260eb5f1d52f624b96cb7e9 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 4 Mar 2024 23:09:44 +0000 Subject: [PATCH 076/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 577c39224..ad1b0e909 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -51,9 +51,9 @@ jobs: android: true dotnet: true haskell: true - large-packages: false + large-packages: true docker-images: true - swap-storage: true + swap-storage: false - name: Check space (post) run: | @@ -125,14 +125,6 @@ jobs: ####################################################################################### - name: Set environment for nvidia compiler. run: | - #export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin - #export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man - #echo "The nvfortran installed is:" - #nvfortran --version - #echo "The path to nvfortran is:" - #command -v nvfortran - #echo "Removing tarball" - #rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV From dbc4334bdfa2101e959bbeee3c6fb2c096ccf14e Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 08:33:15 -0700 Subject: [PATCH 077/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ad1b0e909..9fddf3364 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -48,11 +48,11 @@ jobs: # all of these default to true, but feel free to set to # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true + android: false + dotnet: false + haskell: false + large-packages: false + docker-images: false swap-storage: false - name: Check space (post) From a38f5c0548369b641c6ee16aafe4580d387be700 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 09:13:37 -0700 Subject: [PATCH 078/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9fddf3364..78e2d8022 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -51,7 +51,7 @@ jobs: android: false dotnet: false haskell: false - large-packages: false + large-packages: true docker-images: false swap-storage: false From 3f79deaa700c13960e1616156fff59f0d9929763 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 09:36:31 -0700 Subject: [PATCH 079/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 78e2d8022..440ea688f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -50,7 +50,7 @@ jobs: # "false" if necessary for your workflow android: false dotnet: false - haskell: false + haskell: true large-packages: true docker-images: false swap-storage: false From 008903113e5fe479dc8b53488c801818405ed449 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 09:56:06 -0700 Subject: [PATCH 080/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 440ea688f..6fe53375c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -105,7 +105,7 @@ jobs: NVARCH: Linux_x86_64 NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk - NVHPC_INSTALL_TYPE: single + NVHPC_INSTALL_TYPE: auto run: | mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz @@ -181,10 +181,10 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV - - name: Install Curl - run: | - sudo apt-get update - sudo apt-get install libcurl4-openssl-dev +# - name: Install Curl +# run: | +# sudo apt-get update +# sudo apt-get install libcurl4-openssl-dev - name: Install NetCDF C library run: | From 8031de348c8c073bf4baed816a74868b001212d5 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:07:30 -0700 Subject: [PATCH 081/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 6fe53375c..bc5357169 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -105,7 +105,8 @@ jobs: NVARCH: Linux_x86_64 NVHPC_SILENT: true NVHPC_INSTALL_DIR: /home/runner/hpc_sdk - NVHPC_INSTALL_TYPE: auto + NVHPC_INSTALL_TYPE: network + NVHPC_INSTALL_LOCAL_DIR: /home/runner/hpc_sdk run: | mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz From 789711cbe21bc41a5ebbf418dadb8d1a5c2cfef7 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:24:08 -0700 Subject: [PATCH 082/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index bc5357169..4f0e56c57 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -121,6 +121,10 @@ jobs: echo "Removing tarball" rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz + - name: Call uname command + run: | + `uname -s`_`uname -m` + ####################################################################################### # Install FORTRAN dependencies ####################################################################################### From 28267f8dd50cd7514598545ff273587ac7bbe5a3 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:26:21 -0700 Subject: [PATCH 083/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4f0e56c57..cf48b38af 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -123,7 +123,8 @@ jobs: - name: Call uname command run: | - `uname -s`_`uname -m` + uname -s + uname -m ####################################################################################### # Install FORTRAN dependencies From 157490010cb970de896d46d1046e76e05bebf324 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:28:39 -0700 Subject: [PATCH 084/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cf48b38af..1e7fcfadb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -58,6 +58,8 @@ jobs: - name: Check space (post) run: | df -h + uname -c + uname -s ####################################################################################### # Initial @@ -121,11 +123,6 @@ jobs: echo "Removing tarball" rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - - name: Call uname command - run: | - uname -s - uname -m - ####################################################################################### # Install FORTRAN dependencies ####################################################################################### From 923fa461496ce54e1872ea8b887bd1996b4d6d9d Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 10:32:56 -0700 Subject: [PATCH 085/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 1e7fcfadb..388bbedf0 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -38,6 +38,8 @@ jobs: - name: Check space (pre) run: | df -h + uname -s + uname -m - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -58,8 +60,6 @@ jobs: - name: Check space (post) run: | df -h - uname -c - uname -s ####################################################################################### # Initial From 9768b94748065a114230fc2ca4d6a949fa2e1fbb Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 12:52:45 -0700 Subject: [PATCH 086/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 388bbedf0..ec3c84dac 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -38,8 +38,6 @@ jobs: - name: Check space (pre) run: | df -h - uname -s - uname -m - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -184,8 +182,10 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV -# - name: Install Curl -# run: | + - name: Install Curl + run: | + sudo apt-get install curl + sudo apt-get install libssl-dev libcurl4-openssl-dev # sudo apt-get update # sudo apt-get install libcurl4-openssl-dev From e93fb0329f73fbdceaa9ecc1519471b887b541a0 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 13:26:35 -0700 Subject: [PATCH 087/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ec3c84dac..cc06ef771 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -17,11 +17,13 @@ jobs: # Environmental variables env: NFHOME: /home/runner/netcdf-fortran - NFVERSION: v4.5.3 + NFVERSION: v4.6.1 bacio_ROOT: /home/runner/bacio sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm + zlib_ROOT: /home/runner/zlib + HDF5_ROOT: /home/runner/hdf5 suites: SCM_RAP F90FLAGS: -O3 CC: nvc @@ -131,6 +133,32 @@ jobs: echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV + - name: Install zlib + env: + CFLAGS: -fPIC + run: | + wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz + tar -zxvf zlib-1.2.13.tar.gz + cd zlib-1.2.13 + ./configure --prefix=${ZLIB_ROOT} + make + make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ZLIB_ROOT}/lib + + - name: Install HDF5 + env: + CPPFLAGS: -I${ZLIB_ROOT}/include + LDFLAGS: -L${ZLIB_ROOT}/lib + run: | + wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz + tar -zxvf hdf5-1_14_1-2.tar.gz + cd hdf5-hdf5-1_14_1-2 + ./configure --prefix=${HDF5_ROOT} --with-zlib=${ZLIB_ROOT} + make -j4 + make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib + export PATH=${PATH}:${HDF5_ROOT}/lib + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -191,6 +219,9 @@ jobs: - name: Install NetCDF C library run: | +# wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz +# tar -zvxf v4.7.4.tar.gz +# cd netcdf-c-4.7.4 sudo apt-get update sudo apt-get install libnetcdf-dev From 74428b978eba07b779d86fcdcd01cb01517c4410 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 13:27:37 -0700 Subject: [PATCH 088/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cc06ef771..c7f97f6ea 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -158,7 +158,7 @@ jobs: make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib export PATH=${PATH}:${HDF5_ROOT}/lib - + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 From 2f6c1660abd8e42308484a9e48ab8c8e8f7ab8a8 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 13:28:27 -0700 Subject: [PATCH 089/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c7f97f6ea..360fb9bcc 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -210,18 +210,15 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV - - name: Install Curl - run: | - sudo apt-get install curl - sudo apt-get install libssl-dev libcurl4-openssl-dev +# - name: Install Curl +# run: | +# sudo apt-get install curl +# sudo apt-get install libssl-dev libcurl4-openssl-dev # sudo apt-get update # sudo apt-get install libcurl4-openssl-dev - name: Install NetCDF C library run: | -# wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz -# tar -zvxf v4.7.4.tar.gz -# cd netcdf-c-4.7.4 sudo apt-get update sudo apt-get install libnetcdf-dev From 5e97c33e70e2fd78ac7c2d838e7b1a33b65d86ab Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 13:55:51 -0700 Subject: [PATCH 090/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 360fb9bcc..535e1383d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -140,20 +140,20 @@ jobs: wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz tar -zxvf zlib-1.2.13.tar.gz cd zlib-1.2.13 - ./configure --prefix=${ZLIB_ROOT} + ./configure --prefix=${zlib_ROOT} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ZLIB_ROOT}/lib + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${zlib_ROOT}/lib - name: Install HDF5 env: - CPPFLAGS: -I${ZLIB_ROOT}/include - LDFLAGS: -L${ZLIB_ROOT}/lib + CPPFLAGS: -I${zlib_ROOT}/include + LDFLAGS: -L${zlib_ROOT}/lib run: | wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz tar -zxvf hdf5-1_14_1-2.tar.gz cd hdf5-hdf5-1_14_1-2 - ./configure --prefix=${HDF5_ROOT} --with-zlib=${ZLIB_ROOT} + ./configure --prefix=${HDF5_ROOT} --with-zlib=${zlib_ROOT} make -j4 make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib From 9286df0eba8d268bc030cf1d8c8640cb4d6cfb7c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 14:38:33 -0700 Subject: [PATCH 091/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 535e1383d..cf7877460 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -17,6 +17,7 @@ jobs: # Environmental variables env: NFHOME: /home/runner/netcdf-fortran + NCHOME: /home/runner/netcdf NFVERSION: v4.6.1 bacio_ROOT: /home/runner/bacio sp_ROOT: /home/runner/NCEPLIBS-sp @@ -159,6 +160,41 @@ jobs: export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib export PATH=${PATH}:${HDF5_ROOT}/lib + - name: Install NetCDF C library + env: + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib + run: | + wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz + tar -zvxf v4.7.4.tar.gz + cd netcdf-c-4.7.4 + ./configure --prefix=${NCHOME} + make + make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NCHOME}/lib + export PATH=${PATH}:${NCHOME}/lib +# sudo apt-get update +# sudo apt-get install libnetcdf-dev + + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + + - name: Install NetCDF Fortran library + env: + FCFLAGS: -fPIC + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git + cd netcdf-fortran + ./configure + make -j + sudo make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -217,30 +253,6 @@ jobs: # sudo apt-get update # sudo apt-get install libcurl4-openssl-dev - - name: Install NetCDF C library - run: | - sudo apt-get update - sudo apt-get install libnetcdf-dev - - - name: Cache NetCDF Fortran library - id: cache-netcdf-fortran - uses: actions/cache@v3 - with: - path: /home/runner/netcdf-fortran - key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - - - name: Install NetCDF Fortran library - env: - FCFLAGS: -fPIC - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' - run: | - git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git - cd netcdf-fortran - ./configure - make -j - sudo make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib - ####################################################################################### # Build SCM. ####################################################################################### From fa714f42a4883a4de14def1c43fe2d7e933da0da Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 14:49:02 -0700 Subject: [PATCH 092/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cf7877460..92678ae3a 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -160,6 +160,11 @@ jobs: export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib export PATH=${PATH}:${HDF5_ROOT}/lib + - name: Install Curl + run: | + sudo apt-get install curl + sudo apt-get install libssl-dev libcurl4-openssl-dev + - name: Install NetCDF C library env: CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include From de7cde1e721ead31a62e3362b71a8789a03fa286 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:10:54 -0700 Subject: [PATCH 093/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 92678ae3a..d574af144 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -17,7 +17,7 @@ jobs: # Environmental variables env: NFHOME: /home/runner/netcdf-fortran - NCHOME: /home/runner/netcdf + NETCDF: /home/runner/netcdf NFVERSION: v4.6.1 bacio_ROOT: /home/runner/bacio sp_ROOT: /home/runner/NCEPLIBS-sp @@ -170,35 +170,39 @@ jobs: CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib run: | + echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV + echo "LDFLAGS=-L/home/runner/hdf5/lib -I/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 - ./configure --prefix=${NCHOME} + ./configure --prefix=${NETCDF} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NCHOME}/lib - export PATH=${PATH}:${NCHOME}/lib -# sudo apt-get update -# sudo apt-get install libnetcdf-dev + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib + export PATH=${PATH}:${NETCDF}/lib - - name: Cache NetCDF Fortran library - id: cache-netcdf-fortran - uses: actions/cache@v3 - with: - path: /home/runner/netcdf-fortran - key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key +# - name: Cache NetCDF Fortran library +# id: cache-netcdf-fortran +# uses: actions/cache@v3 +# with: +# path: /home/runner/netcdf-fortran +# key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library env: FCFLAGS: -fPIC - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + FFLAGS: -fPIC + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib +# if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | - git clone --branch ${NFVERSION} https://github.com/Unidata/netcdf-fortran.git - cd netcdf-fortran - ./configure - make -j - sudo make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz + tar -zvxf v4.6.1.tar.gz + cd netcdf-fortran-4.6.1 + ./configure --prefix=${NETCDF} + make + make install + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib - name: Cache bacio library v2.4.1 id: cache-bacio-fortran From 22f18fcae854cc5784d9c7d971562e347f8a2a86 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:12:37 -0700 Subject: [PATCH 094/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index d574af144..eaad201d8 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -200,7 +200,7 @@ jobs: tar -zvxf v4.6.1.tar.gz cd netcdf-fortran-4.6.1 ./configure --prefix=${NETCDF} - make + make make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib From 990e36bcaf5b9bcac97e766bb161e961a5d8826c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:15:30 -0700 Subject: [PATCH 095/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index eaad201d8..332700b6b 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -171,7 +171,7 @@ jobs: LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib run: | echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV - echo "LDFLAGS=-L/home/runner/hdf5/lib -I/home/runner/zlib/lib" >> $GITHUB_ENV + echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From bf45ad5dddbcab7e18a6be415c0c0927eb172f9c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:32:13 -0700 Subject: [PATCH 096/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 332700b6b..ab3646d8f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -31,6 +31,7 @@ jobs: FC: nvfortran CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran + LD_LIBRARY_PATH: ${{ github.workspace }} # Workflow steps steps: From f6673e7ba9c60a411a220b2be16f223e6750e261 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:41:02 -0700 Subject: [PATCH 097/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index ab3646d8f..93b624901 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -31,7 +31,7 @@ jobs: FC: nvfortran CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran - LD_LIBRARY_PATH: ${{ github.workspace }} +# LD_LIBRARY_PATH: ${{ github.workspace }} # Workflow steps steps: @@ -145,7 +145,8 @@ jobs: ./configure --prefix=${zlib_ROOT} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${zlib_ROOT}/lib + #export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${zlib_ROOT}/lib + echo "LD_LIBRARY_PATH=$zlib_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Install HDF5 env: @@ -158,8 +159,10 @@ jobs: ./configure --prefix=${HDF5_ROOT} --with-zlib=${zlib_ROOT} make -j4 make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib - export PATH=${PATH}:${HDF5_ROOT}/lib + echo "LD_LIBRARY_PATH=$HDF5_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=$HDF5_ROOT/lib:$PATH" >> $GITHUB_ENV +# export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib +# export PATH=${PATH}:${HDF5_ROOT}/lib - name: Install Curl run: | @@ -179,8 +182,10 @@ jobs: ./configure --prefix=${NETCDF} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib - export PATH=${PATH}:${NETCDF}/lib + echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=$NETCDF/lib:$PATH" >> $GITHUB_ENV +# export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib +# export PATH=${PATH}:${NETCDF}/lib # - name: Cache NetCDF Fortran library # id: cache-netcdf-fortran @@ -203,7 +208,6 @@ jobs: ./configure --prefix=${NETCDF} make make install - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib - name: Cache bacio library v2.4.1 id: cache-bacio-fortran From 8f57ff042c53bb7808f0de73ff53b515a020f1f7 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 15:57:37 -0700 Subject: [PATCH 098/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 93b624901..4228863b3 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -179,7 +179,7 @@ jobs: wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 - ./configure --prefix=${NETCDF} + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" ./configure --prefix=${NETCDF} make make install echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV From 407280d2f121d184d3425847d970883c2a9aaef7 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 16:12:50 -0700 Subject: [PATCH 099/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4228863b3..0dd82959d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -205,7 +205,7 @@ jobs: wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz tar -zvxf v4.6.1.tar.gz cd netcdf-fortran-4.6.1 - ./configure --prefix=${NETCDF} + FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" ./configure --prefix=${NETCDF} make make install From 3d10750f2466f7902a9f9a29027dde5ef8c3f1cb Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:24:56 -0700 Subject: [PATCH 100/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 61 ++++++++----------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 0dd82959d..4c2f8ae30 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -88,22 +88,18 @@ jobs: run: | conda install --yes -c conda-forge h5py>=3.4 netCDF4 f90nml -# - name: Update system packages -# run: | -# sudo apt-get update - ####################################################################################### # Install Nvidia. ####################################################################################### -# - name: Cache Nvidia HPC -# id: cache-nvidia -# uses: actions/cache@v3 -# with: -# path: /home/runner/hpc_sdk -# key: cache-nvidia-key + - name: Cache Nvidia HPC + id: cache-nvidia + uses: actions/cache@v3 + with: + path: /home/runner/hpc_sdk + key: cache-nvidia-key - name: Nvidia setup compilers -# if: steps.cache-nvidia.outputs.cache-hit != 'true' + if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVARCH: Linux_x86_64 @@ -161,18 +157,24 @@ jobs: make install echo "LD_LIBRARY_PATH=$HDF5_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=$HDF5_ROOT/lib:$PATH" >> $GITHUB_ENV -# export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HDF5_ROOT}/lib -# export PATH=${PATH}:${HDF5_ROOT}/lib - name: Install Curl run: | sudo apt-get install curl sudo apt-get install libssl-dev libcurl4-openssl-dev + - name: Cache NetCDF C library + id: cache-netcdf-c + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-c + key: cache-netcdf-c-${{matrix.fortran-compiler}}-key + - name: Install NetCDF C library env: CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib + if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV @@ -184,23 +186,21 @@ jobs: make install echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=$NETCDF/lib:$PATH" >> $GITHUB_ENV -# export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NETCDF}/lib -# export PATH=${PATH}:${NETCDF}/lib -# - name: Cache NetCDF Fortran library -# id: cache-netcdf-fortran -# uses: actions/cache@v3 -# with: -# path: /home/runner/netcdf-fortran -# key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key + - name: Cache NetCDF Fortran library + id: cache-netcdf-fortran + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-fortran + key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library - env: - FCFLAGS: -fPIC - FFLAGS: -fPIC - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib -# if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' +# env: +# FCFLAGS: -fPIC +# FFLAGS: -fPIC +# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include +# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz tar -zvxf v4.6.1.tar.gz @@ -260,13 +260,6 @@ jobs: make install echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV -# - name: Install Curl -# run: | -# sudo apt-get install curl -# sudo apt-get install libssl-dev libcurl4-openssl-dev -# sudo apt-get update -# sudo apt-get install libcurl4-openssl-dev - ####################################################################################### # Build SCM. ####################################################################################### From cf8830f1926fdf6f7696019dd62e284604714dee Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:32:48 -0700 Subject: [PATCH 101/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4c2f8ae30..c2a31d60d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -31,7 +31,6 @@ jobs: FC: nvfortran CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran -# LD_LIBRARY_PATH: ${{ github.workspace }} # Workflow steps steps: @@ -91,15 +90,8 @@ jobs: ####################################################################################### # Install Nvidia. ####################################################################################### - - name: Cache Nvidia HPC - id: cache-nvidia - uses: actions/cache@v3 - with: - path: /home/runner/hpc_sdk - key: cache-nvidia-key - name: Nvidia setup compilers - if: steps.cache-nvidia.outputs.cache-hit != 'true' env: NVCOMPILERS: /home/runner/hpc_sdk NVARCH: Linux_x86_64 From bb798ac1a60f13bfb9e9ba0a7024327e01dc754d Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:48:22 -0700 Subject: [PATCH 102/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c2a31d60d..d6c9432ba 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -91,7 +91,7 @@ jobs: # Install Nvidia. ####################################################################################### - - name: Nvidia setup compilers + - name: Nvidia setup compilers. env: NVCOMPILERS: /home/runner/hpc_sdk NVARCH: Linux_x86_64 @@ -113,19 +113,29 @@ jobs: echo "Removing tarball" rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz - ####################################################################################### - # Install FORTRAN dependencies - ####################################################################################### - - name: Set environment for nvidia compiler. + - name: Set environment for Nvidia compiler. run: | echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV + ####################################################################################### + # Install FORTRAN dependencies + ####################################################################################### + + - name: Cache Zlib + run: | + id: cache-zlib + uses: actions/cache@v3 + with: + path: /home/runner/zlib + key: cache-zlib-${{matrix.fortran-compiler}}-key + - name: Install zlib env: CFLAGS: -fPIC + if: steps.cache-zlib.outputs.cache-hit != 'true' run: | wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz tar -zxvf zlib-1.2.13.tar.gz @@ -133,13 +143,21 @@ jobs: ./configure --prefix=${zlib_ROOT} make make install - #export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${zlib_ROOT}/lib echo "LD_LIBRARY_PATH=$zlib_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + - name: Cache HDF5 + run: | + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: /home/runner/hdf5 + key: cache-hdf5-${{matrix.fortran-compiler}}-key + - name: Install HDF5 env: CPPFLAGS: -I${zlib_ROOT}/include LDFLAGS: -L${zlib_ROOT}/lib + if: steps.cache-hdf5.outputs.cache-hit != 'true' run: | wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz tar -zxvf hdf5-1_14_1-2.tar.gz @@ -163,13 +181,13 @@ jobs: key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library - env: - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib +# env: +# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include +# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | - echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV - echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV +# echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV +# echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From 398a2920c63cd38f6b86432c565a2926821a828d Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:49:32 -0700 Subject: [PATCH 103/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index d6c9432ba..f2c14142e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -131,7 +131,7 @@ jobs: with: path: /home/runner/zlib key: cache-zlib-${{matrix.fortran-compiler}}-key - + - name: Install zlib env: CFLAGS: -fPIC From 6bc8277b81c8312fc4c396f153f1563505578a18 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:52:02 -0700 Subject: [PATCH 104/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f2c14142e..e5e155e82 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -173,12 +173,12 @@ jobs: sudo apt-get install curl sudo apt-get install libssl-dev libcurl4-openssl-dev - - name: Cache NetCDF C library - id: cache-netcdf-c - uses: actions/cache@v3 - with: - path: /home/runner/netcdf-c - key: cache-netcdf-c-${{matrix.fortran-compiler}}-key +# - name: Cache NetCDF C library +# id: cache-netcdf-c +# uses: actions/cache@v3 +# with: +# path: /home/runner/netcdf-c +# key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library # env: From 24e091e976bceb782b9a4078c33863fab18647bc Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:53:33 -0700 Subject: [PATCH 105/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index e5e155e82..4d7fe545f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -173,17 +173,17 @@ jobs: sudo apt-get install curl sudo apt-get install libssl-dev libcurl4-openssl-dev -# - name: Cache NetCDF C library -# id: cache-netcdf-c -# uses: actions/cache@v3 -# with: -# path: /home/runner/netcdf-c -# key: cache-netcdf-c-${{matrix.fortran-compiler}}-key + - name: Cache NetCDF C library + id: cache-netcdf-c + uses: actions/cache@v3 + with: + path: /home/runner/netcdf-c + key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library -# env: -# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib + env: + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | # echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV From b1ce93cc3b208436f7ff530164e5da4976d3459a Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:55:17 -0700 Subject: [PATCH 106/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4d7fe545f..cc9384522 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -181,13 +181,11 @@ jobs: key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library - env: - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib +# env: +# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include +# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | -# echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV -# echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From 9df5652aa001a0b7f96a398b48c8196fe5d78c2d Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:57:10 -0700 Subject: [PATCH 107/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index cc9384522..2735f2428 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -181,11 +181,13 @@ jobs: key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library -# env: -# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib + env: + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | + echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV + echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From fb7f34f3c52965bb6f1627680ea42a4cdeb913e1 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 19:59:57 -0700 Subject: [PATCH 108/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 2735f2428..9c6d988b1 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -205,11 +205,11 @@ jobs: key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library -# env: -# FCFLAGS: -fPIC -# FFLAGS: -fPIC -# CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include -# LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib + env: + FCFLAGS: -fPIC + FFLAGS: -fPIC + CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include + LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz From af52b7647ec3b8e272394d488f018e70d5e3c95c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:04:18 -0700 Subject: [PATCH 109/172] Update CI --- .../ci_build_scm_ubuntu_22.04_nvidia.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9c6d988b1..c85b1693d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -124,14 +124,6 @@ jobs: # Install FORTRAN dependencies ####################################################################################### - - name: Cache Zlib - run: | - id: cache-zlib - uses: actions/cache@v3 - with: - path: /home/runner/zlib - key: cache-zlib-${{matrix.fortran-compiler}}-key - - name: Install zlib env: CFLAGS: -fPIC @@ -145,14 +137,6 @@ jobs: make install echo "LD_LIBRARY_PATH=$zlib_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - - name: Cache HDF5 - run: | - id: cache-hdf5 - uses: actions/cache@v3 - with: - path: /home/runner/hdf5 - key: cache-hdf5-${{matrix.fortran-compiler}}-key - - name: Install HDF5 env: CPPFLAGS: -I${zlib_ROOT}/include From 035a7659c82af4b2a3e4f9445e0e81afa97e7134 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:08:34 -0700 Subject: [PATCH 110/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c85b1693d..4c2596293 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -165,13 +165,8 @@ jobs: key: cache-netcdf-c-${{matrix.fortran-compiler}}-key - name: Install NetCDF C library - env: - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib if: steps.cache-netcdf-c.outputs.cache-hit != 'true' run: | - echo "CPPFLAGS=-I/home/runner/hdf5/include -I/home/runner/zlib/include" >> $GITHUB_ENV - echo "LDFLAGS=-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" >> $GITHUB_ENV wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 From d7a6bff9b9f0f874e6e2bccb3ed43614a56cf8b0 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:09:36 -0700 Subject: [PATCH 111/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 4c2596293..84c3ab44e 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -184,11 +184,6 @@ jobs: key: cache-netcdf-fortran-${{matrix.fortran-compiler}}-key - name: Install NetCDF Fortran library - env: - FCFLAGS: -fPIC - FFLAGS: -fPIC - CPPFLAGS: -I${zlib_DIR}/include -I${HDF5_ROOT}/include -I${NETCDF}/include - LDFLAGS: -L${zlib_DIR}/lib -L${HDF5_ROOT}/lib -L${NETCDF}/lib if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' run: | wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz From fbf58568a498bddcfa547a5d222f93ca4ff6ac3f Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:10:19 -0700 Subject: [PATCH 112/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 84c3ab44e..5b6401446 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -170,7 +170,9 @@ jobs: wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 - CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" ./configure --prefix=${NETCDF} + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" + LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" + ./configure --prefix=${NETCDF} make make install echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV From 897cfca7d910cc4f07fc56622ae678b34641aa18 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:11:54 -0700 Subject: [PATCH 113/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5b6401446..358a68501 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -191,7 +191,10 @@ jobs: wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz tar -zvxf v4.6.1.tar.gz cd netcdf-fortran-4.6.1 - FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" ./configure --prefix=${NETCDF} + FCFLAGS="-fPIC" FFLAGS="-fPIC" + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" + LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" + ./configure --prefix=${NETCDF} make make install From 3eab6709382c48437e28ae7ec9b6f8705ddb6008 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:16:11 -0700 Subject: [PATCH 114/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 358a68501..5c14d1371 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -25,12 +25,12 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm zlib_ROOT: /home/runner/zlib HDF5_ROOT: /home/runner/hdf5 - suites: SCM_RAP - F90FLAGS: -O3 - CC: nvc - FC: nvfortran - CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc - CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran + suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 +# F90FLAGS: -O3 +# CC: nvc +# FC: nvfortran +# CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc +# CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran # Workflow steps steps: From f030d722eb2a346cec52dcef5be4f3d9baedc71e Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:21:19 -0700 Subject: [PATCH 115/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5c14d1371..45ef63bb3 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -274,4 +274,15 @@ jobs: - name: Build SCM run: | cd ${SCM_ROOT}/scm/bin - make -j4 \ No newline at end of file + make -j4 + + - name: Download data for SCM + run: | + cd ${SCM_ROOT} + ./contrib/get_all_static_data.sh + ./contrib/get_thompson_tables.sh + + - name: Run SCM RTs + run: | + cd ${SCM_ROOT}/scm/bin + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 From 442f3546874a6ed73c8cc2f1b395747bf5e1c62f Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:24:01 -0700 Subject: [PATCH 116/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 45ef63bb3..40ea03bfb 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -170,9 +170,7 @@ jobs: wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz tar -zvxf v4.7.4.tar.gz cd netcdf-c-4.7.4 - CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" - LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" - ./configure --prefix=${NETCDF} + CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib" ./configure --prefix=${NETCDF} make make install echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV @@ -191,10 +189,7 @@ jobs: wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz tar -zvxf v4.6.1.tar.gz cd netcdf-fortran-4.6.1 - FCFLAGS="-fPIC" FFLAGS="-fPIC" - CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" - LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" - ./configure --prefix=${NETCDF} + FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I/home/runner/hdf5/include -I/home/runner/zlib/include -I/home/runner/netcdf/include" LDFLAGS="-L/home/runner/hdf5/lib -L/home/runner/zlib/lib -L/home/runner/netcdf/lib" ./configure --prefix=${NETCDF} make make install From b901cd40d8b8207d3ce7628530a0d82a26463e8b Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:42:43 -0700 Subject: [PATCH 117/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 40ea03bfb..74ace14c7 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -25,12 +25,7 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm zlib_ROOT: /home/runner/zlib HDF5_ROOT: /home/runner/hdf5 - suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 -# F90FLAGS: -O3 -# CC: nvc -# FC: nvfortran -# CMAKE_C_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc -# CMAKE_Fortran_COMPILER: /home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran + suites: SCM_RAP,SCM_RRFS_v1beta # Workflow steps steps: From 7970fdc6cee30f73b454e094b96571816053bf59 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Mar 2024 20:58:17 -0700 Subject: [PATCH 118/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 74ace14c7..f58c174e5 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -275,4 +275,4 @@ jobs: - name: Run SCM RTs run: | cd ${SCM_ROOT}/scm/bin - ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v From 09928ca41cabc5532aba2b45fe532696d174372e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 17:40:14 +0000 Subject: [PATCH 119/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index f58c174e5..731f7f072 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release]#, Debug] + build-type: [Release, Debug] enable-gpu-acc: [False]#, True] py-version: [3.7.13]#, 3.9.12] @@ -275,4 +275,4 @@ jobs: - name: Run SCM RTs run: | cd ${SCM_ROOT}/scm/bin - ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases_nvidia.py --runtime_mult 0.1 -v From 5a461157ff391a69b346a5acc66e41266a218da4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 17:59:55 +0000 Subject: [PATCH 120/172] Add file for Nvidia RTS --- test/rt_test_cases_nvidia.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/rt_test_cases_nvidia.py diff --git a/test/rt_test_cases_nvidia.py b/test/rt_test_cases_nvidia.py new file mode 100644 index 000000000..a19c81ba9 --- /dev/null +++ b/test/rt_test_cases_nvidia.py @@ -0,0 +1,9 @@ +run_list = [\ + #---------------------------------------------------------------------------------------------------------------------------------------------- + # CCPP-SCM v6 supported suites + #---------------------------------------------------------------------------------------------------------------------------------------------- + {"case": "arm_sgp_summer_1997_A", "suite": "SCM_RAP"}, \ + {"case": "twpice", "suite": "SCM_RAP"}, \ + {"case": "bomex", "suite": "SCM_RAP"}, \ + {"case": "astex", "suite": "SCM_RAP"}, \ + {"case": "LASSO_2016051812", "suite": "SCM_RAP"}] From f89b1d6f8747dd453ffd8a0336044a3e7311d34d Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:07:20 +0000 Subject: [PATCH 121/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 731f7f072..7fc467a0f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -11,7 +11,7 @@ jobs: matrix: fortran-compiler: [nvfortran] build-type: [Release, Debug] - enable-gpu-acc: [False]#, True] + enable-gpu-acc: [false, true] py-version: [3.7.13]#, 3.9.12] # Environmental variables @@ -248,11 +248,11 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src -# if: contains(matrix.enable-gpu-acc, 'True') -# cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True -# else: -# cmake -DCCPP_SUITES=${suites} ../src +# cmake -DCCPP_SUITES=${suites} ../src + if: matrix.enable-gpu-acc + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True + else: + cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From 51b7baa992ae844a1c2620fbd309babd1e024259 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:08:32 +0000 Subject: [PATCH 122/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 7fc467a0f..8e0c1bded 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -248,9 +248,8 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin -# cmake -DCCPP_SUITES=${suites} ../src - if: matrix.enable-gpu-acc - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True + if: matrix.enable-gpu-acc + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True else: cmake -DCCPP_SUITES=${suites} ../src From d712d560e994c30337540f27bcd9fb3dc3dec983 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:14:44 +0000 Subject: [PATCH 123/172] Revert change to CMakeLists --- scm/src/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 42942492d..d45360a15 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,6 +30,9 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) +if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") + message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) +endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose @@ -126,8 +129,7 @@ set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets") #------------------------------------------------------------------------------ # Set compile options if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") - #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -foffload=nvptx-none -fopt-info -fno-stack-protector -lm -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") From 0b2ee57118ee9c8ef1e747b604ae9546e68ec5aa Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:16:23 +0000 Subject: [PATCH 124/172] Revert change to CMakeLists --- scm/src/CMakeLists.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index d45360a15..705fa2ae1 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -208,15 +208,6 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") endif() - set(MPI_C_COMPILER mpicc) - set(MPI_CXX_COMPILER mpicxx) - set(MPI_Fortran_COMPILER mpif90) - - set(CMAKE_C_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) - set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) - set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) - set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) - else (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") message (FATAL_ERROR "This program has only been compiled with gfortran and ifort. If another compiler is needed, the appropriate flags must be added in ${CMAKE_SOURCE_DIR}/CMakeLists.txt") endif (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") From 16c79388384d23ec917054b2bff32e20d09afc60 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:30:03 +0000 Subject: [PATCH 125/172] Revert change to CMakeLists --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 8e0c1bded..37cd360f0 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -248,10 +248,10 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - if: matrix.enable-gpu-acc - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True + if: contains(matrix.enable-gpu-acc, 'true') + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True else: - cmake -DCCPP_SUITES=${suites} ../src + cmake -DCCPP_SUITES=${suites} ../src - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') From 9fe09eb53c641f6d8ef3b619bba2ed80a883ddef Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:36:27 +0000 Subject: [PATCH 126/172] Revert change to CMakeLists --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 37cd360f0..eb4eed698 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -1,6 +1,6 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] jobs: @@ -122,7 +122,6 @@ jobs: - name: Install zlib env: CFLAGS: -fPIC - if: steps.cache-zlib.outputs.cache-hit != 'true' run: | wget https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz tar -zxvf zlib-1.2.13.tar.gz @@ -136,7 +135,6 @@ jobs: env: CPPFLAGS: -I${zlib_ROOT}/include LDFLAGS: -L${zlib_ROOT}/lib - if: steps.cache-hdf5.outputs.cache-hit != 'true' run: | wget https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_14_1-2.tar.gz tar -zxvf hdf5-1_14_1-2.tar.gz @@ -272,6 +270,7 @@ jobs: ./contrib/get_thompson_tables.sh - name: Run SCM RTs + if: contains(matrix.enable-gpu-acc, 'false') run: | cd ${SCM_ROOT}/scm/bin ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases_nvidia.py --runtime_mult 0.1 -v From 659d780b1bfb0dd8157450eca6926d25478a163a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:44:20 +0000 Subject: [PATCH 127/172] Update CI --- scm/src/CMakeLists.txt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 705fa2ae1..42942492d 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,9 +30,6 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) -if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") - message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) -endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose @@ -129,7 +126,8 @@ set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets") #------------------------------------------------------------------------------ # Set compile options if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -foffload=nvptx-none -fopt-info -fno-stack-protector -lm -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") @@ -208,6 +206,15 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") endif() + set(MPI_C_COMPILER mpicc) + set(MPI_CXX_COMPILER mpicxx) + set(MPI_Fortran_COMPILER mpif90) + + set(CMAKE_C_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + else (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") message (FATAL_ERROR "This program has only been compiled with gfortran and ifort. If another compiler is needed, the appropriate flags must be added in ${CMAKE_SOURCE_DIR}/CMakeLists.txt") endif (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") From 229473c55406bbf001c06f7ff98c1b209392a1cd Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:53:04 +0000 Subject: [PATCH 128/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index eb4eed698..9ace68508 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -11,7 +11,7 @@ jobs: matrix: fortran-compiler: [nvfortran] build-type: [Release, Debug] - enable-gpu-acc: [false, true] + enable-gpu-acc: [False, True] py-version: [3.7.13]#, 3.9.12] # Environmental variables @@ -246,17 +246,14 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - if: contains(matrix.enable-gpu-acc, 'true') - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=True - else: - cmake -DCCPP_SUITES=${suites} ../src + cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} - name: Configure build with CMake (Debug) if: contains(matrix.build-type, 'Debug') run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug ../src + cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src - name: Build SCM run: | From ba034f497f92080333ff3633b7b89fa4777815b6 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 18:54:31 +0000 Subject: [PATCH 129/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9ace68508..c3f88d40c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -1,6 +1,6 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] jobs: From 59b73cdf29f75325a7827dd283466656f1c62515 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 21:04:53 +0000 Subject: [PATCH 130/172] Update CI --- .../workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 14 +++----------- scm/src/CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c3f88d40c..9f33b0803 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -241,19 +241,11 @@ jobs: # Build SCM. ####################################################################################### - - name: Configure build with CMake (Release) - if: contains(matrix.build-type, 'Release') + - name: Configure build with CMake run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} ../src -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} - - - name: Configure build with CMake (Debug) - if: contains(matrix.build-type, 'Debug') - run: | - cd ${SCM_ROOT}/scm - mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=Debug -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src + cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src - name: Build SCM run: | @@ -267,7 +259,7 @@ jobs: ./contrib/get_thompson_tables.sh - name: Run SCM RTs - if: contains(matrix.enable-gpu-acc, 'false') + if: contains(matrix.enable-gpu-acc, 'False') run: | cd ${SCM_ROOT}/scm/bin ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases_nvidia.py --runtime_mult 0.1 -v diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 42942492d..10885cf28 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -186,7 +186,7 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") - if(DEFINED ENABLE_NVIDIA_OPENACC) + if(ENABLE_NVIDIA_OPENACC MATCHES "True") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc -Minfo=accel") else() From c12544ce21dee4fa6e2cd30c2ec43f85fd4d47a4 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 21:18:40 +0000 Subject: [PATCH 131/172] Update CI --- scm/src/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 10885cf28..3cfe66673 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -30,6 +30,9 @@ endif() message (STATUS "Running ccpp_prebuild.py for CCPP") # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics) +if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py") + message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" ) +endif() if (${CMAKE_BUILD_TYPE} MATCHES "Debug") execute_process( COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose @@ -126,8 +129,7 @@ set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets") #------------------------------------------------------------------------------ # Set compile options if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") - #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -foffload=nvptx-none -fopt-info -fno-stack-protector -lm -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") From f82f995044ebcd78b73404c8cdd06b009c6725c2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 21:40:04 +0000 Subject: [PATCH 132/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9f33b0803..19c13ee06 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -1,4 +1,4 @@ -name: CI test to build the CCPP-SCM on ubuntu v22.04 using NVidia compilers +name: CI test to build the CCPP-SCM on ubuntu v22.04 on: [push,pull_request,workflow_dispatch] @@ -10,7 +10,7 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release, Debug] + build-type: [Release] enable-gpu-acc: [False, True] py-version: [3.7.13]#, 3.9.12] @@ -247,7 +247,7 @@ jobs: mkdir bin && cd bin cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src - - name: Build SCM + - name: Build SCM. run: | cd ${SCM_ROOT}/scm/bin make -j4 @@ -258,7 +258,7 @@ jobs: ./contrib/get_all_static_data.sh ./contrib/get_thompson_tables.sh - - name: Run SCM RTs + - name: Run SCM RTs (w/o GPU) if: contains(matrix.enable-gpu-acc, 'False') run: | cd ${SCM_ROOT}/scm/bin From 8d6a1b553a06ffc1c5daf77c9de9bbbfa35d3ef8 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 6 Mar 2024 22:05:23 +0000 Subject: [PATCH 133/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 19c13ee06..bace35b9d 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -12,7 +12,7 @@ jobs: fortran-compiler: [nvfortran] build-type: [Release] enable-gpu-acc: [False, True] - py-version: [3.7.13]#, 3.9.12] + py-version: [3.7.13, 3.9.12] # Environmental variables env: @@ -25,7 +25,7 @@ jobs: SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm zlib_ROOT: /home/runner/zlib HDF5_ROOT: /home/runner/hdf5 - suites: SCM_RAP,SCM_RRFS_v1beta + suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 # Workflow steps steps: From 9521536894f7d407e504672c10fe09a76f258bd5 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 03:17:56 +0000 Subject: [PATCH 134/172] Update CI --- scm/src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 3cfe66673..2ef20bb84 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -188,7 +188,7 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") - if(ENABLE_NVIDIA_OPENACC MATCHES "True") + if(ENABLE_NVIDIA_OPENACC MATCHES "true") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc -Minfo=accel") else() From 999d6cba8677e22036f2b614a1b19044a49e6427 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 03:24:11 +0000 Subject: [PATCH 135/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index bace35b9d..c4e498cf2 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -10,15 +10,13 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release] + build-type: [Release, Debug] enable-gpu-acc: [False, True] py-version: [3.7.13, 3.9.12] # Environmental variables env: - NFHOME: /home/runner/netcdf-fortran NETCDF: /home/runner/netcdf - NFVERSION: v4.6.1 bacio_ROOT: /home/runner/bacio sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc @@ -238,7 +236,7 @@ jobs: echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV ####################################################################################### - # Build SCM. + # Build and run SCM regression tests (ccpp-scm/test/rt_test_cases.py) ####################################################################################### - name: Configure build with CMake @@ -253,6 +251,7 @@ jobs: make -j4 - name: Download data for SCM + if: contains(matrix.enable-gpu-acc, 'True') run: | cd ${SCM_ROOT} ./contrib/get_all_static_data.sh @@ -262,4 +261,4 @@ jobs: if: contains(matrix.enable-gpu-acc, 'False') run: | cd ${SCM_ROOT}/scm/bin - ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases_nvidia.py --runtime_mult 0.1 -v + ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v From 4d71d1ec518eb579813c80d6d0167aff9e1b8736 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 03:41:50 +0000 Subject: [PATCH 136/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index c4e498cf2..5a1c4ebef 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: fortran-compiler: [nvfortran] - build-type: [Release, Debug] + build-type: [Release]#, Debug] enable-gpu-acc: [False, True] py-version: [3.7.13, 3.9.12] From 208bed3dce56555942965adc979c6fd8f7a0206a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 14:21:50 +0000 Subject: [PATCH 137/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 5a1c4ebef..7bfaa4bcf 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -251,7 +251,6 @@ jobs: make -j4 - name: Download data for SCM - if: contains(matrix.enable-gpu-acc, 'True') run: | cd ${SCM_ROOT} ./contrib/get_all_static_data.sh From 7e58c6d1f1161159e60ce2c464af78d10ea3d2a2 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 7 Mar 2024 15:26:32 +0000 Subject: [PATCH 138/172] Update CI --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 7bfaa4bcf..9e998655f 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -24,6 +24,7 @@ jobs: zlib_ROOT: /home/runner/zlib HDF5_ROOT: /home/runner/hdf5 suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0 + suites_ps: SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps # Workflow steps steps: @@ -243,7 +244,7 @@ jobs: run: | cd ${SCM_ROOT}/scm mkdir bin && cd bin - cmake -DCCPP_SUITES=${suites} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src + cmake -DCCPP_SUITES=${suites},${suites_ps} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src - name: Build SCM. run: | From 82dd9ca91dddc246dd5a59556c90fb1a2b0846f5 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 11 Mar 2024 17:54:19 +0000 Subject: [PATCH 139/172] Cleanup/Housekeeping. Remove unnecessary interpolation step --- scm/etc/scripts/UFS_IC_generator.py | 41 ++++------------------------- 1 file changed, 5 insertions(+), 36 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index af3f45d29..cc570d239 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -1917,37 +1917,6 @@ def get_UFS_forcing_data(nlevs, state_IC, location, use_nearest, forcing_dir, gr pres_adv[t+1,:] = pres_adv[t,:] pres_i_adv[t+1,:] = pres_i_adv[t,:] - if save_comp_data: - # - t_layr = np.zeros([n_files+1,nlevs]) - qv_layr = np.zeros([n_files+1,nlevs]) - u_layr = np.zeros([n_files+1,nlevs]) - v_layr = np.zeros([n_files+1,nlevs]) - p_layr = np.zeros([n_files+1,nlevs]) - - # - for t in range(0,n_files): - from_p[0,:] = stateNATIVE["p_lev"][t,::-1] - to_p[0,:] = stateNATIVE["p_lev"][1,::-1] - log_from_p[0,:] = np.log(from_p[0,:]) - log_to_p[0,:] = np.log(to_p[0,:]) - p_layr[t,:] = stateNATIVE["p_lay"][1,::-1] - for k in range(0,nlevs): dp2[0,k] = to_p[0,k+1] - to_p[0,k] - t_layr[t,:] = fv3_remap.map_scalar(nlevs, log_from_p, stateNATIVE["t_lay"][t:t+1,::-1], \ - dummy, nlevs, log_to_p, 0, 0, 1, np.abs(kord_tm), t_min) - qv_layr[t,:] = fv3_remap.map1_q2(nlevs, from_p, stateNATIVE["qv_lay"][t:t+1,::-1], \ - nlevs, to_p, dp2, 0, 0, 0, kord_tr, q_min) - u_layr[t,:] = fv3_remap.map1_ppm(nlevs, from_p, stateNATIVE["u_lay"][t:t+1,::-1], \ - 0.0, nlevs, to_p, 0, 0, -1, kord_tm) - v_layr[t,:] = fv3_remap.map1_ppm(nlevs, from_p, stateNATIVE["v_lay"][t:t+1,::-1], \ - 0.0, nlevs, to_p, 0, 0, -1, kord_tm) - - t_layr[t+1,:] = t_layr[t,:] - qv_layr[t+1,:] = qv_layr[t,:] - u_layr[t+1,:] = u_layr[t,:] - v_layr[t+1,:] = v_layr[t,:] - p_layr[t+1,:] = p_layr[t,:] - #################################################################################### # # if we had atmf,sfcf files at every timestep (and the SCM timestep is made to match @@ -2116,11 +2085,11 @@ def get_UFS_forcing_data(nlevs, state_IC, location, use_nearest, forcing_dir, gr if (save_comp_data): comp_data = { "time": stateNATIVE["time"]*sec_in_hr, - "pa" : p_layr[:,::-1], - "ta" : t_layr[:,::-1], - "qv" : qv_layr[:,::-1], - "ua" : u_layr[:,::-1], - "va" : v_layr[:,::-1], + "pa" : stateNATIVE["p_lay"][:,:], + "ta" : stateNATIVE["t_lay"][:,:], + "qv" : stateNATIVE["qv_lay"][:,:], + "ua" : stateNATIVE["u_lay"][:,:], + "va" : stateNATIVE["v_lay"][:,:], "vars2d":vars2d} else: comp_data = {} From 99f3654a779ccf11a6433e4e42f8088bfb4bc003 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Mon, 11 Mar 2024 14:23:39 -0400 Subject: [PATCH 140/172] update ccpp/physics and .gitmodules --- .gitmodules | 2 +- ccpp/physics | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0e73a7201..791bb283c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ #url = https://github.com/NCAR/ccpp-physics #branch = main url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR120 + branch = ufs-dev-PR139 [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 65a21cefd..d6498f6eb 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 65a21cefd2fded6718fcbd27c84ab89477a46116 +Subproject commit d6498f6eb0b8a969bfe17e138f98c6c3400b68a4 From 688ce5edccdbc8641b960a085f744a047b4a59fe Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Tue, 12 Mar 2024 10:47:29 -0400 Subject: [PATCH 141/172] update ccpp/physics and .gitmodules --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 791bb283c..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR139 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index d6498f6eb..5c2d490b8 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit d6498f6eb0b8a969bfe17e138f98c6c3400b68a4 +Subproject commit 5c2d490b888aecb7b6ac0021007d0ce58a4d1c20 From f0f85df20d8cde7ee727e6432842b6e10d0a2132 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Tue, 12 Mar 2024 12:14:21 -0400 Subject: [PATCH 142/172] update ccpp/physics and .gitmodules --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index df5cb763d..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR135 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 15b1b796b..ba7ed8f17 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 15b1b796bcd0f19d3e6957dbfdd5f369647c526f +Subproject commit ba7ed8f173b6c91d6180be16051ce0d4c54cb2ad From af34c32c5d9979a2995e39e9010620fc0ca0afd0 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Tue, 12 Mar 2024 13:16:26 -0400 Subject: [PATCH 143/172] point .gitmodules back to my branch for CI testing --- .gitmodules | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index dc0798c32..411885554 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,10 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - url = https://github.com/NCAR/ccpp-physics - branch = main + url = https://github.com/grantfirl/ccpp-physics + branch = ufs-dev-PR153 + #url = https://github.com/NCAR/ccpp-physics + #branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules From 35be99b63205ce3f334149d3cea489695e3ccf2d Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Tue, 12 Mar 2024 15:14:06 -0400 Subject: [PATCH 144/172] update ccpp/physics submodule pointer --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index 4f80f1182..48221d257 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 4f80f11820869419382858bed113416e444821e7 +Subproject commit 48221d25709a361da4f10f27f6a70f5689c97ac6 From 4f248e70ddf5a5afe0811df75f5b70c90d18f21d Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 14 Mar 2024 16:00:58 -0400 Subject: [PATCH 145/172] update ccpp/physics and .gitmodules --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 411885554..6e979209b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR153 - #url = https://github.com/NCAR/ccpp-physics - #branch = main + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 48221d257..7b36320b2 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 48221d25709a361da4f10f27f6a70f5689c97ac6 +Subproject commit 7b36320b2f096adb31064959a28e6bcf95e49b60 From ac738720256a44b710d360e6fc9b98a1bad580f4 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Fri, 15 Mar 2024 14:20:23 -0400 Subject: [PATCH 146/172] update ccpp/physics and .gitmodules --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index a8006a363..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR94 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 2800006bb..d46098d71 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 2800006bb74cb1ee972f1cd084b651f3eaee8df4 +Subproject commit d46098d717c6fb4b3e926b8c3794d83e3a1627d9 From 2da256fafded9abf876529e131c54b4101243aed Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Fri, 15 Mar 2024 14:21:33 -0400 Subject: [PATCH 147/172] update .gitmodules for whitespace change removal --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index dc0798c32..6e979209b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,7 +4,7 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - url = https://github.com/NCAR/ccpp-physics + url = https://github.com/NCAR/ccpp-physics branch = main [submodule "CMakeModules"] path = CMakeModules From db1ce6f107d9b55424f1e339bd611f96fbb7d4c5 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Mon, 18 Mar 2024 16:40:15 -0400 Subject: [PATCH 148/172] update ccpp/physics submodule --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index 5356ef23a..72ab0a6a6 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 5356ef23a826ad5235d5616ba616312e56e5303b +Subproject commit 72ab0a6a64d0d4cd56ed8a8f2d94b9bbcec809cc From 7f7c80b361f18f4536430edfbd3840a1d32fe91f Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 21 Mar 2024 19:54:47 +0000 Subject: [PATCH 149/172] Add capability to use input file with longitude and latitude lists --- scm/etc/scripts/UFS_IC_generator.py | 60 ++++++++++++++----- .../scripts/UFS_forcing_ensemble_generator.py | 54 +++++++++++++---- 2 files changed, 88 insertions(+), 26 deletions(-) diff --git a/scm/etc/scripts/UFS_IC_generator.py b/scm/etc/scripts/UFS_IC_generator.py index cc570d239..5734c78f2 100755 --- a/scm/etc/scripts/UFS_IC_generator.py +++ b/scm/etc/scripts/UFS_IC_generator.py @@ -464,18 +464,20 @@ def get_UFS_IC_data(dir, grid_dir, tile, i, j, old_chgres, lam): #returns dictionaries with the data vgrid_data = get_UFS_vgrid_data(grid_dir) #only needed for ak, bk to calculate pressure - state_data = get_UFS_state_data(vgrid_data, dir, tile, i, j, old_chgres, lam) + (state_data, error_msg) = get_UFS_state_data(vgrid_data, dir, tile, i, j, old_chgres, lam) surface_data = get_UFS_surface_data(dir, tile, i, j, old_chgres, lam) oro_data = get_UFS_oro_data(dir, tile, i, j, lam) - return (state_data, surface_data, oro_data) + return (state_data, surface_data, oro_data, error_msg) ######################################################################################## # ######################################################################################## def get_UFS_state_data(vgrid, dir, tile, i, j, old_chgres, lam): """Get the state data for the given tile and indices""" - + + state = {} + error_msg=None if lam: nc_file_data = Dataset('{0}/{1}'.format(dir,'gfs_data.nc')) else: @@ -600,7 +602,10 @@ def get_UFS_state_data(vgrid, dir, tile, i, j, old_chgres, lam): icewat_model_rev[k] = cloud_water - liqwat_model_rev[0,k] (liqwat_model_rev[0,k], dummy_rain, icewat_model_rev[k], dummy_snow) = fv3_remap.mp_auto_conversion(liqwat_model_rev[0,k], icewat_model_rev[k]) - [u_s, u_n, v_w, v_e] = get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam) + [u_s, u_n, v_w, v_e, unknown_grid] = get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam) + if unknown_grid: + error_msg='unknown grid orientation' + return(state,error_msg) #put C/D grid zonal/meridional winds on model pressure levels u_s_model_rev = fv3_remap.mappm(levp_data, pressure_from_data_rev[np.newaxis, :], u_s[np.newaxis, :], nlevs_model, pressure_model_interfaces_rev[np.newaxis, :], 1, 1, -1, 8, ptop_data) @@ -638,7 +643,7 @@ def get_UFS_state_data(vgrid, dir, tile, i, j, old_chgres, lam): "pa_i": pressure_model_interfaces } - return state + return (state,error_msg) ######################################################################################## # @@ -665,6 +670,10 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam raise Exception(message) nc_file_grid = Dataset('{0}/{1}'.format(dir,filename)) + + # Get grid dimension + nz,nx,ny = np.shape(nc_file_data['u_w']) + if (lam): #strip ghost/halo points and return supergrid @@ -696,7 +705,7 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam east_test_point = np.argmax(test_lon_diff) north_test_point = np.argmax(test_lat_diff) - + unknown=False if east_test_point == 0: #longitude increases most along the positive i axis if north_test_point == 2: @@ -775,7 +784,11 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam (ex, ey) = fv3_remap.get_latlon_vector(p3) v_e = nc_file_data['u_w'][:,j,i+1]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j,i+1]*fv3_remap.inner_prod(e1, ey) else: - print('unknown grid orientation') + u_s = np.zeros(nz) + u_n = np.zeros(nz) + v_w = np.zeros(nz) + v_e = np.zeros(nz) + unknown=True elif east_test_point == 1: #longitude increases most along the negative i axis if north_test_point == 2: @@ -853,7 +866,11 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam (ex, ey) = fv3_remap.get_latlon_vector(p3) v_e = nc_file_data['u_w'][:,j,i-1]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j,i-1]*fv3_remap.inner_prod(e1, ey) else: - print('unknown grid orientation') + u_s = np.zeros(nz) + u_n = np.zeros(nz) + v_w = np.zeros(nz) + v_e = np.zeros(nz) + unknown=True elif east_test_point == 2: #longitude increases most along the positive j axis if north_test_point == 0: @@ -929,9 +946,16 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam p3 = fv3_remap.mid_pt_sphere(p1*deg_to_rad, p2*deg_to_rad) e1 = fv3_remap.get_unit_vect2(p1*deg_to_rad, p2*deg_to_rad) (ex, ey) = fv3_remap.get_latlon_vector(p3) - v_e = nc_file_data['u_w'][:,j+1,i]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j+1,i]*fv3_remap.inner_prod(e1, ey) + if (j < nx -1): + v_e = nc_file_data['u_w'][:,j+1,i]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j+1,i]*fv3_remap.inner_prod(e1, ey) + else: + v_e = nc_file_data['v_w'][:,j,i] else: - print('unknown grid orientation') + u_s = np.zeros(nz) + u_n = np.zeros(nz) + v_w = np.zeros(nz) + v_e = np.zeros(nz) + unknown=True elif east_test_point == 3: #longitude increases most along the negative j axis if north_test_point == 0: @@ -1009,12 +1033,15 @@ def get_zonal_and_meridional_winds_on_cd_grid(tile, dir, i, j, nc_file_data, lam (ex, ey) = fv3_remap.get_latlon_vector(p3) v_e = nc_file_data['u_w'][:,j-1,i]*fv3_remap.inner_prod(e1, ex) + nc_file_data['v_w'][:,j-1,i]*fv3_remap.inner_prod(e1, ey) else: - print('unknown grid orientation') - + u_s = np.zeros(nz) + u_n = np.zeros(nz) + v_w = np.zeros(nz) + v_e = np.zeros(nz) + unknown=True nc_file_grid.close() - return [u_s, u_n, v_w, v_e] + return [u_s, u_n, v_w, v_e, unknown] ######################################################################################## # @@ -2685,8 +2712,11 @@ def main(): # get UFS IC data (TODO: flag to read in RESTART data rather than IC data and implement # different file reads) - (state_data, surface_data, oro_data) = get_UFS_IC_data(in_dir, grid_dir, tile, tile_i,\ - tile_j, old_chgres, lam) + (state_data, surface_data, oro_data, error_msg) = get_UFS_IC_data(in_dir, grid_dir, tile, tile_i,\ + tile_j, old_chgres, lam) + if (error_msg): + print("ERROR: unknown grid orintation") + exit() if not date: # date was not included on command line; look in atmf* file for initial date diff --git a/scm/etc/scripts/UFS_forcing_ensemble_generator.py b/scm/etc/scripts/UFS_forcing_ensemble_generator.py index d057dc6cf..23b90b499 100755 --- a/scm/etc/scripts/UFS_forcing_ensemble_generator.py +++ b/scm/etc/scripts/UFS_forcing_ensemble_generator.py @@ -20,6 +20,7 @@ parser.add_argument('-latl', '--lat_limits', help='latitude range, separated by a space', nargs=2, type=float, required=False) parser.add_argument('-lons', '--lon_list', help='longitudes, separated by a space', nargs='*', type=float, required=False) parser.add_argument('-lats', '--lat_list', help='latitudes, separated by a space', nargs='*', type=float, required=False) +parser.add_argument('-fxy', '--lonlat_file', help='file containing longitudes and latitude',nargs=1, required=False) parser.add_argument('-nens', '--nensmembers', help='number of SCM UFS ensemble memebers to create', type=int, required=False) parser.add_argument('-dt', '--timestep', help='SCM timestep, in seconds', type=int, default = 3600) parser.add_argument('-cres', '--C_RES', help='UFS spatial resolution', type=int, default = 96) @@ -43,25 +44,29 @@ def main(): if (args.lon_limits and args.lon_list): print("ERROR: Can't provide explicit longitude(s) AND a longitude range") exit() + # end if if (args.lat_limits and args.lat_list): print("ERROR: Can't provide explicit latitude(s) AND a latitude range") exit() + # end if if (args.lon_limits or args.lat_limits) and not args.nensmembers: print("ERROR: Longitude/Latitude range provided, but NOT ensemble count.") exit() - + # end if if (args.nensmembers): npts = args.nensmembers if (args.lat_list or args.lon_list): print("ERROR: Can't provide explicit lon/lat range AND number of points for ensemble generation.") exit() - else: - if (args.lon_list and args.lat_list): - if (len(args.lon_list) == len(args.lat_list)): - npts = len(args.lon_list) - else: - print("ERROR: Number of longitude/latitudes are inconsistent") - exit() + # end if + elif (args.lon_list and args.lat_list): + if (len(args.lon_list) == len(args.lat_list)): + npts = len(args.lon_list) + else: + print("ERROR: Number of longitude/latitudes are inconsistent") + exit() + # end if + # end if ########################################################################### # @@ -83,19 +88,44 @@ def main(): lats[ipt] = args.lat_limits[0] + (args.lat_limits[1]-args.lat_limits[0])*rng1[ipt] else: lats[ipt] = rng1[ipt]*180-90 + # end if if args.lon_limits: lons[ipt] = args.lon_limits[0] + (args.lon_limits[1]-args.lon_limits[0])*rng2[ipt] else: lons[ipt] = rng2[ipt]*360 + # end if + # end for ########################################################################### # - # Use longitude and latitude provided + # Use longitude and latitude provided to command line # ########################################################################### - else: + elif (args.lon_list and args.lat_list): lons = np.asarray(args.lon_list) lats = np.asarray(args.lat_list) - + ########################################################################### + # + # Use longitude and latitude from input file + # + ########################################################################### + elif (args.lonlat_file): + fid = open(args.lonlat_file[0], 'r') + lines = fid.read().split('\n') + lon_list = lines[0] + lons = eval(lon_list) + lat_list = lines[1] + lats = eval(lat_list) + lons = lons[563::] + lats = lats[563::] + npts = len(lons) + else: + print("ERROR: Must provide input points in one of the following formats:") + print(" Using -nens [] -lonl [] -latl [] (e.g. -nens 20 -lonl 30 40 -latl 30 35)") + print(" Using -lons [] -lats [] (e.g. -lons 203 204 205 -lats 30 30 30)") + print(" Using -fxy (e.g. -fxy lonlat.txt w/ -lons [] -lats [])") + exit() + # end if + ########################################################################### # # Create SCM case configuration (etc/case_config) file. @@ -154,6 +184,7 @@ def main(): # count = count + 1 + # end for ########################################################################### # @@ -171,6 +202,7 @@ def main(): #print(' {"case": "' , run["case"] , '", "suite": "' , run["suite"] , '"},') fileID.write(' {"case": "' + run["case"] + '", "suite": "' + run["suite"] + '"},') fileID.write('\n') + # end for fileID.write(' ]') fileID.close() From 650e207aa244b1c46b4e7c1e7fc2ccb8a44e4733 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 21 Mar 2024 20:02:35 +0000 Subject: [PATCH 150/172] Omission from previous commit --- scm/etc/scripts/UFS_forcing_ensemble_generator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scm/etc/scripts/UFS_forcing_ensemble_generator.py b/scm/etc/scripts/UFS_forcing_ensemble_generator.py index 23b90b499..cb7f8e426 100755 --- a/scm/etc/scripts/UFS_forcing_ensemble_generator.py +++ b/scm/etc/scripts/UFS_forcing_ensemble_generator.py @@ -115,8 +115,6 @@ def main(): lons = eval(lon_list) lat_list = lines[1] lats = eval(lat_list) - lons = lons[563::] - lats = lats[563::] npts = len(lons) else: print("ERROR: Must provide input points in one of the following formats:") From dac377e3628b3632d27d3574718fee8fc56ac17a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 21 Mar 2024 20:42:19 +0000 Subject: [PATCH 151/172] Added utility script to create input file containing replay points --- scm/etc/scripts/create_lonlat4replay.py | 83 +++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 scm/etc/scripts/create_lonlat4replay.py diff --git a/scm/etc/scripts/create_lonlat4replay.py b/scm/etc/scripts/create_lonlat4replay.py new file mode 100755 index 000000000..844095fcc --- /dev/null +++ b/scm/etc/scripts/create_lonlat4replay.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +################################################################################################# +# Dependencies +################################################################################################# +import os +import numpy as np +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-d', '--dir', help='path to UFS Regression Test output', required=True) +parser.add_argument('-n', '--case_name', help='name of case', required=True) +parser.add_argument('-lonl', '--lon_range', help='longitude range, separated by a space', nargs=2, type=int, required=True) +parser.add_argument('-latl', '--lat_range', help='latitude range, separated by a space', nargs=2, type=int, required=True) +parser.add_argument('-dlat', '--dlat', help='latitude spacing', type=int, required=True) +parser.add_argument('-dlon', '--dlon', help='longitude spacing', type=int, required=True) +parser.add_argument('-dt', '--timestep', help='SCM timestep, in seconds', type=int, required=True) +parser.add_argument('-cres', '--C_RES', help='UFS spatial resolution', type=int, required=True) +parser.add_argument('-sdf', '--suite', help='CCPP suite definition file to use for ensemble', required=True) +parser.add_argument('-fout', '--fileOUT', help='Text file containing lon/lat points', required=False, default='lonlats.txt') + +def main(): + + # Get command line arguments + args = parser.parse_args() + dir = args.dir+'/'+args.case_name + + com_pre = './UFS_forcing_ensemble_generator.py -d '+dir+' -sc --C_RES '+str(args.C_RES)+' -dt '+str(args.timestep)+' -n '+args.case_name+' -sdf '+args.suite + + lons = [] + lats = [] + count = 0 + for lon in range(args.lon_range[0],args.lon_range[1],args.dlon): + for lat in range(args.lat_range[0],args.lat_range[1],args.dlat): + count = count + 1 + lons.append(lon) + lats.append(lat) + # end for + # end for + + comA = '-lons ' + for ij in lons: + comA = comA + str(ij)+' ' + # end for + comB = '-lats ' + for ij in lats: + comB = comB + str(ij)+' ' + # end for + com = com_pre+comA+comB + print(com) + print("#######################################################################################") + + # For python list input file + comA = '' + countA = 0 + for ij in lons: + countA = countA + 1 + if (countA < count): + comA = comA + str(ij)+', ' + else: + comA = comA + str(ij) + #end if + # end for + + comB = '' + countB = 0 + for ij in lats: + countB = countB + 1 + if (countB < count): + comB = comB + str(ij)+', ' + else: + comB = comB + str(ij) + # end if + # end for + + fileID = open(args.fileOUT,'w') + fileID.write(comA+"\n") + fileID.write(comB+"\n") + fileID.close() + com = com_pre+' -fxy '+args.fileOUT + print(com) + +if __name__ == '__main__': + main() From b93bc5110df919812dcd9aa20d9260c316811974 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 21 Mar 2024 21:04:23 +0000 Subject: [PATCH 152/172] Added step to skip gridpoint in ensemble if error during case generation --- .../scripts/UFS_forcing_ensemble_generator.py | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/scm/etc/scripts/UFS_forcing_ensemble_generator.py b/scm/etc/scripts/UFS_forcing_ensemble_generator.py index cb7f8e426..c4a533127 100755 --- a/scm/etc/scripts/UFS_forcing_ensemble_generator.py +++ b/scm/etc/scripts/UFS_forcing_ensemble_generator.py @@ -153,35 +153,37 @@ def main(): print(com) os.system(com) - # Add case to ensemble list. - case_list = case_list + '"'+case_name+'"' - if (count != npts-1): case_list = case_list + ', ' + if (os.path.isfile(file_scminput)): + # Add case to ensemble list. + case_list = case_list + '"'+case_name+'"' + if (count != npts-1): case_list = case_list + ', ' - # What is the surface type? (get from SCM input file) - dataset = xr.open_dataset(file_scminput) - sfc_type = int(np.round_(dataset.slmsk.values[0])) + # What is the surface type? (get from SCM input file) + dataset = xr.open_dataset(file_scminput) + sfc_type = int(np.round_(dataset.slmsk.values[0])) - # Create case_config file(s) - fileOUT = "../../etc/case_config/"+case_name+".nml" - fileID = open(fileOUT, 'w') - fileID.write('$case_config') - fileID.write('\n') - fileID.write('case_name = ' + "'" + case_name + "',") - fileID.write('\n') - fileID.write('sfc_type = ' + str(sfc_type) + ",") - fileID.write('\n') - for opts in case_config: - fileID.write(opts["name"] + ' = ' + opts["values"] + ",") + # Create case_config file(s) + fileOUT = "../../etc/case_config/"+case_name+".nml" + fileID = open(fileOUT, 'w') + fileID.write('$case_config') fileID.write('\n') - fileID.write('$end') - fileID.write('\n') - fileID.close() + fileID.write('case_name = ' + "'" + case_name + "',") + fileID.write('\n') + fileID.write('sfc_type = ' + str(sfc_type) + ",") + fileID.write('\n') + for opts in case_config: + fileID.write(opts["name"] + ' = ' + opts["values"] + ",") + fileID.write('\n') + fileID.write('$end') + fileID.write('\n') + fileID.close() - # Add case to dictionary to be used by run_scm.py - run_list.append({"case": case_name, "suite": args.suite}) + # Add case to dictionary to be used by run_scm.py + run_list.append({"case": case_name, "suite": args.suite}) - # - count = count + 1 + # + count = count + 1 + # end if # end for ########################################################################### From e1619203bc9eb3f8dc07563f19e38fa9d06e3d7f Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Wed, 27 Mar 2024 20:15:57 -0400 Subject: [PATCH 153/172] update ccpp/physics and .gitmodules --- .gitmodules | 6 ++---- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4ded4eb57..dc0798c32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = main [submodule "ccpp-physics"] path = ccpp/physics - #url = https://github.com/NCAR/ccpp-physics - #branch = main - url = https://github.com/grantfirl/ccpp-physics - branch = ufs-dev-PR155 + url = https://github.com/NCAR/ccpp-physics + branch = main [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 72ab0a6a6..6b0599ef6 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 72ab0a6a64d0d4cd56ed8a8f2d94b9bbcec809cc +Subproject commit 6b0599ef6c7ea076336b1f073899c5b97e37d584 From 46ecc1882b5247c37abaccdef9e5e4fe332758e9 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 28 Mar 2024 15:26:05 -0400 Subject: [PATCH 154/172] update CCPP_typedefs to work with PR#181 --- ccpp/physics | 2 +- scm/src/CCPP_typedefs.F90 | 2 +- scm/src/CCPP_typedefs.meta | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ccpp/physics b/ccpp/physics index 6b0599ef6..4d497be1c 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 6b0599ef6c7ea076336b1f073899c5b97e37d584 +Subproject commit 4d497be1c5c5dddee9afcf8662ac30e005062b8e diff --git a/scm/src/CCPP_typedefs.F90 b/scm/src/CCPP_typedefs.F90 index 878422d39..0b49ed010 100644 --- a/scm/src/CCPP_typedefs.F90 +++ b/scm/src/CCPP_typedefs.F90 @@ -622,7 +622,7 @@ subroutine gfs_interstitial_create (Interstitial, IM, Model) allocate (Interstitial%sigma (IM)) allocate (Interstitial%sigmaf (IM)) allocate (Interstitial%sigmafrac (IM,Model%levs)) - allocate (Interstitial%sigmatot (IM,Model%levs)) + allocate (Interstitial%sigmatot (IM,Model%levs+1)) allocate (Interstitial%snowc (IM)) allocate (Interstitial%snohf (IM)) allocate (Interstitial%snowmt (IM)) diff --git a/scm/src/CCPP_typedefs.meta b/scm/src/CCPP_typedefs.meta index 8ab9f4ba2..8208bd1d5 100644 --- a/scm/src/CCPP_typedefs.meta +++ b/scm/src/CCPP_typedefs.meta @@ -1952,7 +1952,7 @@ standard_name = convective_updraft_area_fraction_at_model_interfaces long_name = convective updraft area fraction at model interfaces units = frac - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_loop_extent,vertical_interface_dimension) type = real kind = kind_phys [skip_macro] From efa103fc05fe9f42dbad6aea8d809be127119028 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Thu, 28 Mar 2024 15:37:37 -0400 Subject: [PATCH 155/172] update ccpp/framework --- ccpp/framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/framework b/ccpp/framework index 219f2e9c8..f0b9a18b0 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit 219f2e9c88b7b774becac2bd1453696e105af1c4 +Subproject commit f0b9a18b005d950cb9b0038fbc827b6b37500f43 From adf7b681a9dbb51e5a54697b9105ea06cf2b4652 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Fri, 29 Mar 2024 13:53:18 -0400 Subject: [PATCH 156/172] update ccpp/physics submodule --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index 4d497be1c..212f3ff2f 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 4d497be1c5c5dddee9afcf8662ac30e005062b8e +Subproject commit 212f3ff2f909a43597ce4c9063b09b4e1f2b3e3d From 9e3663a10bbce810f37197e3ad67d3520f1d9852 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 3 Apr 2024 17:32:07 +0000 Subject: [PATCH 157/172] Synced physics --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index aabe63844..69917a6e2 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit aabe63844ae4a81cc20d1937d737c078380a0392 +Subproject commit 69917a6e274f8d4f4b264b2dee58b721088a9918 From be44954e8228d5bbc9a257eb4c4b68b37372a444 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Wed, 3 Apr 2024 15:11:35 -0600 Subject: [PATCH 158/172] Moving documentation to ReadTheDocs (#438) This PR removes the old, LaTeX-based users guide in favor of RST files that are buildable with Sphinx. These new docs have been configured to automatically build with each commit to main on ReadTheDocs, and can be viewed here: https://ccpp-scm.readthedocs.io/en/latest/ --- .readthedocs.yaml | 33 + LICENSE | 4 +- README.md | 2 +- scm/doc/README.md | 17 + scm/doc/TechGuide/Makefile | 13 +- scm/doc/TechGuide/acknow.rst | 12 + scm/doc/TechGuide/acknow.tex | 19 - scm/doc/TechGuide/chap_cases.rst | 611 +++++++++++++++++++ scm/doc/TechGuide/chap_cases.tex | 364 ----------- scm/doc/TechGuide/chap_ccpp.rst | 383 ++++++++++++ scm/doc/TechGuide/chap_ccpp.tex | 160 ----- scm/doc/TechGuide/chap_function.rst | 157 +++++ scm/doc/TechGuide/chap_function.tex | 66 -- scm/doc/TechGuide/chap_hsd.rst | 351 +++++++++++ scm/doc/TechGuide/chap_hsd.tex | 213 ------- scm/doc/TechGuide/chap_intro.rst | 116 ++++ scm/doc/TechGuide/chap_intro.tex | 46 -- scm/doc/TechGuide/chap_quick.rst | 902 ++++++++++++++++++++++++++++ scm/doc/TechGuide/chap_quick.tex | 481 --------------- scm/doc/TechGuide/chap_repo.rst | 62 ++ scm/doc/TechGuide/chap_repo.tex | 46 -- scm/doc/TechGuide/conf.py | 211 +++++++ scm/doc/TechGuide/index.rst | 18 + scm/doc/TechGuide/main.pdf | Bin 813248 -> 0 bytes scm/doc/TechGuide/main.tex | 36 -- scm/doc/TechGuide/preface.rst | 34 ++ scm/doc/TechGuide/preface.tex | 28 - scm/doc/TechGuide/requirements.txt | 1 + scm/doc/TechGuide/title.tex | 35 -- 29 files changed, 2919 insertions(+), 1502 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 scm/doc/README.md create mode 100644 scm/doc/TechGuide/acknow.rst delete mode 100644 scm/doc/TechGuide/acknow.tex create mode 100644 scm/doc/TechGuide/chap_cases.rst delete mode 100644 scm/doc/TechGuide/chap_cases.tex create mode 100644 scm/doc/TechGuide/chap_ccpp.rst delete mode 100644 scm/doc/TechGuide/chap_ccpp.tex create mode 100644 scm/doc/TechGuide/chap_function.rst delete mode 100644 scm/doc/TechGuide/chap_function.tex create mode 100644 scm/doc/TechGuide/chap_hsd.rst delete mode 100644 scm/doc/TechGuide/chap_hsd.tex create mode 100644 scm/doc/TechGuide/chap_intro.rst delete mode 100644 scm/doc/TechGuide/chap_intro.tex create mode 100644 scm/doc/TechGuide/chap_quick.rst delete mode 100644 scm/doc/TechGuide/chap_quick.tex create mode 100644 scm/doc/TechGuide/chap_repo.rst delete mode 100644 scm/doc/TechGuide/chap_repo.tex create mode 100644 scm/doc/TechGuide/conf.py create mode 100644 scm/doc/TechGuide/index.rst delete mode 100644 scm/doc/TechGuide/main.pdf delete mode 100644 scm/doc/TechGuide/main.tex create mode 100644 scm/doc/TechGuide/preface.rst delete mode 100644 scm/doc/TechGuide/preface.tex create mode 100644 scm/doc/TechGuide/requirements.txt delete mode 100644 scm/doc/TechGuide/title.tex diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..95693767e --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,33 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: scm/doc/TechGuide/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - htmlzip + - pdf + - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: scm/doc/TechGuide/requirements.txt diff --git a/LICENSE b/LICENSE index 5f497a483..8e61668f4 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2022, NOAA, UCAR/NCAR, CU/CIRES, CSU/CIRA +Copyright 2024, NOAA, UCAR/NCAR, CU/CIRES, CSU/CIRA Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -10,4 +10,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. diff --git a/README.md b/README.md index 1649d16b4..0db822ec2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # User's Guide -Please find the up-to-date User's Guide located in this repository in [`scm/doc/TechGuide/main.pdf`](https://github.com/NCAR/ccpp-scm/blob/main/scm/doc/TechGuide/main.pdf) \ No newline at end of file +Please find the up-to-date User's Guide located on ReadTheDocs: https://ccpp-scm.readthedocs.io/en/latest/ diff --git a/scm/doc/README.md b/scm/doc/README.md new file mode 100644 index 000000000..fc82b5461 --- /dev/null +++ b/scm/doc/README.md @@ -0,0 +1,17 @@ +# SCM documentation + +The SCM technical documentation/users' guide is written in [reStructured Text markup language (RST)](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html). It is designed to be built with the [Sphinx documentation tool](https://www.sphinx-doc.org/en/master/index.html), and is documentation from the `main` branch is configured to build automatically on ReadTheDocs: https://ccpp-scm.readthedocs.io/en/latest/ + +## Building docs locally + +To build the HTML documentation locally (for example, when testing modifications to the documentation), you must have Sphinx [installed on your machine](https://www.sphinx-doc.org/en/master/usage/installation.html), as well as the required python packages listed in the `requirements.txt` file. Once the prerequisite software is installed, you can build the HTML documentation using the following commands: + + python -m sphinx -T -b html -d _build/doctrees -D language=en . html + +To build a PDF locally, in addition to Sphinx you must have Perl as well as the Perl-based [`latexmk` utility](https://mg.readthedocs.io/latexmk.html#installation). You can then build a PDF using the following commands: + + python -m sphinx -T -b latex -d _build/doctrees -D language=en . latex + latexmk -f -pdf -pdflatex="pdflatex" -use-make index.tex + +In lieu of running the above commands manually, you can also use `make`. The command `make all` will build both HTML and PDF documentation, while `make html` and `make latex` will build HTML and PDF respectively. + diff --git a/scm/doc/TechGuide/Makefile b/scm/doc/TechGuide/Makefile index 501e745a6..6d48b2a61 100644 --- a/scm/doc/TechGuide/Makefile +++ b/scm/doc/TechGuide/Makefile @@ -2,13 +2,16 @@ # Makefile for the CCPP (SCM) Users Guide # -.PHONY: main.pdf all clean +.PHONY: html latex all clean -all: main.pdf +all: html latex -main.pdf: main.tex - latexmk -f -pdf -pdflatex="pdflatex" -use-make main.tex - open main.pdf +latex: + python -m sphinx -T -b latex -d _build/doctrees -D language=en . latex + cd latex && latexmk -f -pdf -pdflatex="pdflatex" -use-make index.tex + +html: + python -m sphinx -T -b html -d _build/doctrees -D language=en . html clean: latexmk -CA diff --git a/scm/doc/TechGuide/acknow.rst b/scm/doc/TechGuide/acknow.rst new file mode 100644 index 000000000..c8014aba5 --- /dev/null +++ b/scm/doc/TechGuide/acknow.rst @@ -0,0 +1,12 @@ +Acknowledgement +=============== +If significant help was provided via the helpdesk, email, or +support forum for work resulting in a publication, please +acknowledge the Developmental Testbed Center team. + +For referencing this document please use: + + Firl, G., D. Swales, L. Carson, L. Bernardet, D. Heinzeller, M. Harrold, T. Hertneky, and + M. Kavulich, 2024. Common Community Physics Package Single Column Model v7.0.0 User and + Technical Guide. Available at https://ccpp-scm.readthedocs.io/en/latest/. + diff --git a/scm/doc/TechGuide/acknow.tex b/scm/doc/TechGuide/acknow.tex deleted file mode 100644 index f089cb91d..000000000 --- a/scm/doc/TechGuide/acknow.tex +++ /dev/null @@ -1,19 +0,0 @@ -\begin{titlepage} -%\BgThispage -%\newgeometry{left=1cm,right=4cm} -\vspace*{0.5cm} -\noindent - -\begin{flushleft} -\textcolor{darkgray}{\LARGE Acknowledgement} -\vspace*{1cm}\par - -If significant help was provided via the helpdesk, email, or support forum for work resulting in a publication, please acknowledge the Developmental Testbed Center team.\\ -\vspace*{1cm}\par -For referencing this document please use:\\ -\vspace*{1cm}\par -Firl, G., D. Swales, L. Carson, L. Bernardet, D. Heinzeller, and M. Harrold, 2022. Common Community Physics Package Single Column Model v6.0.0 User and Technical Guide. 40pp. Available at https://dtcenter.org/sites/default/files/paragraph/scm-ccpp-guide-v6-0-0.pdf. - -\end{flushleft} -\end{titlepage} -\pagebreak{} diff --git a/scm/doc/TechGuide/chap_cases.rst b/scm/doc/TechGuide/chap_cases.rst new file mode 100644 index 000000000..b6bd183dd --- /dev/null +++ b/scm/doc/TechGuide/chap_cases.rst @@ -0,0 +1,611 @@ +Cases +===== + +How to run cases +---------------- + +Only two files are needed to set up and run a case with the SCM. The +first is a configuration namelist file found in ``ccpp-scm/scm/etc/case_config`` that contains parameters +for the SCM infrastructure. The second necessary file is a NetCDF file +containing data to initialize the column state and time-dependent data +to force the column state. The two files are described below. + +.. _`case config`: + +Case configuration namelist parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``case_config`` namelist expects the following parameters: + +- ``case_name`` + + - Identifier for which dataset (initialization and forcing) to load. + This string must correspond to a dataset included in the directory + ``ccpp-scm/scm/data/processed_case_input/`` (without the file extension). + +- ``runtime`` + + - Specify the model runtime in seconds (integer). This should + correspond with the forcing dataset used. If a runtime is + specified that is longer than the supplied forcing, the forcing is + held constant at the last specified values. + +- ``thermo_forcing_type`` + + - An integer representing how forcing for temperature and moisture + state variables is applied (1 :math:`=` total advective + tendencies, 2 :math:`=` horizontal advective tendencies with + prescribed vertical motion, 3 :math:`=` relaxation to observed + profiles with vertical motion prescribed) + +- ``mom_forcing_type`` + + - An integer representing how forcing for horizontal momentum state + variables is applied (1 :math:`=` total advective tendencies; not + implemented yet, 2 :math:`=` horizontal advective tendencies with + prescribed vertical motion, 3 :math:`=` relaxation to observed + profiles with vertical motion prescribed) + +- ``relax_time`` + + - A floating point number representing the timescale in seconds for + the relaxation forcing (only used if ``thermo_forcing_type = 3`` or ``mom_forcing_type = 3``) + +- ``sfc_flux_spec`` + + - A boolean set to ``.true.`` if surface flux are specified from the forcing + data (there is no need to have surface schemes in a suite + definition file if so) + +- ``sfc_roughness_length_cm`` + + - Surface roughness length in cm for calculating surface-related + fields from specified surface fluxes (only used if ``sfc_flux_spec`` is True). + +- ``sfc_type`` + + - An integer representing the character of the surface (0 :math:`=` + sea surface, 1 :math:`=` land surface, 2 :math:`=` sea-ice + surface) + +- ``reference_profile_choice`` + + - An integer representing the choice of reference profile to use + above the supplied initialization and forcing data (1 :math:`=` + “McClatchey” profile, 2 :math:`=` mid-latitude summer standard + atmosphere) + +- ``year`` + + - An integer representing the year of the initialization time + +- ``month`` + + - An integer representing the month of the initialization time + +- ``day`` + + - An integer representing the day of the initialization time + +- ``hour`` + + - An integer representing the hour of the initialization time + +- ``column_area`` + + - A list of floating point values representing the characteristic + horizontal domain area of each atmospheric column in square meters + (this could be analogous to a 3D model’s horizontal grid size or + the characteristic horizontal scale of an observation array; these + values are used in scale-aware schemes; if using multiple columns, + you may specify an equal number of column areas) + +- ``model_ics`` + + - A boolean set to ``.true.`` if UFS atmosphere initial conditions are used + rather than field campaign-based initial conditions + +- ``C_RES`` + + - An integer representing the grid size of the UFS atmosphere + initial conditions; the integer represents the number of grid + points in each horizontal direction of each cube tile + +- ``input_type`` + + - 0 => original DTC format, 1 => DEPHY-SCM format. + +Optional variables (that may be overridden via run script command line +arguments) are: + +- ``vert_coord_file`` + + - File containing FV3 vertical grid coefficients. + +- ``n_levels`` + + - Specify the integer number of vertical levels. + +.. _`case input`: + +Case input data file (CCPP-SCM format) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The initialization and forcing data for each case is stored in a NetCDF +(version 4) file within the ``ccpp-scm/scm/data/processed_case_input`` directory. Each file has at least two +dimensions (``time`` and ``levels``, potentially with additions for vertical snow and soil +levels) and is organized into 3 groups: scalars, initial, and forcing. +Not all fields are required for all cases. For example the fields ``sh_flux_sfc`` and ``lh_flux_sfc`` +are only needed if the variable ``sfc_flx_spec = .true.`` in the case configuration file +and state nudging variables are only required if ``thermo_forcing_type = 3`` or ``mom_forcing_type = 3`` +. Using an active LSM (Noah, NoahMP, RUC) requires many more variables +than are listed here. Example files for using with Noah and NoahMP LSMs +are included in ``ccpp-scm/scm/data/processed_case_input/fv3_model_point_noah[mp].nc``. + +.. _`case input arm`: +.. literalinclude:: arm_case_header.txt + :name: lst_case_input_netcdf_header_arm + :caption: example NetCDF file (CCPP-SCM format) header for case initialization and forcing data + +Case input data file (DEPHY format) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Development and Evaluation of Physics in atmospheric models (DEPHY) +format is an internationally-adopted data format intended for use by SCM +and LESs. The initialization and forcing data for each case is stored in +a NetCDF (version 4) file, although these files are not by default +included in the CCPP SCM repository. To access these cases you need to +clone the DEPHY-SCM repository, and provide the DEPHY-SCM file location +to the SCM. For example: + +.. code:: bash + + cd [...]/ccpp-scm/scm/data + git clone https://github.com/GdR-DEPHY/DEPHY-SCM DEPHY-SCM + cd [...]/ccpp-scm/scm/bin + ./run_scm.py -c MAGIC_LEG04A --case_data_dir [...]/ccpp-scm/scm/data/DEPHY-SCM/MAGIC/LEG04A -v + +Each DEPHY file has three dimensions (``time``, ``t0``, ``levels``) and contains the initial +conditions (``t0``, ``levels``) and forcing data (``time``, ``levels``). Just as when using the CCPP-SCM +formatted inputs, :numref:`Subsection %s `, not all fields +are required for all cases. More information on the DEPHY format +requirements can be found at +`DEPHY `__. + +.. _`case input dephy`: +.. literalinclude:: dephy_case_header.txt + :name: lst_case_input_netcdf_header_dephy + :caption: example NetCDF file (DEPHY format) header for case initialization and forcing data + +Included Cases +-------------- + +Several cases are included in the repository to serve as examples for +users to create their own and for basic research. All case configuration +namelist files for included cases can be found in ``ccpp-scm/scm/etc/case_config`` and represent the +following observational field campaigns: + +- Tropical Warm Pool – International Cloud Experiment (TWP-ICE) + maritime deep convection + +- Atmospheric Radiation Measurement (ARM) Southern Great Plains (SGP) + Summer 1997 continental deep convection + +- Atlantic Stratocumulus Transition EXperiment (ASTEX) maritime + stratocumulus-to-cumulus transition + +- Barbados Oceanographic and Meteorological EXperiment (BOMEX) maritime + shallow convection + +- Large eddy simulation ARM Symbiotic Simulation and Observation + (LASSO) for May 18, 2016 (with capability to run all LASSO dates - + see :numref:`Section %s `) continental shallow convection + +For the ARM SGP case, several case configuration files representing +different time periods of the observational dataset are included, +denoted by a trailing letter. The LASSO case may be run with different +forcing applied, so three case configuration files corresponding to +these different forcing are included. In addition, two example cases are +included for using UFS Atmosphere initial conditions: + +- UFS initial conditions for 38.1 N, 98.5 W (central Kansas) for 00Z on + Oct. 3, 2016 with Noah variables on the C96 FV3 grid (``fv3_model_point_noah.nc``) + +- UFS initial conditions for 38.1 N, 98.5 W (central Kansas) for 00Z on + Oct. 3, 2016 with NoahMP variables on the C96 FV3 grid (``fv3_model_point_noahmp.nc``) + +See :numref:`Section %s ` for information on how to generate these +files for other locations and dates, given appropriate UFS Atmosphere +initial conditions and output. + +How to set up new cases +----------------------- + +Setting up a new case involves preparing the two types of files listed +above. For the case initialization and forcing data file, this typically +involves writing a custom script or program to parse the data from its +original format to the format that the SCM expects, listed above. An +example of this type of script written in Python is included in ``ccpp-scm/scm/etc/scripts/twpice_forcing_file_generator.py``. The +script reads in the data as supplied from its source, converts any +necessary variables, and writes a NetCDF (version 4) file in the format +described in subsections :numref:`Subsection %s ` and +:numref:`Subsection %s `. For reference, the following +formulas are used: + +.. math:: \theta_{il} = \theta - \frac{\theta}{T}\left(\frac{L_v}{c_p}q_l + \frac{L_s}{c_p}q_i\right) + +.. math:: q_t = q_v + q_l + q_i + +where :math:`\theta_{il}` is the ice-liquid water potential temperature, +:math:`\theta` is the potential temperature, :math:`L_v` is the latent +heat of vaporization, :math:`L_s` is the latent heat of sublimation +:math:`c_p` is the specific heat capacity of air at constant pressure, +:math:`T` is absolute temperature, :math:`q_t` is the total water +specific humidity, :math:`q_v` is the water vapor specific humidity, +:math:`q_l` is the suspended liquid water specific humidity, and +:math:`q_i` is the suspended ice water specific humidity. + +As shown in the example NetCDF header, the SCM expects that the vertical +dimension is pressure levels (index 1 is the surface) and the time +dimension is in seconds. The initial conditions expected are the height +of the pressure levels in meters, and arrays representing vertical +columns of :math:`\theta_{il}` in K, :math:`q_t`, :math:`q_l`, and +:math:`q_i` in kg kg\ :math:`^{-1}`, :math:`u` and :math:`v` in m +s\ :math:`^{-1}`, turbulence kinetic energy in m\ :math:`^2` +s\ :math:`^{-2}` and ozone mass mixing ratio in kg kg\ :math:`^{-1}`. + +For forcing data, the SCM expects a time series of the following +variables: latitude and longitude in decimal degrees [in case the +column(s) is moving in time (e.g., Lagrangian column)], the surface +pressure (Pa) and surface temperature (K). If surface fluxes are +specified for the new case, one must also include a time series of the +kinematic surface sensible heat flux (K m s\ :math:`^{-1}`) and +kinematic surface latent heat flux (kg kg\ :math:`^{-1}` m +s\ :math:`^{-1}`). The following variables are expected as 2-dimensional +arrays (vertical levels first, time second): the geostrophic u (E-W) and +v (N-S) winds (m s\ :math:`^{-1}`), and the horizontal and vertical +advective tendencies of :math:`\theta_{il}` (K s\ :math:`^{-1}`) and +:math:`q_t` (kg kg\ :math:`^{-1}` s\ :math:`^{-1}`), the large scale +vertical velocity (m s\ :math:`^{-1}`), large scale pressure vertical +velocity (Pa s\ :math:`^{-1}`), the prescribed radiative heating rate (K +s\ :math:`^{-1}`), and profiles of u, v, T, :math:`\theta_{il}` and +:math:`q_t` to use for nudging. + +Although it is expected that all variables are in the NetCDF file, only +those that are used with the chosen forcing method are required to be +nonzero. For example, the following variables are required depending on +the values of ``mom_forcing_type`` and ``thermo_forcing_type`` specified in the case configuration file: + +- ``mom_forcing_type = 1`` + + - Not implemented yet + +- ``mom_forcing_type = 2`` + + - geostrophic winds and large scale vertical velocity + +- ``mom_forcing_type = 3`` + + - u and v nudging profiles + +- ``thermo_forcing_type = 1`` + + - horizontal and vertical advective tendencies of + :math:`\theta_{il}` and :math:`q_t` and prescribed radiative + heating (can be zero if radiation scheme is active) + +- ``thermo_forcing_type = 2`` + + - horizontal advective tendencies of :math:`\theta_{il}` and + :math:`q_t`, prescribed radiative heating (can be zero if + radiation scheme is active), and the large scale vertical pressure + velocity + +- ``thermo_forcing_type = 3`` + + - :math:`\theta_{il}` and :math:`q_t` nudging profiles and the large + scale vertical pressure velocity + +For the case configuration file, it is most efficient to copy an +existing file in ``ccpp-scm/scm/etc/case_config`` and edit it to suit one’s case. Recall from subsection +:numref:`Subsection %s ` that this file is used to configure +the SCM framework parameters for a given case. Be sure to check that +model timing parameters such as the time step and output frequency are +appropriate for the physics suite being used. There is likely some +stability criterion that governs the maximum time step based on the +chosen parameterizations and number of vertical levels (grid spacing). +The ``case_name`` parameter should match the name of the case input data file that was +configured for the case (without the file extension). The parameter +should be less than or equal to the length of the forcing data unless +the desired behavior of the simulation is to proceed with the last +specified forcing values after the length of the forcing data has been +surpassed. The initial date and time should fall within the forcing +period specified in the case input data file. If the case input data is +specified to a lower altitude than the vertical domain, the remainder of +the column will be filled in with values from a reference profile. There +is a tropical profile and mid-latitude summer profile provided, although +one may add more choices by adding a data file to ``ccpp-scm/scm/data/processed_case_input`` and adding a parser +section to the subroutine ``get_reference_profile`` in ``scm/src/scm_input.f90``. Surface fluxes can either be specified in +the case input data file or calculated using a surface scheme using +surface properties. If surface fluxes are specified from data, set ``sfc_flux_spec`` to ``.true.`` +and specify ``sfc_roughness_length_cm`` for the surface over which the column resides. Otherwise,` +specify a ``sfc_type``. In addition, one must specify a ``column_area`` for each column. + +To control the forcing method, one must choose how the momentum and +scalar variable forcing are applied. The three methods of Randall and +Cripe (1999, JGR) have been implemented: “revealed forcing” where total +(horizontal :math:`+` vertical) advective tendencies are applied (type +1), “horizontal advective forcing” where horizontal advective tendencies +are applied and vertical advective tendencies are calculated from a +prescribed vertical velocity and the calculated (modeled) profiles (type +2), and “relaxation forcing” where nudging to observed profiles replaces +horizontal advective forcing combined with vertical advective forcing +from prescribed vertical velocity (type 3). If relaxation forcing is +chosen, a ``relax_time`` that represents the timescale over which the profile would +return to the nudging profiles must be specified. + +.. _`lasso`: + +Using other LASSO cases +----------------------- + +In order to use other LASSO cases than the one provided, perform the +following steps: + +#. Access http://archive.arm.gov/lassobrowser and use the navigation on + the left to choose the dates for which you would like to run a SCM + simulation. Pay attention to the “Large Scale Forcing” tab where you + can choose how the large scale forcing was generated, with options + for ECMWF, MSDA, and VARANAL. All are potentially valid, and it is + likely worth exploring the differences among forcing methods. Click + on Submit to view a list of simulations for the selected criteria. + Choose from the simulations (higher skill scores are preferred) and + check the “Config Obs Model Tar” box to download the data. Once the + desired simulations have been checked, order the data (you may need + to create an ARM account to do so). + +#. Once the data is downloaded, decompress it. From the ``config`` directory, copy + the files ``input_ls_forcing.nc``, ``input_sfc_forcing.nc``, and ``wrfinput_d01.nc`` into their own directory under ``ccpp-scm/scm/data/raw_case_input/``. + +#. Modify ``ccpp-scm/scm/etc/scripts/lasso1_forcing_file_generator_gjf.py`` to point to the input files listed above. Execute the script + in order to generate a case input file for the SCM (to be put in ``ccpp-scm/scm/data/processed_case_input/``): + + .. code:: bash + + ./lasso1_forcing_file_generator_gjf.py + +#. Create a new case configuration file (or copy and modify an existing + one) in ``ccpp-scm/scm/etc/case_config``. Be sure that the ``case_name`` variable points to the newly + created/processed case input file from above. + +.. _`UFSreplay`: + +Using UFS Output to Create SCM Cases: UFS-Replay +------------------------------------------------ + +.. _`pydepend_replay`: + +Python Dependencies +~~~~~~~~~~~~~~~~~~~ + +The scripts here require a few python packages that may not be found by +default in all python installations. There is a YAML file with the +python environment needed to run the script in ``ccpp-scm/environment-ufsreplay.yml``. To create and activate +this environment using conda: + +Create environment (only once): + +.. code:: bash + + > conda env create -f environment-ufsreplay.yml + +This will create the conda environment ``env_ufsreplay`` + +Activate environment: + +.. code:: bash + + > conda activate env_ufsreplay + +.. _`ufsicgenerator`: + +UFS_IC_generator.py +~~~~~~~~~~~~~~~~~~~ + +A script exists in ``scm/etc/scripts/UFS_IC_generator.py`` to read in UFS history (output) files and their +initial conditions to generate a SCM case input data file, in DEPHY +format. + +.. code:: bash + + ./UFS_IC_generator.py [-h] (-l LOCATION LOCATION | -ij INDEX INDEX) -d + DATE -i IN_DIR -g GRID_DIR -f FORCING_DIR -n + CASE_NAME [-t {1,2,3,4,5,6,7}] [-a AREA] [-oc] + [-lam] [-sc] [-near] + +Mandatory arguments: + +#. ``--location (-l)`` OR ``--index (-ij)``: Either longitude and latitude in decimal degrees east and north + of a location OR the UFS grid index with the tile number + + - -l 261.51 38.2 (two floating point values separated by a space) + + - -ij 8 49 (two integer values separated by a space; this option + must also use the ``--tile (-t)`` argument to specify the tile number) + +#. ``--date (-d)`` YYYYMMDDHHMMSS: date corresponding to the UFS initial conditions + +#. ``--in_dir (-i)``: path to the directory containing the UFS initial conditions + +#. ``--grid_dir (-g)``: path to the directory containing the UFS supergrid files (AKA "fix" + directory) + +#. ``--forcing_dir (-f)``: path to the directory containing the UFS history files + +#. ``--case_name (-n)``: name of case + +Optional arguments: + +#. ``--tile (-t)``: if one already knows the correct tile for the given longitude and + latitude OR one is specifying the UFS grid index (``--index`` argument) + +#. ``--area (-a)``: area of grid cell in :math:`m^2` (if known or different than the + value calculated from the supergrid file) + +#. ``--old_chgres (-oc)``: flag if UFS initial conditions were generated using older version + of chgres (global_chgres); might be the case for pre-2018 data + +#. ``--lam (-lam)``: flag to signal that the ICs and forcing is from a limited-area + model run + +#. ``--save_comp (-sc)``: flag to create UFS reference file for comparison + +#. ``--use_nearest (-near)``: flag to indicate using the nearest UFS history file gridpoint + +.. _`ufsforcingensemblegenerator`: + +UFS_forcing_ensemble_generator.py +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There is an additional script in ``scm/etc/scripts/UFS_forcing_ensemble_generator.py`` to create UFS-replay case(s) starting +with output from UFS Weather Model (UWM) Regression Tests (RTs). + +.. code:: bash + + UFS_forcing_ensemble_generator.py [-h] -d DIR -n CASE_NAME + (-lonl LON_1 LON_2 -latl LAT_1 LAT_2 -nens NENSMEMBERS | + -lons [LON_LIST] -lats [LAT_LIST]) + [-dt TIMESTEP] [-cres C_RES] [-sdf SUITE] [-sc] [-near] + +Mandatory arguments: + +#. ``--dir (-d)``: path to UFS Regression Test output + +#. ``--case_name (-n)``: name of cases + +#. Either: (see examples below) + + - ``--lon_limits (-lonl)`` AND ``--lat_limits (-latl)`` AND ``--nensmembers (-nens)``: longitude range, latitude range, and number of cases to + create + + - ``--lon_list (-lons)`` AND ``--lat_list (-lats)``: longitude and latitude of cases + +Optional arguments: + +#. ``--timestep (-dt)``: SCM timestep, in seconds + +#. ``--C_res (-cres)``: UFS spatial resolution + +#. ``--suite (-sdf)``: CCPP suite definition file to use for ensemble + +#. ``--save_comp (-sc)``: flag to create UFS reference file for comparison + +#. ``--use_nearest (-near)``: flag to indicate using the nearest UFS history file gridpoint + +Examples to run from within the ``scm/etc/scripts`` directory to create SCM cases starting +with the output from a UFS Weather Model regression test(s): + +On the supported platforms Cheyenne (NCAR) and Hera (NOAA), there are +staged UWM RTs located at: + +- Cheyenne ``/glade/scratch/epicufsrt/GMTB/CCPP-SCM/UFS_RTs`` +- Hera ``/scratch1/BMC/gmtb/CCPP-SCM/UFS_RTs`` + +.. _`example1`: + +Example 1: UFS-replay for single point +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +UFS regression test, ``control_c192``, for single point. + +.. code:: bash + + ./UFS_forcing_ensemble_generator.py -d /glade/scratch/epicufsrt/GMTB/CCPP-SCM/UFS_RTs/control_c192/ -sc --C_RES 192 -dt 360 -n control_c192 -lons 300 -lats 34 + +Upon successful completion of the script, the command to run the case(s) +will print to the screen. For example, + +.. code:: bash + + ./run_scm.py --npz_type gfs --file scm_ufsens_control_c192.py --timestep 360 + +The file ``scm_ufsens_control_c192.py`` is created in ``ccpp-scm/scm/bin/``, where the SCM run script is to be exectued. + +.. _`example2`: + +Example 2: UFS-replay for list of points +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +UFS regression test, ``control_c384``, for multiple points. + +.. code:: bash + + ./UFS_forcing_ensemble_generator.py -d /glade/scratch/epicufsrt/GMTB/CCPP-SCM/UFS_RTs/control_c384/ -sc --C_RES 384 -dt 225 -n control_c384 -lons 300 300 300 300 -lats 34 35 35 37 + +Upon successful completion of the script, the command to run the case(s) +will print to the screen. For example, + +.. code:: bash + + ./run_scm.py --npz_type gfs --file scm_ufsens_control_c384.py --timestep 225 + +The file ``scm_ufsens_control_c384.py`` contains ALL of the cases created. Each case created will have the +naming convention ``case_name_nXXX``, where the suffix ``XXX`` is the case number from 0 to the +number of points provided. The contents of the file should look like: + +.. code:: bash + + run_list = [{"case": "control_c384_n000", "suite": "SCM_GFS_v16"}, + {"case": "control_c384_n001", "suite": "SCM_GFS_v16"}, + {"case": "control_c384_n002", "suite": "SCM_GFS_v16"}, + {"case": "control_c384_n003", "suite": "SCM_GFS_v16"}] + +.. _`example3`: + +Example 3: UFS-replay for an ensemble of points +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +UFS regression test, ``control_p8``, for an ensemble (10) of randomly selected points +over a specified longitude (:math:`300-320^oW`) and latitude +(:math:`40-50^oN`) range + +But first, to use the ``control_p8`` test we need to rerun the regression test to +generate UFS history files with a denser and constant output interval. +First, in ``control_p8/model_configure``, change ``--output_fh`` to ``"interval -1"``, where ``interval`` is the UFS history file output frequency +(in hours), see `UFS Weather Model Users +Guide `__ +for more details. + +For the purposes of this example the ``control_p8`` test has already been rerun, but if +starting from your own UWM RTs, you can rerun the UWM regression test, +on Cheyenne for example, by running the following command in the RT +directory: ``qsub job_card`` + +Now the cases can be generated with the following command: + +.. code:: bash + + ./UFS_forcing_ensemble_generator.py -d /glade/scratch/epicufsrt/GMTB/CCPP-SCM/UFS_RTs/control_p8/ -sc --C_RES 96 -dt 720 -n control_p8 -lonl 300 320 -latl 40 50 -nens 10 -sdf SCM_GFS_v17_p8 + +Upon successful completion of the script, the command to run the case(s) +will print to the screen. For example, + +.. code:: bash + + ./run_scm.py --npz_type gfs --file scm_ufsens_control_p8.py --timestep 720 + +The file ``scm_ufsens_control_p8.py`` contains ten cases (n000-n009) to be run. The contents of the +file should look like: + +.. code:: bash + + run_list = [{"case": "control_p8_n000", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n001", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n002", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n003", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n004", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n005", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n006", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n007", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n008", "suite": "SCM_GFS_v17_p8"}, + {"case": "control_p8_n009", "suite": "SCM_GFS_v17_p8"}] diff --git a/scm/doc/TechGuide/chap_cases.tex b/scm/doc/TechGuide/chap_cases.tex deleted file mode 100644 index e12b75ead..000000000 --- a/scm/doc/TechGuide/chap_cases.tex +++ /dev/null @@ -1,364 +0,0 @@ -\chapter{Cases} -\label{chapter: cases} - -\section{How to run cases} -Only two files are needed to set up and run a case with the SCM. The first is a configuration namelist file found in \execout{ccpp-scm/scm/etc/case\_config} that contains parameters for the SCM infrastructure. The second necessary file is a NetCDF file containing data to initialize the column state and time-dependent data to force the column state. The two files are described below. - -\subsection{Case configuration namelist parameters} -\label{subsection: case config} -The \execout{case\_config} namelist expects the following parameters: -\begin{itemize} -\item \execout{case\_name} - \begin{itemize} - \item Identifier for which dataset (initialization and forcing) to load. This string must correspond to a dataset included in the directory \execout{ccpp-scm/scm/data/processed\_case\_input/} (without the file extension). - \end{itemize} -\item \execout{runtime} - \begin{itemize} - \item Specify the model runtime in seconds (integer). This should correspond with the forcing dataset used. If a runtime is specified that is longer than the supplied forcing, the forcing is held constant at the last specified values. - \end{itemize} -\item \execout{thermo\_forcing\_type} - \begin{itemize} - \item An integer representing how forcing for temperature and moisture state variables is applied (1 $=$ total advective tendencies, 2 $=$ horizontal advective tendencies with prescribed vertical motion, 3 $=$ relaxation to observed profiles with vertical motion prescribed) - \end{itemize} -\item \execout{mom\_forcing\_type} - \begin{itemize} - \item An integer representing how forcing for horizontal momentum state variables is applied (1 $=$ total advective tendencies; not implemented yet, 2 $=$ horizontal advective tendencies with prescribed vertical motion, 3 $=$ relaxation to observed profiles with vertical motion prescribed) - \end{itemize} -\item \execout{relax\_time} - \begin{itemize} - \item A floating point number representing the timescale in seconds for the relaxation forcing (only used if \execout{thermo\_forcing\_type} $=$ \exec{3} or \execout{mom\_forcing\_type} $=$ \exec{3}) - \end{itemize} -\item \execout{sfc\_flux\_spec} - \begin{itemize} - \item A boolean set to \exec{.true.} if surface flux are specified from the forcing data (there is no need to have surface schemes in a suite definition file if so) - \end{itemize} -\item \execout{sfc\_roughness\_length\_cm} - \begin{itemize} - \item Surface roughness length in cm for calculating surface-related fields from specified surface fluxes (only used if \execout{sfc\_flux\_spec} is True). - \end{itemize} -\item \execout{sfc\_type} - \begin{itemize} - \item An integer representing the character of the surface (0 $=$ sea surface, 1 $=$ land surface, 2 $=$ sea-ice surface) - \end{itemize} -\item \execout{reference\_profile\_choice} - \begin{itemize} - \item An integer representing the choice of reference profile to use above the supplied initialization and forcing data (1 $=$ ``McClatchey'' profile, 2 $=$ mid-latitude summer standard atmosphere) - \end{itemize} -\item \execout{year} - \begin{itemize} - \item An integer representing the year of the initialization time - \end{itemize} -\item \execout{month} - \begin{itemize} - \item An integer representing the month of the initialization time - \end{itemize} -\item \execout{day} - \begin{itemize} - \item An integer representing the day of the initialization time - \end{itemize} -\item \execout{hour} - \begin{itemize} - \item An integer representing the hour of the initialization time - \end{itemize} -\item \execout{column\_area} - \begin{itemize} - \item A list of floating point values representing the characteristic horizontal domain area of each atmospheric column in square meters (this could be analogous to a 3D model's horizontal grid size or the characteristic horizontal scale of an observation array; these values are used in scale-aware schemes; if using multiple columns, you may specify an equal number of column areas) - \end{itemize} -\item \execout{model\_ics} - \begin{itemize} - \item A boolean set to \exec{.true.} if UFS atmosphere initial conditions are used rather than field campaign-based initial conditions - \end{itemize} -\item \execout{C\_RES} - \begin{itemize} - \item An integer representing the grid size of the UFS atmosphere initial conditions; the integer represents the number of grid points in each horizontal direction of each cube tile - \end{itemize} -\item \execout{input\_type} - \begin{itemize} - \item 0 => original DTC format, 1 => DEPHY-SCM format. - \end{itemize} -\end{itemize} - -Optional variables (that may be overridden via run script command line arguments) are: -\begin{itemize} -\item \execout{vert\_coord\_file} - \begin{itemize} - \item File containing FV3 vertical grid coefficients. - \end{itemize} -\item \execout{n\_levels} - \begin{itemize} - \item Specify the integer number of vertical levels. - \end{itemize} -\end{itemize} - -\subsection{Case input data file (CCPP-SCM format)} -\label{subsection: case input} - -The initialization and forcing data for each case is stored in a NetCDF (version 4) file within the \execout{ccpp-scm/scm/data/processed\_case\_input} directory. Each file has at least two dimensions (\execout{time} and \execout{levels}, potentially with additions for vertical snow and soil levels) and is organized into 3 groups: scalars, initial, and forcing. Not all fields are required for all cases. For example the fields \execout{sh\_flux\_sfc} and \execout{lh\_flux\_sfc} are only needed if the variable \execout{sfc\_flx\_spec} $=$ \exec{.true.} in the case configuration file and state nudging variables are only required if \execout{thermo\_forcing\_type} $=$ \exec{3} or \execout{mom\_forcing\_type} $=$ \exec{3}. Using an active LSM (Noah, NoahMP, RUC) requires many more variables than are listed here. Example files for using with Noah and NoahMP LSMs are included in \execout{ccpp-scm/scm/data/processed\_case\_input/fv3\_model\_point\_noah[mp].nc}. - -\lstinputlisting[ - basicstyle=\scriptsize\ttfamily, - label=lst_case_input_netcdf_header, - caption=example NetCDF file (CCPP-SCM format) header for case initialization and forcing data - ]{./arm_case_header.txt} - -\subsection{Case input data file (DEPHY format)} -\label{subsection: case input dephy} -The Development and Evaluation of Physics in atmospheric models (DEPHY) format is an internationally-adopted data format intended for use by SCM and LESs. The initialization and forcing data for each case is stored in a NetCDF (version 4) file, although these files are not by default included in the CCPP SCM repository. To access these cases you need to clone the DEPHY-SCM repository, and provide the DEPHY-SCM file location to the SCM. For example: -\begin{lstlisting}[language=bash] -cd [...]/ccpp-scm/scm/data -git clone https://github.com/GdR-DEPHY/DEPHY-SCM DEPHY-SCM -cd [...]/ccpp-scm/scm/bin -./run_scm.py -c MAGIC_LEG04A --case_data_dir [...]/ccpp-scm/scm/data/DEPHY-SCM/MAGIC/LEG04A -v -\end{lstlisting} -Each DEPHY file has three dimensions (\execout{time}, \execout{t0}, \execout{levels}) and contains the initial conditions (\execout{t0}, \execout{levels}) and forcing data (\execout{time}, \execout{levels}). Just as when using the CCPP-SCM formatted inputs, \ref{subsection: case input}, not all fields are required for all cases. More information on the DEPHY format requirements can be found at \href{https://github.com/GdR-DEPHY/DEPHY-SCM}{DEPHY}. - -\lstinputlisting[ - basicstyle=\scriptsize\ttfamily, - label=lst_case_input_netcdf_header, - caption=example NetCDF file (DEPHY format) header for case initialization and forcing data - ]{./dephy_case_header.txt} - -\section{Included Cases} -Several cases are included in the repository to serve as examples for users to create their own and for basic research. All case configuration namelist files for included cases can be found in \execout{ccpp-scm/scm/etc/case\_config} and represent the following observational field campaigns: -\begin{itemize} -\item Tropical Warm Pool -- International Cloud Experiment (TWP-ICE) maritime deep convection -\item Atmospheric Radiation Measurement (ARM) Southern Great Plains (SGP) Summer 1997 continental deep convection -\item Atlantic Stratocumulus Transition EXperiment (ASTEX) maritime stratocumulus-to-cumulus transition -\item Barbados Oceanographic and Meteorological EXperiment (BOMEX) maritime shallow convection -\item Large eddy simulation ARM Symbiotic Simulation and Observation (LASSO) for May 18, 2016 (with capability to run all LASSO dates - see \ref{sec:lasso}) continental shallow convection -\end{itemize} -For the ARM SGP case, several case configuration files representing different time periods of the observational dataset are included, denoted by a trailing letter. The LASSO case may be run with different forcing applied, so three case configuration files corresponding to these different forcing are included. In addition, two example cases are included for using UFS Atmosphere initial conditions: -\begin{itemize} -\item UFS initial conditions for 38.1 N, 98.5 W (central Kansas) for 00Z on Oct. 3, 2016 with Noah variables on the C96 FV3 grid (\execout{fv3\_model\_point\_noah.nc}) -\item UFS initial conditions for 38.1 N, 98.5 W (central Kansas) for 00Z on Oct. 3, 2016 with NoahMP variables on the C96 FV3 grid (\execout{fv3\_model\_point\_noahmp.nc}) -\end{itemize} -See \ref{sec:UFSreplay} for information on how to generate these files for other locations and dates, given appropriate UFS Atmosphere initial conditions and output. - -\section{How to set up new cases} - -Setting up a new case involves preparing the two types of files listed above. For the case initialization and forcing data file, this typically involves writing a custom script or program to parse the data from its original format to the format that the SCM expects, listed above. An example of this type of script written in Python is included in \execout{/ccpp-scm/scm/etc/scripts/twpice\_forcing\_file\_generator.py}. The script reads in the data as supplied from its source, converts any necessary variables, and writes a NetCDF (version 4) file in the format described in subsections \ref{subsection: case input} and \ref{subsection: case input dephy}. For reference, the following formulas are used: -\begin{equation} -\theta_{il} = \theta - \frac{\theta}{T}\left(\frac{L_v}{c_p}q_l + \frac{L_s}{c_p}q_i\right) -\end{equation} -\begin{equation} -q_t = q_v + q_l + q_i -\end{equation} -where $\theta_{il}$ is the ice-liquid water potential temperature, $\theta$ is the potential temperature, $L_v$ is the latent heat of vaporization, $L_s$ is the latent heat of sublimation $c_p$ is the specific heat capacity of air at constant pressure, $T$ is absolute temperature, $q_t$ is the total water specific humidity, $q_v$ is the water vapor specific humidity, $q_l$ is the suspended liquid water specific humidity, and $q_i$ is the suspended ice water specific humidity. - -As shown in the example NetCDF header, the SCM expects that the vertical dimension is pressure levels (index 1 is the surface) and the time dimension is in seconds. The initial conditions expected are the height of the pressure levels in meters, and arrays representing vertical columns of $\theta_{il}$ in K, $q_t$, $q_l$, and $q_i$ in kg kg$^{-1}$, $u$ and $v$ in m s$^{-1}$, turbulence kinetic energy in m$^2$ s$^{-2}$ and ozone mass mixing ratio in kg kg$^{-1}$. - -For forcing data, the SCM expects a time series of the following variables: latitude and longitude in decimal degrees [in case the column(s) is moving in time (e.g., Lagrangian column)], the surface pressure (Pa) and surface temperature (K). If surface fluxes are specified for the new case, one must also include a time series of the kinematic surface sensible heat flux (K m s$^{-1}$) and kinematic surface latent heat flux (kg kg$^{-1}$ m s$^{-1}$). The following variables are expected as 2-dimensional arrays (vertical levels first, time second): the geostrophic u (E-W) and v (N-S) winds (m s$^{-1}$), and the horizontal and vertical advective tendencies of $\theta_{il}$ (K s$^{-1}$) and $q_t$ (kg kg$^{-1}$ s$^{-1}$), the large scale vertical velocity (m s$^{-1}$), large scale pressure vertical velocity (Pa s$^{-1}$), the prescribed radiative heating rate (K s$^{-1}$), and profiles of u, v, T, $\theta_{il}$ and $q_t$ to use for nudging. - -Although it is expected that all variables are in the NetCDF file, only those that are used with the chosen forcing method are required to be nonzero. For example, the following variables are required depending on the values of \execout{mom\_forcing\_type} and \execout{thermo\_forcing\_type} specified in the case configuration file: - -\begin{itemize} -\item \execout{mom\_forcing\_type} $=$ \exec{1} - \begin{itemize} - \item Not implemented yet - \end{itemize} -\item \execout{mom\_forcing\_type} $=$ \exec{2} - \begin{itemize} - \item geostrophic winds and large scale vertical velocity - \end{itemize} -\item \execout{mom\_forcing\_type} $=$ \exec{3} - \begin{itemize} - \item u and v nudging profiles - \end{itemize} -\item \execout{thermo\_forcing\_type} $=$ \exec{1} - \begin{itemize} - \item horizontal and vertical advective tendencies of $\theta_{il}$ and $q_t$ and prescribed radiative heating (can be zero if radiation scheme is active) - \end{itemize} -\item \execout{thermo\_forcing\_type} $=$ \exec{2} - \begin{itemize} - \item horizontal advective tendencies of $\theta_{il}$ and $q_t$, prescribed radiative heating (can be zero if radiation scheme is active), and the large scale vertical pressure velocity - \end{itemize} -\item \execout{thermo\_forcing\_type} $=$ \exec{3} - \begin{itemize} - \item $\theta_{il}$ and $q_t$ nudging profiles and the large scale vertical pressure velocity - \end{itemize} -\end{itemize} - -For the case configuration file, it is most efficient to copy an existing file in \execout{ccpp-scm/scm/etc/case\_config} and edit it to suit one's case. Recall from subsection \ref{subsection: case config} that this file is used to configure the SCM framework parameters for a given case. Be sure to check that model timing parameters such as the time step and output frequency are appropriate for the physics suite being used. There is likely some stability criterion that governs the maximum time step based on the chosen parameterizations and number of vertical levels (grid spacing). The \execout{case\_name} parameter should match the name of the case input data file that was configured for the case (without the file extension). The \execout{runtime} parameter should be less than or equal to the length of the forcing data unless the desired behavior of the simulation is to proceed with the last specified forcing values after the length of the forcing data has been surpassed. The initial date and time should fall within the forcing period specified in the case input data file. If the case input data is specified to a lower altitude than the vertical domain, the remainder of the column will be filled in with values from a reference profile. There is a tropical profile and mid-latitude summer profile provided, although one may add more choices by adding a data file to \execout{ccpp-scm/scm/data/processed\_case\_input} and adding a parser section to the subroutine \execout{get\_reference\_profile} in \execout{-scm/scm/src/scm\_input.f90}. Surface fluxes can either be specified in the case input data file or calculated using a surface scheme using surface properties. If surface fluxes are specified from data, set \execout{sfc\_flux\_spec} to \exec{.true.} and specify \execout{sfc\_roughness\_length\_cm} for the surface over which the column resides. Otherwise, specify a \execout{sfc\_type}. In addition, one must specify a \execout{column\_area} for each column. - -To control the forcing method, one must choose how the momentum and scalar variable forcing are applied. The three methods of Randall and Cripe (1999, JGR) have been implemented: ``revealed forcing'' where total (horizontal $+$ vertical) advective tendencies are applied (type 1), ``horizontal advective forcing'' where horizontal advective tendencies are applied and vertical advective tendencies are calculated from a prescribed vertical velocity and the calculated (modeled) profiles (type 2), and ``relaxation forcing'' where nudging to observed profiles replaces horizontal advective forcing combined with vertical advective forcing from prescribed vertical velocity (type 3). If relaxation forcing is chosen, a \execout{relax\_time} that represents the timescale over which the profile would return to the nudging profiles must be specified. - -\section{Using other LASSO cases} -\label{sec:lasso} - -In order to use other LASSO cases than the one provided, perform the following steps: -\begin{enumerate} -\item Access \url{http://archive.arm.gov/lassobrowser} and use the navigation on the left to choose the dates for which you would like to run a SCM simulation. Pay attention to the ``Large Scale Forcing'' tab where you can choose how the large scale forcing was generated, with options for ECMWF, MSDA, and VARANAL. All are potentially valid, and it is likely worth exploring the differences among forcing methods. Click on Submit to view a list of simulations for the selected criteria. Choose from the simulations (higher skill scores are preferred) and check the ``Config Obs Model Tar'' box to download the data. Once the desired simulations have been checked, order the data (you may need to create an ARM account to do so). -\item Once the data is downloaded, decompress it. From the \execout{config} directory, copy the files \execout{input\_ls\_forcing.nc}, \execout{input\_sfc\_forcing.nc}, and \execout{wrfinput\_d01.nc} into their own directory under \execout{ccpp-scm/scm/data/raw\_case\_input/}. -\item Modify \execout{ccpp-scm/scm/etc/scripts/lasso1\_forcing\_file\_generator\_gjf.py} to point to the input files listed above. Execute the script in order to generate a case input file for the SCM (to be put in \execout{ccpp-scm/scm/data/processed\_case\_input/}): -\begin{lstlisting}[language=bash] -./lasso1_forcing_file_generator_gjf.py -\end{lstlisting} -\item Create a new case configuration file (or copy and modify an existing one) in \execout{ccpp-scm/scm/etc/case\_config}. Be sure that the \execout{case\_name} variable points to the newly created/processed case input file from above. -\end{enumerate} - -\section{Using UFS Output to Create SCM Cases: UFS-Replay} -\label{sec:UFSreplay} - - -\subsection{Python Dependencies} -\label{subsection: pydepend} -The scripts here require a few python packages that may not be found by default in all python installations. There is a YAML file with the python environment needed to run the script in \execout{ccpp-scm/environment-ufsreplay.yml}. To create and activate this environment using conda: - -Create environment (only once): - -\execout{> conda env create -f environment-ufsreplay.yml} - -This will create the conda environment \execout{env\_ufsreplay} - - -Activate environment: - -\execout{> conda activate env\_ufsreplay} - - -\subsection{UFS\_IC\_generator.py} -\label{subsection: ufsicgenerator} -A script exists in \execout{scm/etc/scripts/UFS\_IC\_generator.py} to read in UFS history (output) files and their initial conditions to generate a SCM case input data file, in DEPHY format. - - -\begin{lstlisting}[language=bash] -./UFS_IC_generator.py [-h] (-l LOCATION LOCATION | -ij INDEX INDEX) -d -DATE -i IN_DIR -g GRID_DIR -f FORCING_DIR -n -CASE_NAME [-t {1,2,3,4,5,6,7}] [-a AREA] [-oc] -[-lam] [-sc] [-near] -\end{lstlisting} - -Mandatory arguments: -\begin{enumerate} -\item \exec{-{}-location (-l)} OR \exec{-{}-index (-ij)}: Either longitude and latitude in decimal degrees east and north of a location OR the UFS grid index with the tile number - \begin{itemize} - \item -l 261.51 38.2 (two floating point values separated by a space) - \item -ij 8 49 (two integer values separated by a space; this option must also use the \exec{-{}-tile (-t)} argument to specify the tile number) - \end{itemize} -\item \exec{-{}-date (-d)} YYYYMMDDHHMMSS: date corresponding to the UFS initial conditions -\item \exec{-{}-in\_dir (-i)}: path to the directory containing the UFS initial conditions -\item \exec{-{}-grid\_dir (-g)}: path to the directory containing the UFS supergrid files (AKA "fix" directory) -\item \exec{-{}-forcing\_dir (-f)}: path to the directory containing the UFS history files -\item \exec{-{}-case\_name (-n)}: name of case -\end{enumerate} - -Optional arguments: -\begin{enumerate} -\item \exec{-{}-tile (-t)}: if one already knows the correct tile for the given longitude and latitude OR one is specifying the UFS grid index (\exec{-{}-index} argument) -\item \exec{-{}-area (-a)}: area of grid cell in $m^2$ (if known or different than the value calculated from the supergrid file) -\item \exec{-{}-old\_chgres (-oc)}: flag if UFS initial conditions were generated using older version of chgres (global\_chgres); might be the case for pre-2018 data -\item \exec{-{}-lam (-lam)}: flag to signal that the ICs and forcing is from a limited-area model run -\item \exec{-{}-save\_comp (-sc)}: flag to create UFS reference file for comparison -\item \exec{-{}-use\_nearest (-near)}: flag to indicate using the nearest UFS history file gridpoint -\end{enumerate} - -\subsection{UFS\_forcing\_ensemble\_generator.py} -\label{subsection: ufsforcingensemblegenerator} -There is an additional script in \execout{scm/etc/scripts/UFS\_forcing\_ensemble\_generator.py} to create UFS-replay case(s) starting with output from UFS Weather Model (UWM) Regression Tests (RTs). - -\begin{lstlisting}[language=bash] -UFS_forcing_ensemble_generator.py [-h] -d DIR -n CASE_NAME -(-lonl LON_1 LON_2 -latl LAT_1 LAT_2 -nens NENSMEMBERS | --lons [LON_LIST] -lats [LAT_LIST]) -[-dt TIMESTEP] [-cres C_RES] [-sdf SUITE] [-sc] [-near] -\end{lstlisting} - -Mandatory arguments: -\begin{enumerate} -\item \exec{-{}-dir (-d)}: path to UFS Regression Test output -\item \exec{-{}-case\_name (-n)}: name of cases -\item Either: (see examples below) - \begin{itemize} - \item \exec{-{}-lon\_limits (-lonl)} AND \exec{-{}-lat\_limits (-latl)} AND \exec{-{}-nensmembers (-nens)}: longitude range, latitude range, and number of cases to create - \item \exec{-{}-lon\_list (-lons)} AND \exec{-{}-lat\_list (-lats)}: longitude and latitude of cases - \end{itemize} -\end{enumerate} - -Optional arguments: -\begin{enumerate} -\item \exec{-{}-timestep (-dt)}: SCM timestep, in seconds -\item \exec{-{}-C\_res (-cres)}: UFS spatial resolution -\item \exec{-{}-suite (-sdf)}: CCPP suite definition file to use for ensemble -\item \exec{-{}-save\_comp (-sc)}: flag to create UFS reference file for comparison -\item \exec{-{}-use\_nearest (-near)}: flag to indicate using the nearest UFS history file gridpoint -\end{enumerate} - -Examples to run from within the \exec{scm/etc/scripts} directory to create SCM cases starting with the output from a UFS Weather Model regression test(s): - -On the supported platforms Cheyenne (NCAR) and Hera (NOAA), there are staged UWM RTs located at: -\begin{itemize} -\item \execout{Cheyenne /glade/scratch/epicufsrt/GMTB/CCPP-SCM/UFS\_RTs} -\item \execout{Hera /scratch1/BMC/gmtb/CCPP-SCM/UFS\_RTs} -\end{itemize} - -\subsection{Example 1: UFS-replay for single point} -\label{subsection: example1} - -UFS regression test, \execout{control\_c192}, for single point. -\begin{lstlisting}[language=bash] -./UFS_forcing_ensemble_generator.py -d /glade/scratch/epicufsrt/GMTB/CCPP-SCM/UFS_RTs/control_c192/ -sc --C_RES 192 -dt 360 -n control_c192 -lons 300 -lats 34 -\end{lstlisting} - -Upon successful completion of the script, the command to run the case(s) will print to the screen. For example, - -\begin{lstlisting}[language=bash] -./run_scm.py --npz_type gfs --file scm_ufsens_control_c192.py --timestep 360 -\end{lstlisting} - -The file \exec{scm\_ufsens\_control\_c192.py} is created in \exec{ccpp-scm/scm/bin/}, where the SCM run script is to be exectued. - -\subsection{Example 2: UFS-replay for list of points} -\label{subsection: example2} - -UFS regression test, \execout{control\_c384}, for multiple points. -\begin{lstlisting}[language=bash] -./UFS_forcing_ensemble_generator.py -d /glade/scratch/epicufsrt/GMTB/CCPP-SCM/UFS_RTs/control_c384/ -sc --C_RES 384 -dt 225 -n control_c384 -lons 300 300 300 300 -lats 34 35 35 37 -\end{lstlisting} - -Upon successful completion of the script, the command to run the case(s) will print to the screen. For example, - -\begin{lstlisting}[language=bash] -./run_scm.py --npz_type gfs --file scm_ufsens_control_c384.py --timestep 225 -\end{lstlisting} - -The file \exec{scm\_ufsens\_control\_c384.py} contains \exec{ALL} of the cases created. Each case created will have the naming convention \exec{case\_name\_nXXX}, where the suffix \exec{XXX} is the case number from 0 to the number of points provided. The contents of the file should look like: -\begin{lstlisting}[language=bash] -run_list = [{"case": "control_c384_n000", "suite": "SCM_GFS_v16"}, - {"case": "control_c384_n001", "suite": "SCM_GFS_v16"}, - {"case": "control_c384_n002", "suite": "SCM_GFS_v16"}, - {"case": "control_c384_n003", "suite": "SCM_GFS_v16"}] -\end{lstlisting} - -\subsection{Example 3: UFS-replay for an ensemble of points} -\label{subsection: example3} - -UFS regression test, \execout{control\_p8}, for an ensemble (10) of randomly selected points over a specified longitude ($300-320^oW$) and latitude ($40-50^oN$) range - -But first, to use the \execout{control\_p8} test we need to rerun the regression test to generate UFS history files with a denser and constant output interval. First, in \execout{control\_p8/model\_configure}, change \exec{-{}-output\_fh} to \execout{"interval -1"}, where \execout{interval} is the UFS history file output frequency (in hours), see \href{https://ufs-weather-model.readthedocs.io/en/latest/InputsOutputs.html}{UFS Weather Model Users Guide} for more details. - -For the purposes of this example the \execout{control\_p8} test has already been rerun, but if starting from your own UWM RTs, you can rerun the UWM regression test, on Cheyenne for example, by running the following command in the RT directory: \execout{qsub job\_card} - -Now the cases can be generated with the following command: -\begin{lstlisting}[language=bash] -./UFS_forcing_ensemble_generator.py -d /glade/scratch/epicufsrt/GMTB/CCPP-SCM/UFS_RTs/control_p8/ -sc --C_RES 96 -dt 720 -n control_p8 -lonl 300 320 -latl 40 50 -nens 10 -sdf SCM_GFS_v17_p8 -\end{lstlisting} - -Upon successful completion of the script, the command to run the case(s) will print to the screen. For example, - -\begin{lstlisting}[language=bash] -./run_scm.py --npz_type gfs --file scm_ufsens_control_p8.py --timestep 720 -\end{lstlisting} - -The file \exec{scm\_ufsens\_control\_p8.py} contains ten cases (n000-n009) to be run. The contents of the file should look like: -\begin{lstlisting}[language=bash] -run_list = [{"case": "control_p8_n000", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n001", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n002", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n003", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n004", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n005", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n006", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n007", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n008", "suite": "SCM_GFS_v17_p8"}, - {"case": "control_p8_n009", "suite": "SCM_GFS_v17_p8"}] -\end{lstlisting} diff --git a/scm/doc/TechGuide/chap_ccpp.rst b/scm/doc/TechGuide/chap_ccpp.rst new file mode 100644 index 000000000..1f4e9f829 --- /dev/null +++ b/scm/doc/TechGuide/chap_ccpp.rst @@ -0,0 +1,383 @@ +.. _`chapter: ccpp_interface`: + +CCPP Interface +============== + +Chapter 6 of the CCPP v6 Technical Documentation +(https://ccpp-techdoc.readthedocs.io/en/v6.0.0/) provides a wealth of +information on the overall process of connecting a host model to the +CCPP framework for calling physics. This chapter describes the +particular implementation within this SCM, including how to set up, +initialize, call, and change a physics suite using the CCPP framework. + +Setting up a suite +------------------ + +Setting up a physics suite for use in the CCPP SCM with the CCPP +framework involves three steps: preparing data to be made available to +physics through the CCPP, running the ``ccpp_prebuild.py`` script to reconcile SCM-provided +variables with physics-required variables, and preparing a suite +definition file. + +Preparing data from the SCM +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As described in sections 6.1 and 6.2 of the `CCPP Technical +Documentation `__ a host +model must allocate memory and provide metadata for variables that are +passed into and out of the schemes within the physics suite. As of this +release, in practice this means that a host model must do this for all +variables needed by all physics schemes that are expected to be used +with the host model. For this SCM, all variables needed by the physics +schemes are allocated and documented in the file ``ccpp-scm/scm/src/scm_type_defs.f90`` and are contained +within the ``physics`` derived data type. This derived data type initializes its +component variables in a ``create`` type-bound procedure. As mentioned in section +6.2 of the `CCPP Technical +Documentation `__, files +containing all required metadata was constructed for describing all +variables in the derived data type. These files are ``scm/src/GFS_typedefs.meta,``, ``scm/src/CCPP_typedefs.meta``, and ``scm_physical_constants.meta``. Further, ``scm_type_defs.meta`` +exists to provide metadata for derived data type definitions and their +instances, which is needed by the ccpp-framework to properly reference +the data. The standard names of all variables in this table must match +with a corresponding variable within one or more of the physics schemes. +A list of all standard names used can be found in ``ccpp/framework/doc/DevelopersGuide/CCPP_VARIABLES_SCM.pdf``. + +Editing and running ``ccpp_prebuild.py`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +General instructions for configuring and running the ``ccpp_prebuild.py`` script can be found +in chapter 8 of the `CCPP Technical +Documentation `__. The +script expects to be run with a host-model-dependent configuration file, +passed as argument ``–config=path_to_config_file``. Within this configuration file are variables that +hold paths to the variable definition files (where metadata tables can +be found on the host model side), the scheme files (a list of paths to +all source files containing scheme entry points), the auto-generated +physics schemes makefile snippet, the auto-generated physics scheme caps +makefile snippet, and the directory where the auto-generated physics +caps should be written out to. As mentioned in :numref:`Section %s `, this script must be run +to reconcile data provided by the SCM with data required by the physics +schemes before compilation – this is done automatically by ``cmake``. + +Preparing a suite definition file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The suite definition file is a text file read by the model at compile +time. It is used to specify the physical parameterization suite, and +includes information about the number of parameterization groupings, +which parameterizations that are part of each of the groups, the order +in which the parameterizations should be run, and whether subcycling +will be used to run any of the parameterizations with shorter timesteps. + +In addition to the six or so major parameterization categories (such as +radiation, boundary layer, deep convection, resolved moist physics, +etc.), the suite definition file can also have an arbitrary number of +additional interstitial schemes in between the parameterizations to +prepare or postprocess data. In many models, this interstitial code is +not known to the model user but with the suite definition file, both the +physical parameterizations and the interstitial processing are listed +explicitly. + +For this release, supported suite definition files used with this SCM +are found in ``ccpp-scm/ccpp/suites`` and have default namelist, tracer configuration, and +timesteps attached in ``ccpp-scm/scm/src/suite_info.py``. For all of these suites, the physics schemes +have been organized into 3 groupings following how the physics are +called in the UFS Atmosphere model, although no code is executed in the +SCM time loop between execution of the grouped schemes. Several +“interstitial” schemes are included in the suite definition file to +execute code that previously was part of a hard-coded physics driver. +Some of these schemes may eventually be rolled into the schemes +themselves, improving portability. + +Initializing/running a suite +---------------------------- + +The process for initializing and running a suite in this SCM is +described in sections +:numref:`%s ` and :numref:`%s