From 45b3cdf1b76e5a3dfa958099601f74828434bd04 Mon Sep 17 00:00:00 2001 From: Eric Kemp Date: Mon, 13 Dec 2021 16:26:20 -0500 Subject: [PATCH] Bug fixes for autotuning Bratseth error settings. * Updated compile script to reflect new path to Fortran codes. * Updated Discover batch script for new environment module. * Updates to disable GEOPRECIP, SSMI, and IMERG. * Changed first guess settings for fitting semivariogram. * Tweaked blacklist settings for 10-m wind speed and 2-m temperature to weed out more questionable data. --- lis/compile | 2 +- .../usaf/retune_bratseth/cfgs/autotune.cfg | 12 +-- .../retune_bratseth/cfgs/blacklist_spd10m.cfg | 2 +- .../retune_bratseth/cfgs/blacklist_t2m.cfg | 2 +- .../usaf/retune_bratseth/scripts/autotune.py | 95 ++++++++++--------- .../scripts/fit_semivariogram.py | 5 +- .../scripts/run_autotune_discover.sh | 12 ++- 7 files changed, 71 insertions(+), 59 deletions(-) diff --git a/lis/compile b/lis/compile index 72b8ace17..3b036437f 100755 --- a/lis/compile +++ b/lis/compile @@ -116,7 +116,7 @@ fi # Compile standalone Fortran utilities for tuning Air Force Bratseth scheme. if [ "$compile_autotune" = "YES" ]; then echo "Compiling Air Force Bratseth tuning utilities" - if cd utils/afwa/retune_bratseth/src + if cd utils/usaf/retune_bratseth/src # EMK Fix path then echo "" make -f Makefile diff --git a/lis/utils/usaf/retune_bratseth/cfgs/autotune.cfg b/lis/utils/usaf/retune_bratseth/cfgs/autotune.cfg index b5c196e2d..87030daa8 100644 --- a/lis/utils/usaf/retune_bratseth/cfgs/autotune.cfg +++ b/lis/utils/usaf/retune_bratseth/cfgs/autotune.cfg @@ -3,14 +3,14 @@ # [Input] -workdir: /discover/nobackup/emkemp/AFWA/autoretune/work +workdir: /discover/nobackup/projects/lis_aist17/emkemp/AFWA/lis74_bratseth_retuning_for_ops/2021 -scriptdir: /discover/nobackup/emkemp/AFWA/autoretune/LISF/lis/utils/usaf/retune_bratseth/scripts +scriptdir: /discover/nobackup/projects/lis_aist17/emkemp/AFWA/lis74_bratseth_retuning_for_ops/build/LISF/lis/utils/usaf/retune_bratseth/scripts -cfgdir: /discover/nobackup/emkemp/AFWA/autoretune/LISF/lis/utils/usaf/retune_bratseth/cfgs +cfgdir: /discover/nobackup/projects/lis_aist17/emkemp/AFWA/lis74_bratseth_retuning_for_ops/build/LISF/lis/utils/usaf/retune_bratseth/cfgs -bindir: /discover/nobackup/emkemp/AFWA/autoretune/LISF/lis/utils/usaf/retune_bratseth/src +bindir: /discover/nobackup/projects/lis_aist17/emkemp/AFWA/lis74_bratseth_retuning_for_ops/build/LISF/lis/utils/usaf/retune_bratseth/src -lis_cfg_template: /discover/nobackup/emkemp/AFWA/autoretune/LISF/lis/configs/557WW-7.3-FOC/lis.config.global.jules50.ops +lis_cfg_template: /discover/nobackup/projects/lis_aist17/emkemp/AFWA/lis74_bratseth_retuning_for_ops/build/LISF/lis/configs/557WW-7.4-FOC/NRT_GLOBAL/lis.config.global.noah39.ops -varlist: rh2m,spd10m,t2m,gage,cmorph,geoprecip,imerg,ssmi +varlist: rh2m,spd10m,t2m,gage,imerg diff --git a/lis/utils/usaf/retune_bratseth/cfgs/blacklist_spd10m.cfg b/lis/utils/usaf/retune_bratseth/cfgs/blacklist_spd10m.cfg index 1e291702e..ca1c81d36 100644 --- a/lis/utils/usaf/retune_bratseth/cfgs/blacklist_spd10m.cfg +++ b/lis/utils/usaf/retune_bratseth/cfgs/blacklist_spd10m.cfg @@ -5,7 +5,7 @@ [Input] network_thresh: 1 count_thresh: 28 -omb_thresh: 3 +omb_thresh: 1.5 data_directory: spd10m_OBA data_frequency: 01 diff --git a/lis/utils/usaf/retune_bratseth/cfgs/blacklist_t2m.cfg b/lis/utils/usaf/retune_bratseth/cfgs/blacklist_t2m.cfg index 10fd18c1e..c9f352131 100644 --- a/lis/utils/usaf/retune_bratseth/cfgs/blacklist_t2m.cfg +++ b/lis/utils/usaf/retune_bratseth/cfgs/blacklist_t2m.cfg @@ -5,7 +5,7 @@ [Input] network_thresh: 1 count_thresh: 28 -omb_thresh: 5 +omb_thresh: 2.5 data_directory: t2m_OBA data_frequency: 01 diff --git a/lis/utils/usaf/retune_bratseth/scripts/autotune.py b/lis/utils/usaf/retune_bratseth/scripts/autotune.py index c58d30838..4b58436d7 100755 --- a/lis/utils/usaf/retune_bratseth/scripts/autotune.py +++ b/lis/utils/usaf/retune_bratseth/scripts/autotune.py @@ -8,6 +8,7 @@ REVISION HISTORY: 03 Nov 2020: Eric Kemp. Initial specification. 16 Dec 2020: Eric Kemp. Now calls create_blacklist via function call. +13 Dec 2021: Eric Kemp. Added checks for missing satellite data. """ import configparser @@ -272,49 +273,57 @@ def assemble_new_lines(self): line += " %s\n" %(self.sigma2o["gage"]) lines.append(line) - if self.Lo["geoprecip"] > 0: - line = \ - "AGRMET GALWEM Precip GEOPRECIP observation error scale length (m):" - line += " %s\n" %(self.Lo["geoprecip"]) - lines.append(line) - - if self.sigma2o["geoprecip"] > 0: - line = "AGRMET GALWEM Precip GEOPRECIP observation error variance:" - line += " %s\n" %(self.sigma2o["geoprecip"]) - lines.append(line) - - if self.Lo["ssmi"] > 0: - line = \ - "AGRMET GALWEM Precip SSMI observation error scale length (m):" - line += " %s\n" %(self.Lo["ssmi"]) - lines.append(line) - - if self.sigma2o["ssmi"] > 0: - line = "AGRMET GALWEM Precip SSMI observation error variance:" - line += " %s\n" %(self.sigma2o["ssmi"]) - lines.append(line) - - if self.Lo["cmorph"] > 0: - line = \ - "AGRMET GALWEM Precip CMORPH observation error scale length (m):" - line += " %s\n" %(self.Lo["cmorph"]) - lines.append(line) - - if self.sigma2o["cmorph"] > 0: - line = "AGRMET GALWEM Precip CMORPH observation error variance:" - line += " %s\n" %(self.sigma2o["cmorph"]) - lines.append(line) - - if self.Lo["imerg"] > 0: - line = \ - "AGRMET GALWEM Precip IMERG observation error scale length (m)" - line += " %s\n" %(self.Lo["imerg"]) - lines.append(line) - - if self.sigma2o["imerg"] > 0: - line = "AGRMET GALWEM Precip IMERG observation error variance:" - line += " %s\n" %(self.sigma2o["imerg"]) - lines.append(line) + if "geoprecip" in self.Lo: + if self.Lo["geoprecip"] > 0: + line = \ + "AGRMET GALWEM Precip GEOPRECIP observation error scale length (m):" + line += " %s\n" %(self.Lo["geoprecip"]) + lines.append(line) + + if "geoprecip" in self.sigma2o: + if self.sigma2o["geoprecip"] > 0: + line = "AGRMET GALWEM Precip GEOPRECIP observation error variance:" + line += " %s\n" %(self.sigma2o["geoprecip"]) + lines.append(line) + + if "ssmi" in self.Lo: + if self.Lo["ssmi"] > 0: + line = \ + "AGRMET GALWEM Precip SSMI observation error scale length (m):" + line += " %s\n" %(self.Lo["ssmi"]) + lines.append(line) + + if "ssmi" in self.sigma2o: + if self.sigma2o["ssmi"] > 0: + line = "AGRMET GALWEM Precip SSMI observation error variance:" + line += " %s\n" %(self.sigma2o["ssmi"]) + lines.append(line) + + if "cmorph" in self.Lo: + if self.Lo["cmorph"] > 0: + line = \ + "AGRMET GALWEM Precip CMORPH observation error scale length (m):" + line += " %s\n" %(self.Lo["cmorph"]) + lines.append(line) + + if "cmorph" in self.sigma2o: + if self.sigma2o["cmorph"] > 0: + line = "AGRMET GALWEM Precip CMORPH observation error variance:" + line += " %s\n" %(self.sigma2o["cmorph"]) + lines.append(line) + + if "imerg" in self.Lo: + if self.Lo["imerg"] > 0: + line = \ + "AGRMET GALWEM Precip IMERG observation error scale length (m)" + line += " %s\n" %(self.Lo["imerg"]) + lines.append(line) + + if "imerg" in self.sigma2o: + if self.sigma2o["imerg"] > 0: + line = "AGRMET GALWEM Precip IMERG observation error variance:" + line += " %s\n" %(self.sigma2o["imerg"]) + lines.append(line) if self.Lb["t2m"] > 0: line = "AGRMET GALWEM T2M background error scale length (m):" diff --git a/lis/utils/usaf/retune_bratseth/scripts/fit_semivariogram.py b/lis/utils/usaf/retune_bratseth/scripts/fit_semivariogram.py index 4ed93923f..ed871e523 100755 --- a/lis/utils/usaf/retune_bratseth/scripts/fit_semivariogram.py +++ b/lis/utils/usaf/retune_bratseth/scripts/fit_semivariogram.py @@ -10,6 +10,7 @@ 26 Oct 2020: Eric Kemp. Initial Specification. 03 Nov 2020: Eric Kemp. Removed plotting. Fitted parameters are now saved to file. +13 Dec 2021: Eric Kemp. Changed first guess for covariance settings. """ # Standard library @@ -56,8 +57,8 @@ def usage(): # Fit function # Comparing gages to background field. Three parameters must be fit. -sigma2_gage_guess = np.amin(variovector) -sigma2_back_guess = np.amax(variovector) - sigma2_gage_guess +sigma2_gage_guess = (np.amin(variovector) + np.amax(variovector)) / 2. +sigma2_back_guess = sigma2_gage_guess L_back_guess = distvector[2] fit_func = semivar.fit_func_dict[function_type] sigma2_gage_min = 0.1*sigma2_gage_guess diff --git a/lis/utils/usaf/retune_bratseth/scripts/run_autotune_discover.sh b/lis/utils/usaf/retune_bratseth/scripts/run_autotune_discover.sh index 965d6369d..c561d4fd7 100644 --- a/lis/utils/usaf/retune_bratseth/scripts/run_autotune_discover.sh +++ b/lis/utils/usaf/retune_bratseth/scripts/run_autotune_discover.sh @@ -24,6 +24,8 @@ # # REVISION HISTORY: # 17 Dec 2020: Eric Kemp. Initial specification. +# 13 Dec 2021: Eric Kemp. Updated module, and removed obsolete satellite +# data feeds. # #------------------------------------------------------------------------------ @@ -39,16 +41,16 @@ fi module purge unset LD_LIBRARY_PATH module use --append /home/emkemp/privatemodules -module load lisf_7_intel_19_1_3_304_traceback-work-around +module load lisf_7_intel_2021.4.0_s2s # Define the variables to be processed NWPVARS=(gage rh2m spd10m t2m) -SATVARS=(cmorph geoprecip imerg ssmi) +SATVARS=(imerg) # Paths on local system -SCRIPTDIR=/discover/nobackup/emkemp/AFWA/autoretune/LISF/lis/utils/usaf/retune_bratseth/scripts -CFGDIR=/discover/nobackup/emkemp/AFWA/autoretune/LISF/lis/utils/usaf/retune_bratseth/cfgs -BINDIR=/discover/nobackup/emkemp/AFWA/autoretune/LISF/lis/utils/usaf/retune_bratseth/src +SCRIPTDIR=/discover/nobackup/projects/lis_aist17/emkemp/AFWA/lis74_bratseth_retuning_for_ops/build/LISF/lis/utils/usaf/retune_bratseth/scripts +CFGDIR=/discover/nobackup/projects/lis_aist17/emkemp/AFWA/lis74_bratseth_retuning_for_ops/build/LISF/lis/utils/usaf/retune_bratseth/cfgs +BINDIR=/discover/nobackup/projects/lis_aist17/emkemp/AFWA/lis74_bratseth_retuning_for_ops/build/LISF/lis/utils/usaf/retune_bratseth/src # Get the command line arguments to specify the training period. if [ -z "$1" ] ; then