From 1abeba0aed1ab04e30f6f74d74ea45336618e9e2 Mon Sep 17 00:00:00 2001 From: "Edward.Safford" Date: Thu, 1 Dec 2022 13:32:36 +0000 Subject: [PATCH] Ref #28 Work in progress. --- modulefiles/wcoss2-run.lua | 1 + src/Conventional_Monitor/ConMon_install.pl | 326 ------------------ .../data_extract/ush/ConMon_DE.sh | 49 +-- src/Conventional_Monitor/get_hostname.pl | 35 -- ush/find_last_cycle.sh | 8 +- 5 files changed, 31 insertions(+), 388 deletions(-) delete mode 100755 src/Conventional_Monitor/ConMon_install.pl delete mode 100755 src/Conventional_Monitor/get_hostname.pl diff --git a/modulefiles/wcoss2-run.lua b/modulefiles/wcoss2-run.lua index 78729264..4e410e90 100644 --- a/modulefiles/wcoss2-run.lua +++ b/modulefiles/wcoss2-run.lua @@ -11,6 +11,7 @@ load ("GrADS/2.2.2") prepend_path("MODULEPATH", "/apps/ops/para/nco/modulefiles/core/") load ("prod_util/2.0.13") +load ("prod_envir/2.0.6") load("common-run") diff --git a/src/Conventional_Monitor/ConMon_install.pl b/src/Conventional_Monitor/ConMon_install.pl deleted file mode 100755 index 0100f3dd..00000000 --- a/src/Conventional_Monitor/ConMon_install.pl +++ /dev/null @@ -1,326 +0,0 @@ -#! /usr/bin/perl - -#------------------------------------------------------------------- -# -# ConMon_install.pl -# -# This script makes sets all necessary configuration definitions -# and calls the makeall.sh script to build all the necessary -# executables. This script works for wcoss, theia, and cray -# machines. -#------------------------------------------------------------------- - - use IO::File; - use File::Copy qw(move); - - print "--> ConMon_install.sh\n"; - - my $machine = `/usr/bin/perl ./get_hostname.pl`; - my $my_machine="export MY_MACHINE=$machine"; - - if( $machine ne "hera" && $machine ne "wcoss" && $machine ne "wcoss_c" - && $machine ne "wcoss_d" && $machine ne "wcoss2" ) { - die( "ERROR --- Unrecognized machine hostname, $machine. Exiting now...\n" ); - } - else { - print "machine = $machine\n"; - } - - #--------------------------------------------------------------------------------- - # - # All 3 currently supported platforms are little endian machines and linux OSes. - # I'm keeping these switches though because that will surely change at some - # point and I'll just have to re-introduce the same switches. - # - my $little_endian = "export LITTLE_ENDIAN=1"; - - my $os = "linux"; - my $my_os = "export MY_OS=$os"; - - - #--------------------------------------------------------------- - # - # Idenfity basedir location of package - # - print "\n"; - print "locating and saving ConMon package location\n"; - my $conmon_dir; - $conmon_dir = `dirname $0`; - $conmon_dir =~ s/^\s+|\s+$//g; - - if( $conmon_dir eq "." ) { - $conmon_dir = `pwd`; - $conmon_dir =~ s/^\s+|\s+$//g; - } - my $my_conmon = "export MY_CONMON=$conmon_dir"; - print "my_conmon = $my_conmon \n"; - print"\n\n"; - - sleep( 1 ); - - #--------------------------------------------------------------- - # - # TANKDIR location - # - my $user_name = $ENV{ 'USER' }; - - if( $machine eq "hera" ) { - $tankdir = "/scratch1/NCEPDEV/da/$user_name/save/nbns"; - } - elsif( $machine eq "wcoss" ) { - $tankdir = "/global/save/$user_name/nbns"; - } - elsif( $machine eq "wcoss_c" ) { - $tankdir = "/gpfs/hps/emc/da/noscrub/$user_name" - } - elsif( $machine eq "wcoss_d" ) { - $tankdir = "/gpfs/dell2/emc/modeling/noscrub/$user_name/nbns"; - } - elsif( $machine eq "wcoss2" ){ - $tankdir = "/lfs/h2/emc/da/noscrub/$user_name/nbns"; - } - - print "Please specify TANKDIR location for storage of data and image files.\n"; - print " Return to accept default location or enter new location now.\n"; - print "\n"; - print " Default TANKDIR: $tankdir \n"; - print " ?\n"; - my $new_tankdir = <>; - $new_tankdir =~ s/^\s+|\s+$//g; - - if( length($new_tankdir ) > 0 ) { - $tankdir = $new_tankdir; - } - my $my_tankdir="export CONMON_TANKDIR=$tankdir"; - print "my_tankdir = $my_tankdir\n"; - print "\n\n"; - sleep( 1 ); - - - #--------------------------------------------------------------- - # - # Web sever name - # - my $server = "emcrzdm"; - print "Please specify web server name.\n"; - print " Return to accept default server name or enter new server name.\n"; - print " \n"; - print " Default web server: $server\n"; - print " ?\n"; - my $new_server =<>; - $new_server =~ s/^\s+|\s+$//g; - if( length($new_server ) > 0 ) { - $server = $new_server; - } - my $my_server="export WEBSVR=$server"; - print "my_server = $my_server\n"; - print "\n\n"; - sleep( 1 ); - - - #--------------------------------------------------------------- - # - # Web server user name - # - my $webuser = $ENV{ 'USER' }; - print "Please specify your user name on the $server server.\n"; - print " Return to accept default user name or enter new user name.\n"; - print " \n"; - print " Default user name on $server: $webuser\n"; - print " ?\n"; - my $new_webuser =<>; - $new_webuser =~ s/^\s+|\s+$//g; - if( length($new_webuser ) > 0 ) { - $webuser = $new_webuser; - } - my $my_webuser="export WEBUSER=$webuser"; - print "my_webuser = $my_webuser\n"; - print "\n\n"; - sleep( 1 ); - - - #--------------------------------------------------------------- - # - # Web directory - # - my $webdir = "/home/people/emc/www/htdocs/gmb/gdas/es_conv"; - print "Please specify the top level web site directory $server.\n"; - print " Return to accept default directory location or enter new location.\n"; - print " \n"; - print " Default directory on $server: $webdir\n"; - print " ?\n"; - my $new_webdir =<>; - $new_webdir =~ s/^\s+|\s+$//g; - if( length($new_webdir ) > 0 ) { - $webdir = $new_webdir; - } - my $my_webdir="export WEBDIR=$webdir"; - print "my_webdir = $my_webdir\n"; - print "\n\n"; - sleep( 1 ); - - - #---------------------------------------------------- - # Set up ptmp and stmp locations according to $arch. - # - my $ptmp = "/ptmpd1"; - my $stmp = "/stmpd1"; - my $my_ptmp = "export C_PTMP=\${C_PTMP:-$ptmp}"; - my $my_stmp = "export C_STMP=\${C_STMP:-$stmp}"; - - if( $machine eq "hera" ) { - $my_ptmp="export C_PTMP=\${C_PTMP:-/scratch2/NCEPDEV/stmp3}"; - $my_stmp="export C_STMP=\${C_STMP:-/scratch2/NCEPDEV/stmp1}"; - } - elsif( $machine eq "wcoss_c" ) { - $my_ptmp="export C_PTMP=\${C_PTMP:-/gpfs/hps/ptmp}"; - $my_stmp="export C_STMP=\${C_STMP:-/gpfs/hps/stmp}"; - } - elsif( $machine eq "wcoss_d" ) { - $my_ptmp="export C_PTMP=\${C_PTMP:-/gpfs/dell2/ptmp}"; - $my_stmp="export C_STMP=\${C_STMP:-/gpfs/dell2/stmp}"; - } - elsif( $machine eq "wcoss2" ){ - $my_ptmp="export MY_PTMP=\${MY_PTMP:-/lfs/h2/emc/ptmp}"; - $my_stmp="export MY_STMP=\${MY_STMP:-/lfs/h2/emc/stmp}"; - } - - #--------------------------------------- - # - # wcoss has several options available: - # - else { - print "Please specify PTMP location. This is used for temporary work space.\n"; - print " Available options are: \n"; - print " /ptmpd1 (default)\n"; - print " /ptmpd2\n"; - print " /ptmpd3\n"; - print " /ptmpp1\n"; - print " /ptmpp2\n"; - - print " Return to accept default location or enter new location now.\n"; - print "\n"; - print " Default PTMP: $ptmp \n"; - print " ?\n"; - my $new_ptmp = <>; - $new_ptmp =~ s/^\s+|\s+$//g; - - if( length($new_ptmp ) > 0 ) { - $ptmp = $new_ptmp; - } - $my_ptmp="export C_PTMP=\${C_PTMP:-$ptmp}"; - print "\n\n"; - sleep( 1 ); - - - print "Please specify STMP location. This is used for temporary work space.\n"; - print " Available options are: \n"; - print " /stmpd1 (default)\n"; - print " /stmpd2\n"; - print " /stmpd3\n"; - print " /stmpp1\n"; - print " /stmpp2\n"; - - print " Return to accept default location or enter new location now.\n"; - print "\n"; - print " Default STMP: $stmp \n"; - print " ?\n"; - my $new_stmp = <>; - $new_stmp =~ s/^\s+|\s+$//g; - - if( length($new_stmp ) > 0 ) { - $stmp = $new_stmp; - } - $my_stmp="export C_STMP=\${C_STMP:-$stmp}"; - print "my_stmp = $my_stmp\n"; - print "\n\n"; - sleep( 1 ); - } - - print "my_ptmp = $my_ptmp\n"; - print "my_stmp = $my_stmp\n"; - print "\n"; - - - my $account = "export ACCOUNT=\${ACCOUNT:-}"; - if( $machine eq "hera" ) { - $account = "export ACCOUNT=\${ACCOUNT:-fv3-cpu}"; - } elsif( $machine eq "wcoss2" ){ - $account = "export ACCOUNT=\${ACCOUNT:-GFS-DEV}"; - } - - my $project = "export PROJECT=\${PROJECT:-GDAS-T2O}"; - if( $machine eq "wcoss2" ){ - $project = "export PROJECT=\${PROJECT:-GDAS-DEV}"; - } elsif( $machine ne "wcoss" && $machine ne "cray" && $machine ne "wcoss_d" ) { - $project="export PROJECT="; - } - - my $job_queue="export JOB_QUEUE="; - if( $machine eq "cray" || $machine eq "wcoss2" ) { - $job_queue="export JOB_QUEUE=\${JOB_QUEUE:-dev}"; - } elsif( $machine eq "wcoss" || $machine eq "wcoss_d" ){ - $job_queue = "export JOB_QUEUE=\${JOB_QUEUE:-dev_shared}"; - } - - #------------------------------------------------------------ - # - # Update the config file with the configuration information - # - my $config = "parm/ConMon_config"; - open my $in, '<', $config or die "Can't read $config $!"; - open my $out, '>', "$config.new" or die "Can't write $config.new: $!"; - - while( <$in> ) { - if( $_ =~ "MY_CONMON=" ) { - print $out "$my_conmon\n"; - } - elsif( $_ =~ "ACCOUNT=" ) { - print $out "$account\n"; - } - elsif( $_ =~ "PROJECT=" ) { - print $out "$project\n"; - } - elsif( $_ =~ "JOB_QUEUE=" ) { - print $out "$job_queue\n"; - } - elsif( $_ =~ "CONMON_TANKDIR=" ) { - print $out "$my_tankdir\n"; - } - elsif( $_ =~ "WEBSVR=" ) { - print $out "$my_server\n"; - } - elsif( $_ =~ "WEBUSER=" ) { - print $out "$my_webuser\n"; - } - elsif( $_ =~ "WEBDIR=" ) { - print $out "$my_webdir\n"; - } - elsif( $_ =~ "LITTLE_ENDIAN=" ) { - print $out "$little_endian\n"; - } - elsif( $_ =~ "MY_OS=" ) { - print $out "$my_os\n"; - } - elsif( $_ =~ "MY_MACHINE=" ) { - print $out "$my_machine\n"; - } - elsif( $_ =~ "C_PTMP=" ) { - print $out "$my_ptmp\n"; - } - elsif( $_ =~ "C_STMP=" ) { - print $out "$my_stmp\n"; - } - else { - print $out $_; - } - } - close $out; - close $in; - - move "$config.new", $config; - - - print "<-- ConMon_install.sh\n"; - -exit 0; diff --git a/src/Conventional_Monitor/data_extract/ush/ConMon_DE.sh b/src/Conventional_Monitor/data_extract/ush/ConMon_DE.sh index bf9e48cf..df1a0b64 100755 --- a/src/Conventional_Monitor/data_extract/ush/ConMon_DE.sh +++ b/src/Conventional_Monitor/data_extract/ush/ConMon_DE.sh @@ -1,4 +1,4 @@ -#!/bin/sh -l +#!/bin/bash #-------------------------------------------------------------------- # @@ -31,7 +31,7 @@ function usage { #-------------------------------------------------------------------- # ConMon_DE.sh begins here #-------------------------------------------------------------------- -set -ax + echo "Begin ConMon_DE.sh" nargs=$# @@ -46,6 +46,8 @@ fi # export RUN=gdas +PDATE="" +cnvstat_location="" while [[ $# -ge 1 ]] do @@ -62,7 +64,7 @@ do shift # past argument ;; -c|--cnv) - export CNVSTAT_LOCATION="$2" + cnvstat_location="$2" shift # past argument ;; *) @@ -79,14 +81,9 @@ this_file=`basename $0` this_dir=`dirname $0` -#-------------------------------------------------------------------- -# RUN_ENVIR: can be "dev", "para", or "prod". -#-------------------------------------------------------------------- -export RUN_ENVIR=${RUN_ENVIR:-"prod"} - +gfs_ver=v16.2 echo CONMON_SUFFIX = $CONMON_SUFFIX -echo RUN_ENVIR = $RUN_ENVIR export NET=${CONMON_SUFFIX} top_parm=${this_dir}/../../parm @@ -109,6 +106,8 @@ jobname=ConMon_DE_${CONMON_SUFFIX} echo "C_TANKDIR = ${C_TANKDIR}" echo "C_LOGDIR = ${C_LOGDIR}" echo "C_IMGNDIR = ${C_IMGNDIR}" +echo "MON_USH = $MON_USH" + if [[ ! -d ${C_TANKDIR} ]]; then mkdir -p ${C_TANKDIR} fi @@ -123,8 +122,8 @@ fi #-------------------------------------------------------------------- # Get date of cycle to process and/or previous cycle processed. # -if [[ $PDATE = "" ]]; then - GDATE=`${C_DE_SCRIPTS}/find_cycle.pl --cyc 1 --dir ${C_TANKDIR} --run $RUN ` +if [[ ${#PDATE} -le 0 ]]; then + GDATE=`${MON_USH}/find_last_cycle.sh --net ${CONMON_SUFFIX} --run ${RUN} --tank ${TANKDIR} --mon conmon` PDATE=`$NDATE +06 $GDATE` else GDATE=`$NDATE -06 $PDATE` @@ -138,24 +137,23 @@ export CYC=`echo $PDATE|cut -c9-10` export GCYC=`echo $GDATE|cut -c9-10` export PDYm6h=`echo $GDATE|cut -c1-8` - -if [[ $MY_MACHINE == "hera" ]]; then - export CNVSTAT_LOCATION=${CNVSTAT_LOCATION:-/scratch1/NCEPDEV/da/Edward.Safford/noscrub/test_data} -else - export CNVSTAT_LOCATION=${CNVSTAT_LOCATION:-${COMROOTp3}/gfs/${RUN_ENVIR}} +if [[ ${#cnvstat_location} -le 0 ]]; then + export cnvstat_location=${COMROOT}/gfs/${gfs_ver} fi +export CNVSTAT_LOCATION=${cnvstat_location} +echo "CNVSTAT_LOCATION = $CNVSTAT_LOCATION" export COMPONENT=${COMPONENT:-atmos} -export C_DATDIR=${C_DATDIR:-${CNVSTAT_LOCATION}/${RUN}.${PDY}/${CYC}/${COMPONENT}} -if [[ ! -d ${C_DATDIR} ]]; then - export C_DATDIR=${CNVSTAT_LOCATION}/${RUN}.${PDY}/${CYC} -fi +# +# Is there any difference in location from ops to wkfl? +# +export C_DATDIR=${C_DATDIR:-${CNVSTAT_LOCATION}/${RUN}.${PDY}/${CYC}/${COMPONENT}} export C_GDATDIR=${C_GDATDIR:-${CNVSTAT_LOCATION}/${RUN}.${PDYm6h}/${GCYC}/${COMPONENT}} -if [[ ! -d ${C_GDATDIR} ]]; then - export C_GDATDIR=${CNVSTAT_LOCATION}/${RUN}.${PDYm6h}/${GCYC} -fi + +echo "C_DATDIR = $C_DATDIR" +echo "C_GDATDIR = $C_GDATDIR" export C_COMIN=${C_DATDIR} export C_COMINm6h=${C_GDATDIR} @@ -190,6 +188,11 @@ if [[ ! -s ${pgrbf06} ]]; then export pgrbf06="${C_GDATDIR}/gdas.t${GCYC}z.pgrb2.1p00.f006" fi +echo "cnvstat = $cnvstat" +echo "pgrbf00 = $pgrbf00" +echo "pgrbf06 = $pgrbf06" + +exit #--------------------------------------------- # override the default convinfo definition diff --git a/src/Conventional_Monitor/get_hostname.pl b/src/Conventional_Monitor/get_hostname.pl deleted file mode 100755 index 1d6e9316..00000000 --- a/src/Conventional_Monitor/get_hostname.pl +++ /dev/null @@ -1,35 +0,0 @@ -#! /usr/bin/perl - -#----------------------------------------------------------------------- -# get_hostname.pl -# -# This script determines the hostname of the current machine. The -# possiblities are wcoss, wcoss_c, wcoss_d, or hera. A null -# string ("") if the machine is not recognized. -# -# NOTE: shell scripts call this and then read the output string -# which is in $machine. So don't leave uncommented debug -# statements in place here; that will confuse calling scripts. -#----------------------------------------------------------------------- - - my $machine = ""; - - if (-d "/dcom" and -d "/hwrf") { - $machine = "wcoss"; - } - elsif( -d "/cm" ) { - $machine = "wcoss_c"; - } - elsif( -d "/ioddev_dell" ) { - $machine = "wcoss_d"; - } - elsif( -d "/scratch1" ) { - $machine = "hera"; - } - elsif( -d "/lfs/h2" ) { - $machine = "wcoss2"; - } - - print "$machine"; - - exit 0; diff --git a/ush/find_last_cycle.sh b/ush/find_last_cycle.sh index 9842913b..fdcef2a5 100755 --- a/ush/find_last_cycle.sh +++ b/ush/find_last_cycle.sh @@ -79,7 +79,7 @@ if [[ ${#path} -gt 0 ]]; then # options for ops, wkfl, and mon, halting when the first valid # output files are found. # - flist=`find "${path}" -maxdepth 1 -mindepth 1 -type d -name "${run}.*" -printf "%f\n" 2>/dev/null` + flist=`find -L "${path}" -maxdepth 1 -mindepth 1 -type d -name "${run}.*" -printf "%f\n" 2>/dev/null` sorted=`echo ${flist[@]} | awk 'BEGIN{RS=" ";} {print $1}' | sort -r` lcyc="" @@ -87,9 +87,9 @@ if [[ ${#path} -gt 0 ]]; then for hr in $hrs; do - mon_test=`find "${path}/${file}/${hr}/${monitor}" -maxdepth 2 -mindepth 1 -name "*.ieee_d" -printf "%f\n" 2>/dev/null` - wkfl_test=`find "${path}/${file}/${hr}/" -maxdepth 2 -mindepth 1 -name "*.ieee_d" -printf "%f\n" 2>/dev/null` - ops_test=`find "${path}/${file}/${hr}/atmos/${monitor}" -maxdepth 2 -mindepth 1 -name "*.ieee_d" -printf "%f\n" 2>/dev/null` + mon_test=`find -L "${path}/${file}/${hr}/${monitor}" -maxdepth 2 -mindepth 1 -name "*.ieee_d" -printf "%f\n" 2>/dev/null` + wkfl_test=`find -L "${path}/${file}/${hr}/" -maxdepth 2 -mindepth 1 -name "*.ieee_d" -printf "%f\n" 2>/dev/null` + ops_test=`find -L "${path}/${file}/${hr}/atmos/${monitor}" -maxdepth 2 -mindepth 1 -name "*.ieee_d" -printf "%f\n" 2>/dev/null` if [[ ${#mon_test} -gt 0 || ${#ops_test} -gt 0 || ${#wkfl_test} -gt 0 ]]; then lcyc=`echo $file | gawk -F. '{print $2}'`