Skip to content

Commit

Permalink
Merge pull request #13 from LIVVkit/mkstratos/tsc_merge_fix
Browse files Browse the repository at this point in the history 
Bugfix for new pandas version
  • Loading branch information
@mkstratos
mkstratos authored Apr 17, 2024
2 parents b0d023d + 1650321 commit 43e3117
Show file tree
Hide file tree
Showing 9 changed files with 1,022 additions and 585 deletions.
1 change: 1 addition & 0 deletions .git-blame-ignore-revs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0e03e4395fae0aa0b6ab475feb054e53135f2352
59 changes: 37 additions & 22 deletions evv4esm/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,35 +28,50 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


__version_info__ = (0, 5, 0)
__version__ = '.'.join(str(vi) for vi in __version_info__)
__version_info__ = (0, 5, 1)
__version__ = ".".join(str(vi) for vi in __version_info__)

PASS_COLOR = '#389933'
L_PASS_COLOR = '#93DA90'
PASS_COLOR = "#389933"
L_PASS_COLOR = "#93DA90"

FAIL_COLOR = '#BF3F46'
L_FAIL_COLOR = '#E68388'
FAIL_COLOR = "#BF3F46"
L_FAIL_COLOR = "#E68388"

human_color_names = {'pass': ('green', 'light green'),
PASS_COLOR: 'green',
L_PASS_COLOR: 'light green',
'fail': ('red', 'light red'),
FAIL_COLOR: 'red',
L_FAIL_COLOR: 'light red'}
human_color_names = {
"pass": ("green", "light green"),
PASS_COLOR: "green",
L_PASS_COLOR: "light green",
"fail": ("red", "light red"),
FAIL_COLOR: "red",
L_FAIL_COLOR: "light red",
}

pf_color_picker = {'Pass': PASS_COLOR, 'pass': PASS_COLOR,
'Accept': PASS_COLOR, 'accept': PASS_COLOR,
'Fail': FAIL_COLOR, 'fail': FAIL_COLOR,
'Reject': FAIL_COLOR, 'reject': FAIL_COLOR}
pf_color_picker = {
"Pass": PASS_COLOR,
"pass": PASS_COLOR,
"Accept": PASS_COLOR,
"accept": PASS_COLOR,
"Fail": FAIL_COLOR,
"fail": FAIL_COLOR,
"Reject": FAIL_COLOR,
"reject": FAIL_COLOR,
}

light_pf_color_picker = {'Pass': L_PASS_COLOR, 'pass': L_PASS_COLOR,
'Accept': L_PASS_COLOR, 'accept': L_PASS_COLOR,
PASS_COLOR: L_PASS_COLOR,
'Fail': L_FAIL_COLOR, 'fail': L_FAIL_COLOR,
'Reject': L_FAIL_COLOR, 'reject': L_FAIL_COLOR,
FAIL_COLOR: L_FAIL_COLOR}
light_pf_color_picker = {
"Pass": L_PASS_COLOR,
"pass": L_PASS_COLOR,
"Accept": L_PASS_COLOR,
"accept": L_PASS_COLOR,
PASS_COLOR: L_PASS_COLOR,
"Fail": L_FAIL_COLOR,
"fail": L_FAIL_COLOR,
"Reject": L_FAIL_COLOR,
"reject": L_FAIL_COLOR,
FAIL_COLOR: L_FAIL_COLOR,
}


class EVVException(Exception):
"""Base class for EVV exceptions"""

pass
139 changes: 84 additions & 55 deletions evv4esm/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,21 @@
# coding=utf-8
# Copyright (c) 2018 UT-BATTELLE, LLC
# All rights reserved.
#
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
Expand All @@ -39,53 +39,72 @@


def parse_args(args=None):
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)

parser.add_argument('-e', '--extensions',
action='store',
nargs='+',
default=None,
help='Specify the location of the JSON configuration files for the extended V&V tests to run.')

parser.add_argument('-o', '--out-dir',
default=os.path.join(os.getcwd(), "vv_" + time.strftime("%Y-%m-%d")),
help='Location to output the EVV webpages.')

parser.add_argument('-s', '--serve',
nargs='?', type=int, const=8000,
help=' '.join(['Start a simple HTTP server for the output website specified',
'by OUT_DIR on port SERVE.'
])
)

parser.add_argument('-p', '--pool-size',
nargs='?',
type=int,
default=(options.mp.cpu_count() - 1 or 1),
help='The number of multiprocessing processes to run '
'analyses in. If zero, processes will run serially '
'outside of the multiprocessing module.')

parser.add_argument('--version',
action='version',
version='EVV {}'.format(evv4esm.__version__),
help="Show EVV's version number and exit"
)
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter
)

parser.add_argument(
"-e",
"--extensions",
action="store",
nargs="+",
default=None,
help="Specify the location of the JSON configuration files for the extended V&V tests to run.",
)

parser.add_argument(
"-o",
"--out-dir",
default=os.path.join(os.getcwd(), "vv_" + time.strftime("%Y-%m-%d")),
help="Location to output the EVV webpages.",
)

parser.add_argument(
"-s",
"--serve",
nargs="?",
type=int,
const=8000,
help=" ".join(
[
"Start a simple HTTP server for the output website specified",
"by OUT_DIR on port SERVE.",
]
),
)

parser.add_argument(
"-p",
"--pool-size",
nargs="?",
type=int,
default=(options.mp.cpu_count() - 1 or 1),
help="The number of multiprocessing processes to run "
"analyses in. If zero, processes will run serially "
"outside of the multiprocessing module.",
)

parser.add_argument(
"--version",
action="version",
version="EVV {}".format(evv4esm.__version__),
help="Show EVV's version number and exit",
)

args = parser.parse_args(args)

if args.extensions:
options.parse_args(['-V']+args.extensions + ['-o', args.out_dir])
options.parse_args(["-V"] + args.extensions + ["-o", args.out_dir])

from evv4esm import resources

args.livv_resource_dir = livvkit.resource_dir
livvkit.resource_dir = os.sep.join(resources.__path__)
return args


def main(cl_args=None):
""" Direct execution. """
"""Direct execution."""

if cl_args is None and len(sys.argv) > 1:
cl_args = sys.argv[1:]
Expand Down Expand Up @@ -123,10 +142,14 @@ def main(cl_args=None):
print(" -----------------------------------------------------------------")
print("")
for conf in livvkit.validation_model_configs:
validation_config = functions.merge_dicts(validation_config,
functions.read_json(conf))
summary_elements.extend(scheduler.run_quiet("validation", validation, validation_config,
group=False))
validation_config = functions.merge_dicts(
validation_config, functions.read_json(conf)
)
summary_elements.extend(
scheduler.run_quiet(
"validation", validation, validation_config, group=False
)
)
print(" -----------------------------------------------------------------")
print(" Extensions test suite complete ")
print(" -----------------------------------------------------------------")
Expand All @@ -137,28 +160,34 @@ def main(cl_args=None):
index_data.write(result._repr_json())
print("-------------------------------------------------------------------")
print(" Done! Results can be seen in a web browser at:")
print(" " + os.path.join(livvkit.output_dir, 'index.html'))
print(" " + os.path.join(livvkit.output_dir, "index.html"))
print("-------------------------------------------------------------------")

if args.serve:
import http.server as server
import socketserver as socket

httpd = socket.TCPServer(('', args.serve), server.SimpleHTTPRequestHandler)
httpd = socket.TCPServer(("", args.serve), server.SimpleHTTPRequestHandler)

sa = httpd.socket.getsockname()
print('\nServing HTTP on {host} port {port} (http://{host}:{port}/)'.format(host=sa[0], port=sa[1]))
print('\nView the generated website by navigating to:')
print('\n http://{host}:{port}/{path}/index.html'.format(host=sa[0], port=sa[1],
path=os.path.relpath(args.out_dir)
))
print('\nExit by pressing `ctrl+c` to send a keyboard interrupt.\n')
print(
"\nServing HTTP on {host} port {port} (http://{host}:{port}/)".format(
host=sa[0], port=sa[1]
)
)
print("\nView the generated website by navigating to:")
print(
"\n http://{host}:{port}/{path}/index.html".format(
host=sa[0], port=sa[1], path=os.path.relpath(args.out_dir)
)
)
print("\nExit by pressing `ctrl+c` to send a keyboard interrupt.\n")
try:
httpd.serve_forever()
except KeyboardInterrupt:
print('\nKeyboard interrupt received, exiting.\n')
print("\nKeyboard interrupt received, exiting.\n")
sys.exit(0)


if __name__ == '__main__':
if __name__ == "__main__":
main()
34 changes: 26 additions & 8 deletions evv4esm/ensembles/e3sm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,19 @@ def file_date_str(case_file, style="short", hist_name="h0"):
return result.replace("{}.".format(hist_name), "").replace(".nc", "")


def component_monthly_files(dir_, component, ninst, hist_name="h0", nmonth_max=12, date_style="short"):
def component_monthly_files(
dir_, component, ninst, hist_name="h0", nmonth_max=12, date_style="short"
):
if date_style == "full":
date_search = "????-??-??-??"
elif date_style == "med":
date_search = "????-??-??"
else:
date_search = "????-??"

base = "{d}/*{c}_????.{n}.{ds}.nc".format(d=dir_, c=component, n=hist_name, ds=date_search)
base = "{d}/*{c}_????.{n}.{ds}.nc".format(
d=dir_, c=component, n=hist_name, ds=date_search
)
search = os.path.normpath(base)
result = sorted(glob.glob(search))

Expand Down Expand Up @@ -115,22 +119,36 @@ def gather_monthly_averages(ensemble_files, variable_set=None):
if variable_set is None:
variable_set = set(data.variables.keys())
except OSError as E:
six.raise_from(BaseException('Could not open netCDF dataset: {}'.format(file_)), E)
six.raise_from(
BaseException(
"Could not open netCDF dataset: {}".format(file_)
),
E,
)

for var in data.variables.keys():
if var not in variable_set:
continue
if len(data.variables[var].shape) < 2 or var in ['time_bnds', 'date_written', 'time_written']:
if len(data.variables[var].shape) < 2 or var in [
"time_bnds",
"date_written",
"time_written",
]:
continue
elif 'ncol' not in data.variables[var].dimensions:
elif "ncol" not in data.variables[var].dimensions:
continue
else:
m = np.mean(data.variables[var][0, ...])

desc = "{long_name}{units}".format(**get_variable_meta(data, var))
monthly_avgs.append((case, var, '{:04}'.format(inst), date_str, m, desc))

monthly_avgs = pd.DataFrame(monthly_avgs, columns=('case', 'variable', 'instance', 'date', 'monthly_mean', 'desc'))
monthly_avgs.append(
(case, var, "{:04}".format(inst), date_str, m, desc)
)

monthly_avgs = pd.DataFrame(
monthly_avgs,
columns=("case", "variable", "instance", "date", "monthly_mean", "desc"),
)
return monthly_avgs


Expand Down
10 changes: 8 additions & 2 deletions evv4esm/ensembles/tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,16 @@ def prob_plot(

else:
ax3.hist(
norm_ref, bins=n_q, color=pf_color_picker.get(pf, "#1F77B4"), edgecolor="k"
norm_ref,
bins=n_q,
color=pf_color_picker.get(pf, "#1F77B4"),
edgecolor="k",
)
ax4.hist(
norm_test, bins=n_q, color=pf_color_picker.get(pf, "#1F77B4"), edgecolor="k"
norm_test,
bins=n_q,
color=pf_color_picker.get(pf, "#1F77B4"),
edgecolor="k",
)

# Check if these distributions are wildly different. If they are, use different
Expand Down
Loading

0 comments on commit 43e3117

Please sign in to comment.